Merge branch 'stats_histo' into write_completion_stats

BUILD

@@ -1597,4 +1597,19 @@ grpc_cc_library(
     ],
 )
 
+grpc_cc_library(
+    name = "grpc++_core_stats",
+    srcs = [
+        "src/cpp/util/core_stats.cc",
+    ],
+    hdrs = [
+        "src/cpp/util/core_stats.h",
+    ],
+    language = "c++",
+    deps = [
+        ":grpc++",
+        "//src/proto/grpc/core:stats_proto",
+    ],
+)
+
 grpc_generate_one_off_targets()

CMakeLists.txt

@@ -762,6 +762,7 @@ endif()
 add_dependencies(buildtests_cxx server_crash_test_client)
 add_dependencies(buildtests_cxx server_request_call_test)
 add_dependencies(buildtests_cxx shutdown_test)
+add_dependencies(buildtests_cxx stats_test)
 add_dependencies(buildtests_cxx status_test)
 if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX)
 add_dependencies(buildtests_cxx streaming_throughput_test)
@@ -2766,6 +2767,68 @@ if (gRPC_INSTALL)
   )
 endif()
 
+if (gRPC_BUILD_TESTS)
+
+add_library(grpc++_core_stats
+  ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/core/stats.pb.cc
+  ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/core/stats.grpc.pb.cc
+  ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/core/stats.pb.h
+  ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/core/stats.grpc.pb.h
+  src/cpp/util/core_stats.cc
+)
+
+if(WIN32 AND MSVC)
+  set_target_properties(grpc++_core_stats PROPERTIES COMPILE_PDB_NAME "grpc++_core_stats"
+    COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}"
+  )
+  if (gRPC_INSTALL)
+    install(FILES ${CMAKE_CURRENT_BINARY_DIR}/grpc++_core_stats.pdb
+      DESTINATION ${gRPC_INSTALL_LIBDIR} OPTIONAL
+    )
+  endif()
+endif()
+
+protobuf_generate_grpc_cpp(
+  src/proto/grpc/core/stats.proto
+)
+
+target_include_directories(grpc++_core_stats
+  PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
+  PRIVATE ${BORINGSSL_ROOT_DIR}/include
+  PRIVATE ${PROTOBUF_ROOT_DIR}/src
+  PRIVATE ${ZLIB_INCLUDE_DIR}
+  PRIVATE ${BENCHMARK}/include
+  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
+  PRIVATE ${CARES_BUILD_INCLUDE_DIR}
+  PRIVATE ${CARES_INCLUDE_DIR}
+  PRIVATE ${CARES_PLATFORM_INCLUDE_DIR}
+  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
+  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE third_party/googletest/googletest/include
+  PRIVATE third_party/googletest/googletest
+  PRIVATE third_party/googletest/googlemock/include
+  PRIVATE third_party/googletest/googlemock
+  PRIVATE ${_gRPC_PROTO_GENS_DIR}
+)
+
+target_link_libraries(grpc++_core_stats
+  ${_gRPC_PROTOBUF_LIBRARIES}
+  ${_gRPC_ALLTARGETS_LIBRARIES}
+  grpc++
+)
+
+foreach(_hdr
+  src/cpp/util/core_stats.h
+)
+  string(REPLACE "include/" "" _path ${_hdr})
+  get_filename_component(_path ${_path} PATH)
+  install(FILES ${_hdr}
+    DESTINATION "${gRPC_INSTALL_INCLUDEDIR}/${_path}"
+  )
+endforeach()
+
+endif (gRPC_BUILD_TESTS)
 
 add_library(grpc++_cronet
   src/cpp/client/cronet_credentials.cc
@@ -4590,6 +4653,7 @@ target_link_libraries(qps
   ${_gRPC_ALLTARGETS_LIBRARIES}
   grpc_test_util
   grpc++_test_util
+  grpc++_core_stats
   grpc++
   grpc
 )
@@ -10458,6 +10522,7 @@ target_include_directories(codegen_test_full
 target_link_libraries(codegen_test_full
   ${_gRPC_PROTOBUF_LIBRARIES}
   ${_gRPC_ALLTARGETS_LIBRARIES}
+  grpc++_core_stats
   grpc++
   grpc
   gpr
@@ -10533,6 +10598,7 @@ target_include_directories(codegen_test_minimal
 target_link_libraries(codegen_test_minimal
   ${_gRPC_PROTOBUF_LIBRARIES}
   ${_gRPC_ALLTARGETS_LIBRARIES}
+  grpc++_core_stats
   grpc
   gpr
   ${_gRPC_GFLAGS_LIBRARIES}
@@ -12104,6 +12170,7 @@ target_link_libraries(qps_json_driver
   ${_gRPC_PROTOBUF_LIBRARIES}
   ${_gRPC_ALLTARGETS_LIBRARIES}
   qps
+  grpc++_core_stats
   grpc++_test_util
   grpc_test_util
   grpc++
@@ -12149,6 +12216,7 @@ target_link_libraries(qps_openloop_test
   ${_gRPC_PROTOBUF_LIBRARIES}
   ${_gRPC_ALLTARGETS_LIBRARIES}
   qps
+  grpc++_core_stats
   grpc++_test_util
   grpc_test_util
   grpc++
@@ -12194,6 +12262,7 @@ target_link_libraries(qps_worker
   ${_gRPC_PROTOBUF_LIBRARIES}
   ${_gRPC_ALLTARGETS_LIBRARIES}
   qps
+  grpc++_core_stats
   grpc++_test_util
   grpc_test_util
   grpc++
@@ -12411,6 +12480,7 @@ target_link_libraries(secure_sync_unary_ping_pong_test
   ${_gRPC_PROTOBUF_LIBRARIES}
   ${_gRPC_ALLTARGETS_LIBRARIES}
   qps
+  grpc++_core_stats
   grpc++_test_util
   grpc_test_util
   grpc++
@@ -12748,6 +12818,47 @@ target_link_libraries(shutdown_test
 endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 
+add_executable(stats_test
+  test/core/debug/stats_test.cc
+  third_party/googletest/googletest/src/gtest-all.cc
+  third_party/googletest/googlemock/src/gmock-all.cc
+)
+
+
+target_include_directories(stats_test
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
+  PRIVATE ${BORINGSSL_ROOT_DIR}/include
+  PRIVATE ${PROTOBUF_ROOT_DIR}/src
+  PRIVATE ${BENCHMARK_ROOT_DIR}/include
+  PRIVATE ${ZLIB_ROOT_DIR}
+  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
+  PRIVATE ${CARES_BUILD_INCLUDE_DIR}
+  PRIVATE ${CARES_INCLUDE_DIR}
+  PRIVATE ${CARES_PLATFORM_INCLUDE_DIR}
+  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
+  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE third_party/googletest/googletest/include
+  PRIVATE third_party/googletest/googletest
+  PRIVATE third_party/googletest/googlemock/include
+  PRIVATE third_party/googletest/googlemock
+  PRIVATE ${_gRPC_PROTO_GENS_DIR}
+)
+
+target_link_libraries(stats_test
+  ${_gRPC_PROTOBUF_LIBRARIES}
+  ${_gRPC_ALLTARGETS_LIBRARIES}
+  grpc++_test_util
+  grpc_test_util
+  grpc
+  gpr_test_util
+  gpr
+  ${_gRPC_GFLAGS_LIBRARIES}
+)
+
+endif (gRPC_BUILD_TESTS)
+if (gRPC_BUILD_TESTS)
+
 add_executable(status_test
   test/cpp/util/status_test.cc
   third_party/googletest/googletest/src/gtest-all.cc

build.yaml

@@ -1328,6 +1328,16 @@ libs:
   - grpc++_codegen_base_src
   secure: check
   vs_project_guid: '{C187A093-A0FE-489D-A40A-6E33DE0F9FEB}'
+- name: grpc++_core_stats
+  build: private
+  language: c++
+  public_headers:
+  - src/cpp/util/core_stats.h
+  src:
+  - src/proto/grpc/core/stats.proto
+  - src/cpp/util/core_stats.cc
+  deps:
+  - grpc++
 - name: grpc++_cronet
   build: all
   language: c++
@@ -1670,6 +1680,7 @@ libs:
   deps:
   - grpc_test_util
   - grpc++_test_util
+  - grpc++_core_stats
   - grpc++
   - grpc
 - name: grpc_csharp_ext
@@ -3831,6 +3842,7 @@ targets:
   - src/proto/grpc/testing/stats.proto
   - test/cpp/codegen/codegen_test_full.cc
   deps:
+  - grpc++_core_stats
   - grpc++
   - grpc
   - gpr
@@ -3848,6 +3860,7 @@ targets:
   - src/proto/grpc/testing/stats.proto
   - test/cpp/codegen/codegen_test_minimal.cc
   deps:
+  - grpc++_core_stats
   - grpc
   - gpr
   filegroups:
@@ -4338,6 +4351,7 @@ targets:
   - test/cpp/qps/qps_json_driver.cc
   deps:
   - qps
+  - grpc++_core_stats
   - grpc++_test_util
   - grpc_test_util
   - grpc++
@@ -4353,6 +4367,7 @@ targets:
   - test/cpp/qps/qps_openloop_test.cc
   deps:
   - qps
+  - grpc++_core_stats
   - grpc++_test_util
   - grpc_test_util
   - grpc++
@@ -4375,6 +4390,7 @@ targets:
   - test/cpp/qps/worker.cc
   deps:
   - qps
+  - grpc++_core_stats
   - grpc++_test_util
   - grpc_test_util
   - grpc++
@@ -4438,6 +4454,7 @@ targets:
   - test/cpp/qps/secure_sync_unary_ping_pong_test.cc
   deps:
   - qps
+  - grpc++_core_stats
   - grpc++_test_util
   - grpc_test_util
   - grpc++
@@ -4550,6 +4567,18 @@ targets:
   - grpc
   - gpr_test_util
   - gpr
+- name: stats_test
+  gtest: true
+  build: test
+  language: c++
+  src:
+  - test/core/debug/stats_test.cc
+  deps:
+  - grpc++_test_util
+  - grpc_test_util
+  - grpc
+  - gpr_test_util
+  - gpr
 - name: status_test
   build: test
   language: c++

grpc.gyp

@@ -1218,6 +1218,17 @@
         'src/cpp/codegen/codegen_init.cc',
       ],
     },
+    {
+      'target_name': 'grpc++_core_stats',
+      'type': 'static_library',
+      'dependencies': [
+        'grpc++',
+      ],
+      'sources': [
+        'src/proto/grpc/core/stats.proto',
+        'src/cpp/util/core_stats.cc',
+      ],
+    },
     {
       'target_name': 'grpc++_error_details',
       'type': 'static_library',
@@ -1500,6 +1511,7 @@
       'dependencies': [
         'grpc_test_util',
         'grpc++_test_util',
+        'grpc++_core_stats',
         'grpc++',
         'grpc',
       ],

src/core/ext/transport/chttp2/transport/chttp2_transport.c

@@ -34,6 +34,7 @@
 #include "src/core/ext/transport/chttp2/transport/varint.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/compression/stream_compression.h"
+#include "src/core/lib/debug/stats.h"
 #include "src/core/lib/http/parser.h"
 #include "src/core/lib/iomgr/executor.h"
 #include "src/core/lib/iomgr/timer.h"
@@ -1297,6 +1298,8 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op,
   grpc_transport_stream_op_batch_payload *op_payload = op->payload;
   grpc_chttp2_transport *t = s->t;
 
+  GRPC_STATS_INC_HTTP2_OP_BATCHES(exec_ctx);
+
   if (GRPC_TRACER_ON(grpc_http_trace)) {
     char *str = grpc_transport_stream_op_batch_string(op);
     gpr_log(GPR_DEBUG, "perform_stream_op_locked: %s; on_complete = %p", str,
@@ -1330,11 +1333,13 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op,
   }
 
   if (op->cancel_stream) {
+    GRPC_STATS_INC_HTTP2_OP_CANCEL(exec_ctx);
     grpc_chttp2_cancel_stream(exec_ctx, t, s,
                               op_payload->cancel_stream.cancel_error);
   }
 
   if (op->send_initial_metadata) {
+    GRPC_STATS_INC_HTTP2_OP_SEND_INITIAL_METADATA(exec_ctx);
     GPR_ASSERT(s->send_initial_metadata_finished == NULL);
     on_complete->next_data.scratch |= CLOSURE_BARRIER_MAY_COVER_WRITE;
 
@@ -1411,6 +1416,9 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op,
   }
 
   if (op->send_message) {
+    GRPC_STATS_INC_HTTP2_OP_SEND_MESSAGE(exec_ctx);
+    GRPC_STATS_INC_HTTP2_SEND_MESSAGE_SIZE(
+        exec_ctx, op->payload->send_message.send_message->length);
     on_complete->next_data.scratch |= CLOSURE_BARRIER_MAY_COVER_WRITE;
     s->fetching_send_message_finished = add_closure_barrier(op->on_complete);
     if (s->write_closed) {
@@ -1448,6 +1456,7 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op,
   }
 
   if (op->send_trailing_metadata) {
+    GRPC_STATS_INC_HTTP2_OP_SEND_TRAILING_METADATA(exec_ctx);
     GPR_ASSERT(s->send_trailing_metadata_finished == NULL);
     on_complete->next_data.scratch |= CLOSURE_BARRIER_MAY_COVER_WRITE;
     s->send_trailing_metadata_finished = add_closure_barrier(on_complete);
@@ -1496,6 +1505,7 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op,
   }
 
   if (op->recv_initial_metadata) {
+    GRPC_STATS_INC_HTTP2_OP_RECV_INITIAL_METADATA(exec_ctx);
     GPR_ASSERT(s->recv_initial_metadata_ready == NULL);
     s->recv_initial_metadata_ready =
         op_payload->recv_initial_metadata.recv_initial_metadata_ready;
@@ -1507,6 +1517,7 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op,
   }
 
   if (op->recv_message) {
+    GRPC_STATS_INC_HTTP2_OP_RECV_MESSAGE(exec_ctx);
     size_t already_received;
     GPR_ASSERT(s->recv_message_ready == NULL);
     GPR_ASSERT(!s->pending_byte_stream);
@@ -1528,6 +1539,7 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op,
   }
 
   if (op->recv_trailing_metadata) {
+    GRPC_STATS_INC_HTTP2_OP_RECV_TRAILING_METADATA(exec_ctx);
     GPR_ASSERT(s->recv_trailing_metadata_finished == NULL);
     s->recv_trailing_metadata_finished = add_closure_barrier(on_complete);
     s->recv_trailing_metadata =

src/core/ext/transport/chttp2/transport/writing.c

@@ -22,6 +22,7 @@
 
 #include <grpc/support/log.h>
 
+#include "src/core/lib/debug/stats.h"
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/transport/http2_errors.h"
@@ -116,6 +117,7 @@ static void maybe_initiate_ping(grpc_exec_ctx *exec_ctx,
                          &pq->lists[GRPC_CHTTP2_PCL_INFLIGHT]);
   grpc_slice_buffer_add(&t->outbuf,
                         grpc_chttp2_ping_create(false, pq->inflight_id));
+  GRPC_STATS_INC_HTTP2_PINGS_SENT(exec_ctx);
   t->ping_state.last_ping_sent_time = now;
   t->ping_state.pings_before_data_required -=
       (t->ping_state.pings_before_data_required != 0);
@@ -166,6 +168,8 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write(
     grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t) {
   grpc_chttp2_stream *s;
 
+  GRPC_STATS_INC_HTTP2_WRITES_BEGUN(exec_ctx);
+
   GPR_TIMER_BEGIN("grpc_chttp2_begin_write", 0);
 
   if (t->dirtied_local_settings && !t->sent_local_settings) {

src/core/lib/debug/stats.c

@@ -45,7 +45,95 @@ void grpc_stats_collect(grpc_stats_data *output) {
       output->counters[i] += gpr_atm_no_barrier_load(
           &grpc_stats_per_cpu_storage[core].counters[i]);
     }
+    for (size_t i = 0; i < GRPC_STATS_HISTOGRAM_BUCKETS; i++) {
+      output->histograms[i] += gpr_atm_no_barrier_load(
+          &grpc_stats_per_cpu_storage[core].histograms[i]);
+    }
+  }
+}
+
+void grpc_stats_diff(const grpc_stats_data *b, const grpc_stats_data *a,
+                     grpc_stats_data *c) {
+  for (size_t i = 0; i < GRPC_STATS_COUNTER_COUNT; i++) {
+    c->counters[i] = b->counters[i] - a->counters[i];
+  }
+  for (size_t i = 0; i < GRPC_STATS_HISTOGRAM_BUCKETS; i++) {
+    c->histograms[i] = b->histograms[i] - a->histograms[i];
+  }
+}
+
+int grpc_stats_histo_find_bucket_slow(grpc_exec_ctx *exec_ctx, int value,
+                                      const int *table, int table_size) {
+  GRPC_STATS_INC_HISTOGRAM_SLOW_LOOKUPS(exec_ctx);
+  const int *const start = table;
+  while (table_size > 0) {
+    int step = table_size / 2;
+    const int *it = table + step;
+    if (value >= *it) {
+      table = it + 1;
+      table_size -= step + 1;
+    } else {
+      table_size = step;
+    }
+  }
+  return (int)(table - start) - 1;
+}
+
+size_t grpc_stats_histo_count(const grpc_stats_data *stats,
+                              grpc_stats_histograms histogram) {
+  size_t sum = 0;
+  for (int i = 0; i < grpc_stats_histo_buckets[histogram]; i++) {
+    sum += (size_t)stats->histograms[grpc_stats_histo_start[histogram] + i];
+  }
+  return sum;
+}
+
+static double threshold_for_count_below(const gpr_atm *bucket_counts,
+                                        const int *bucket_boundaries,
+                                        int num_buckets, double count_below) {
+  double count_so_far;
+  double lower_bound;
+  double upper_bound;
+  int lower_idx;
+  int upper_idx;
+
+  /* find the lowest bucket that gets us above count_below */
+  count_so_far = 0.0;
+  for (lower_idx = 0; lower_idx < num_buckets; lower_idx++) {
+    count_so_far += (double)bucket_counts[lower_idx];
+    if (count_so_far >= count_below) {
+      break;
+    }
   }
+  if (count_so_far == count_below) {
+    /* this bucket hits the threshold exactly... we should be midway through
+       any run of zero values following the bucket */
+    for (upper_idx = lower_idx + 1; upper_idx < num_buckets; upper_idx++) {
+      if (bucket_counts[upper_idx]) {
+        break;
+      }
+    }
+    return (bucket_boundaries[lower_idx] + bucket_boundaries[upper_idx]) / 2.0;
+  } else {
+    /* treat values as uniform throughout the bucket, and find where this value
+       should lie */
+    lower_bound = bucket_boundaries[lower_idx];
+    upper_bound = bucket_boundaries[lower_idx + 1];
+    return upper_bound -
+           (upper_bound - lower_bound) * (count_so_far - count_below) /
+               (double)bucket_counts[lower_idx];
+  }
+}
+
+double grpc_stats_histo_percentile(const grpc_stats_data *stats,
+                                   grpc_stats_histograms histogram,
+                                   double percentile) {
+  size_t count = grpc_stats_histo_count(stats, histogram);
+  if (count == 0) return 0.0;
+  return threshold_for_count_below(
+      stats->histograms + grpc_stats_histo_start[histogram],
+      grpc_stats_histo_bucket_boundaries[histogram],
+      grpc_stats_histo_buckets[histogram], (double)count * percentile / 100.0);
 }
 
 char *grpc_stats_data_as_json(const grpc_stats_data *data) {
@@ -60,6 +148,25 @@ char *grpc_stats_data_as_json(const grpc_stats_data *data) {
     gpr_strvec_add(&v, tmp);
     is_first = false;
   }
+  for (size_t i = 0; i < GRPC_STATS_HISTOGRAM_COUNT; i++) {
+    gpr_asprintf(&tmp, "%s\"%s\": [", is_first ? "" : ", ",
+                 grpc_stats_histogram_name[i]);
+    gpr_strvec_add(&v, tmp);
+    for (int j = 0; j < grpc_stats_histo_buckets[i]; j++) {
+      gpr_asprintf(&tmp, "%s%" PRIdPTR, j == 0 ? "" : ",",
+                   data->histograms[grpc_stats_histo_start[i] + j]);
+      gpr_strvec_add(&v, tmp);
+    }
+    gpr_asprintf(&tmp, "], \"%s_bkt\": [", grpc_stats_histogram_name[i]);
+    gpr_strvec_add(&v, tmp);
+    for (int j = 0; j < grpc_stats_histo_buckets[i]; j++) {
+      gpr_asprintf(&tmp, "%s%d", j == 0 ? "" : ",",
+                   grpc_stats_histo_bucket_boundaries[i][j]);
+      gpr_strvec_add(&v, tmp);
+    }
+    gpr_strvec_add(&v, gpr_strdup("]"));
+    is_first = false;
+  }
   gpr_strvec_add(&v, gpr_strdup("}"));
   tmp = gpr_strvec_flatten(&v, NULL);
   gpr_strvec_destroy(&v);

src/core/lib/debug/stats.h

@@ -25,6 +25,7 @@
 
 typedef struct grpc_stats_data {
   gpr_atm counters[GRPC_STATS_COUNTER_COUNT];
+  gpr_atm histograms[GRPC_STATS_HISTOGRAM_BUCKETS];
 } grpc_stats_data;
 
 extern grpc_stats_data *grpc_stats_per_cpu_storage;
@@ -36,9 +37,25 @@ extern grpc_stats_data *grpc_stats_per_cpu_storage;
   (gpr_atm_no_barrier_fetch_add(              \
       &GRPC_THREAD_STATS_DATA((exec_ctx))->counters[(ctr)], 1))
 
+#define GRPC_STATS_INC_HISTOGRAM(exec_ctx, histogram, index) \
+  (gpr_atm_no_barrier_fetch_add(                             \
+      &GRPC_THREAD_STATS_DATA((exec_ctx))                    \
+           ->histograms[histogram##_FIRST_SLOT + (index)],   \
+      1))
+
 void grpc_stats_init(void);
 void grpc_stats_shutdown(void);
 void grpc_stats_collect(grpc_stats_data *output);
+// c = b-a
+void grpc_stats_diff(const grpc_stats_data *b, const grpc_stats_data *a,
+                     grpc_stats_data *c);
 char *grpc_stats_data_as_json(const grpc_stats_data *data);
+int grpc_stats_histo_find_bucket_slow(grpc_exec_ctx *exec_ctx, int value,
+                                      const int *table, int table_size);
+double grpc_stats_histo_percentile(const grpc_stats_data *data,
+                                   grpc_stats_histograms histogram,
+                                   double percentile);
+size_t grpc_stats_histo_count(const grpc_stats_data *data,
+                              grpc_stats_histograms histogram);
 
 #endif

src/core/lib/debug/stats_data.c

@@ -19,7 +19,227 @@
  */
 
 #include "src/core/lib/debug/stats_data.h"
+#include <grpc/support/useful.h>
+#include "src/core/lib/debug/stats.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
 const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT] = {
-    "client_calls_created", "server_calls_created", "syscall_write",
-    "syscall_read",         "syscall_poll",         "syscall_wait",
+    "client_calls_created",
+    "server_calls_created",
+    "syscall_poll",
+    "syscall_wait",
+    "histogram_slow_lookups",
+    "syscall_write",
+    "syscall_read",
+    "http2_op_batches",
+    "http2_op_cancel",
+    "http2_op_send_initial_metadata",
+    "http2_op_send_message",
+    "http2_op_send_trailing_metadata",
+    "http2_op_recv_initial_metadata",
+    "http2_op_recv_message",
+    "http2_op_recv_trailing_metadata",
+    "http2_pings_sent",
+    "http2_writes_begun",
+    "combiner_locks_initiated",
+    "combiner_locks_scheduled_items",
+    "combiner_locks_scheduled_final_items",
+    "combiner_locks_offloaded",
+    "executor_scheduled_items",
+    "executor_scheduled_to_self",
+    "executor_wakeup_initiated",
+    "executor_queue_drained",
 };
+const char *grpc_stats_histogram_name[GRPC_STATS_HISTOGRAM_COUNT] = {
+    "tcp_write_size", "tcp_write_iov_size", "tcp_read_size",
+    "tcp_read_offer", "tcp_read_iov_size",  "http2_send_message_size",
+};
+const int grpc_stats_table_0[65] = {
+    0,       1,       2,       3,       4,       6,       8,        11,
+    15,      20,      26,      34,      44,      57,      73,       94,
+    121,     155,     199,     255,     327,     419,     537,      688,
+    881,     1128,    1444,    1848,    2365,    3026,    3872,     4954,
+    6338,    8108,    10373,   13270,   16976,   21717,   27782,    35541,
+    45467,   58165,   74409,   95189,   121772,  155778,  199281,   254933,
+    326126,  417200,  533707,  682750,  873414,  1117323, 1429345,  1828502,
+    2339127, 2992348, 3827987, 4896985, 6264509, 8013925, 10251880, 13114801,
+    16777216};
+const uint8_t grpc_stats_table_1[87] = {
+    0,  0,  1,  1,  2,  3,  3,  4,  4,  5,  6,  6,  7,  8,  8,  9,  10, 11,
+    11, 12, 13, 13, 14, 15, 15, 16, 17, 17, 18, 19, 20, 20, 21, 22, 22, 23,
+    24, 25, 25, 26, 27, 27, 28, 29, 29, 30, 31, 31, 32, 33, 34, 34, 35, 36,
+    36, 37, 38, 39, 39, 40, 41, 41, 42, 43, 44, 44, 45, 45, 46, 47, 48, 48,
+    49, 50, 51, 51, 52, 53, 53, 54, 55, 56, 56, 57, 58, 58, 59};
+const int grpc_stats_table_2[65] = {
+    0,   1,   2,   3,   4,   5,   6,   7,   8,   9,   10,  11,  12,
+    14,  16,  18,  20,  22,  24,  27,  30,  33,  36,  39,  43,  47,
+    51,  56,  61,  66,  72,  78,  85,  92,  100, 109, 118, 128, 139,
+    151, 164, 178, 193, 209, 226, 244, 264, 285, 308, 333, 359, 387,
+    418, 451, 486, 524, 565, 609, 656, 707, 762, 821, 884, 952, 1024};
+const uint8_t grpc_stats_table_3[102] = {
+    0,  0,  0,  1,  1,  1,  1,  2,  2,  3,  3,  4,  4,  5,  5,  6,  6,
+    6,  7,  7,  7,  8,  8,  9,  9,  10, 11, 11, 12, 12, 13, 13, 14, 14,
+    14, 15, 15, 16, 16, 17, 17, 18, 19, 19, 20, 20, 21, 21, 22, 22, 23,
+    23, 24, 24, 24, 25, 26, 27, 27, 28, 28, 29, 29, 30, 30, 31, 31, 32,
+    32, 33, 33, 34, 35, 35, 36, 37, 37, 38, 38, 39, 39, 40, 40, 41, 41,
+    42, 42, 43, 44, 44, 45, 46, 46, 47, 48, 48, 49, 49, 50, 50, 51, 51};
+void grpc_stats_inc_tcp_write_size(grpc_exec_ctx *exec_ctx, int value) {
+  value = GPR_CLAMP(value, 0, 16777216);
+  if (value < 5) {
+    GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE,
+                             value);
+    return;
+  }
+  union {
+    double dbl;
+    uint64_t uint;
+  } _val, _bkt;
+  _val.dbl = value;
+  if (_val.uint < 4683743612465315840ull) {
+    int bucket =
+        grpc_stats_table_1[((_val.uint - 4617315517961601024ull) >> 50)] + 5;
+    _bkt.dbl = grpc_stats_table_0[bucket];
+    bucket -= (_val.uint < _bkt.uint);
+    GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE,
+                             bucket);
+    return;
+  }
+  GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE,
+                           grpc_stats_histo_find_bucket_slow(
+                               (exec_ctx), value, grpc_stats_table_0, 64));
+}
+void grpc_stats_inc_tcp_write_iov_size(grpc_exec_ctx *exec_ctx, int value) {
+  value = GPR_CLAMP(value, 0, 1024);
+  if (value < 13) {
+    GRPC_STATS_INC_HISTOGRAM((exec_ctx),
+                             GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE, value);
+    return;
+  }
+  union {
+    double dbl;
+    uint64_t uint;
+  } _val, _bkt;
+  _val.dbl = value;
+  if (_val.uint < 4637863191261478912ull) {
+    int bucket =
+        grpc_stats_table_3[((_val.uint - 4623507967449235456ull) >> 48)] + 13;
+    _bkt.dbl = grpc_stats_table_2[bucket];
+    bucket -= (_val.uint < _bkt.uint);
+    GRPC_STATS_INC_HISTOGRAM((exec_ctx),
+                             GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE, bucket);
+    return;
+  }
+  GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE,
+                           grpc_stats_histo_find_bucket_slow(
+                               (exec_ctx), value, grpc_stats_table_2, 64));
+}
+void grpc_stats_inc_tcp_read_size(grpc_exec_ctx *exec_ctx, int value) {
+  value = GPR_CLAMP(value, 0, 16777216);
+  if (value < 5) {
+    GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_SIZE,
+                             value);
+    return;
+  }
+  union {
+    double dbl;
+    uint64_t uint;
+  } _val, _bkt;
+  _val.dbl = value;
+  if (_val.uint < 4683743612465315840ull) {
+    int bucket =
+        grpc_stats_table_1[((_val.uint - 4617315517961601024ull) >> 50)] + 5;
+    _bkt.dbl = grpc_stats_table_0[bucket];
+    bucket -= (_val.uint < _bkt.uint);
+    GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_SIZE,
+                             bucket);
+    return;
+  }
+  GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_SIZE,
+                           grpc_stats_histo_find_bucket_slow(
+                               (exec_ctx), value, grpc_stats_table_0, 64));
+}
+void grpc_stats_inc_tcp_read_offer(grpc_exec_ctx *exec_ctx, int value) {
+  value = GPR_CLAMP(value, 0, 16777216);
+  if (value < 5) {
+    GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_OFFER,
+                             value);
+    return;
+  }
+  union {
+    double dbl;
+    uint64_t uint;
+  } _val, _bkt;
+  _val.dbl = value;
+  if (_val.uint < 4683743612465315840ull) {
+    int bucket =
+        grpc_stats_table_1[((_val.uint - 4617315517961601024ull) >> 50)] + 5;
+    _bkt.dbl = grpc_stats_table_0[bucket];
+    bucket -= (_val.uint < _bkt.uint);
+    GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_OFFER,
+                             bucket);
+    return;
+  }
+  GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_OFFER,
+                           grpc_stats_histo_find_bucket_slow(
+                               (exec_ctx), value, grpc_stats_table_0, 64));
+}
+void grpc_stats_inc_tcp_read_iov_size(grpc_exec_ctx *exec_ctx, int value) {
+  value = GPR_CLAMP(value, 0, 1024);
+  if (value < 13) {
+    GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_IOV_SIZE,
+                             value);
+    return;
+  }
+  union {
+    double dbl;
+    uint64_t uint;
+  } _val, _bkt;
+  _val.dbl = value;
+  if (_val.uint < 4637863191261478912ull) {
+    int bucket =
+        grpc_stats_table_3[((_val.uint - 4623507967449235456ull) >> 48)] + 13;
+    _bkt.dbl = grpc_stats_table_2[bucket];
+    bucket -= (_val.uint < _bkt.uint);
+    GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_IOV_SIZE,
+                             bucket);
+    return;
+  }
+  GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_IOV_SIZE,
+                           grpc_stats_histo_find_bucket_slow(
+                               (exec_ctx), value, grpc_stats_table_2, 64));
+}
+void grpc_stats_inc_http2_send_message_size(grpc_exec_ctx *exec_ctx,
+                                            int value) {
+  value = GPR_CLAMP(value, 0, 16777216);
+  if (value < 5) {
+    GRPC_STATS_INC_HISTOGRAM(
+        (exec_ctx), GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE, value);
+    return;
+  }
+  union {
+    double dbl;
+    uint64_t uint;
+  } _val, _bkt;
+  _val.dbl = value;
+  if (_val.uint < 4683743612465315840ull) {
+    int bucket =
+        grpc_stats_table_1[((_val.uint - 4617315517961601024ull) >> 50)] + 5;
+    _bkt.dbl = grpc_stats_table_0[bucket];
+    bucket -= (_val.uint < _bkt.uint);
+    GRPC_STATS_INC_HISTOGRAM(
+        (exec_ctx), GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE, bucket);
+    return;
+  }
+  GRPC_STATS_INC_HISTOGRAM((exec_ctx),
+                           GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE,
+                           grpc_stats_histo_find_bucket_slow(
+                               (exec_ctx), value, grpc_stats_table_0, 64));
+}
+const int grpc_stats_histo_buckets[6] = {64, 64, 64, 64, 64, 64};
+const int grpc_stats_histo_start[6] = {0, 64, 128, 192, 256, 320};
+const int *const grpc_stats_histo_bucket_boundaries[6] = {
+    grpc_stats_table_0, grpc_stats_table_2, grpc_stats_table_0,
+    grpc_stats_table_0, grpc_stats_table_2, grpc_stats_table_0};
+void (*const grpc_stats_inc_histogram[6])(grpc_exec_ctx *exec_ctx, int x) = {
+    grpc_stats_inc_tcp_write_size,    grpc_stats_inc_tcp_write_iov_size,
+    grpc_stats_inc_tcp_read_size,     grpc_stats_inc_tcp_read_offer,
+    grpc_stats_inc_tcp_read_iov_size, grpc_stats_inc_http2_send_message_size};

src/core/lib/debug/stats_data.h

@@ -21,27 +21,146 @@
 #ifndef GRPC_CORE_LIB_DEBUG_STATS_DATA_H
 #define GRPC_CORE_LIB_DEBUG_STATS_DATA_H
 
+#include <inttypes.h>
+#include "src/core/lib/iomgr/exec_ctx.h"
+
 typedef enum {
   GRPC_STATS_COUNTER_CLIENT_CALLS_CREATED,
   GRPC_STATS_COUNTER_SERVER_CALLS_CREATED,
-  GRPC_STATS_COUNTER_SYSCALL_WRITE,
-  GRPC_STATS_COUNTER_SYSCALL_READ,
   GRPC_STATS_COUNTER_SYSCALL_POLL,
   GRPC_STATS_COUNTER_SYSCALL_WAIT,
+  GRPC_STATS_COUNTER_HISTOGRAM_SLOW_LOOKUPS,
+  GRPC_STATS_COUNTER_SYSCALL_WRITE,
+  GRPC_STATS_COUNTER_SYSCALL_READ,
+  GRPC_STATS_COUNTER_HTTP2_OP_BATCHES,
+  GRPC_STATS_COUNTER_HTTP2_OP_CANCEL,
+  GRPC_STATS_COUNTER_HTTP2_OP_SEND_INITIAL_METADATA,
+  GRPC_STATS_COUNTER_HTTP2_OP_SEND_MESSAGE,
+  GRPC_STATS_COUNTER_HTTP2_OP_SEND_TRAILING_METADATA,
+  GRPC_STATS_COUNTER_HTTP2_OP_RECV_INITIAL_METADATA,
+  GRPC_STATS_COUNTER_HTTP2_OP_RECV_MESSAGE,
+  GRPC_STATS_COUNTER_HTTP2_OP_RECV_TRAILING_METADATA,
+  GRPC_STATS_COUNTER_HTTP2_PINGS_SENT,
+  GRPC_STATS_COUNTER_HTTP2_WRITES_BEGUN,
+  GRPC_STATS_COUNTER_COMBINER_LOCKS_INITIATED,
+  GRPC_STATS_COUNTER_COMBINER_LOCKS_SCHEDULED_ITEMS,
+  GRPC_STATS_COUNTER_COMBINER_LOCKS_SCHEDULED_FINAL_ITEMS,
+  GRPC_STATS_COUNTER_COMBINER_LOCKS_OFFLOADED,
+  GRPC_STATS_COUNTER_EXECUTOR_SCHEDULED_ITEMS,
+  GRPC_STATS_COUNTER_EXECUTOR_SCHEDULED_TO_SELF,
+  GRPC_STATS_COUNTER_EXECUTOR_WAKEUP_INITIATED,
+  GRPC_STATS_COUNTER_EXECUTOR_QUEUE_DRAINED,
   GRPC_STATS_COUNTER_COUNT
 } grpc_stats_counters;
+extern const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT];
+typedef enum {
+  GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE,
+  GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE,
+  GRPC_STATS_HISTOGRAM_TCP_READ_SIZE,
+  GRPC_STATS_HISTOGRAM_TCP_READ_OFFER,
+  GRPC_STATS_HISTOGRAM_TCP_READ_IOV_SIZE,
+  GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE,
+  GRPC_STATS_HISTOGRAM_COUNT
+} grpc_stats_histograms;
+extern const char *grpc_stats_histogram_name[GRPC_STATS_HISTOGRAM_COUNT];
+typedef enum {
+  GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE_FIRST_SLOT = 0,
+  GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE_BUCKETS = 64,
+  GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE_FIRST_SLOT = 64,
+  GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE_BUCKETS = 64,
+  GRPC_STATS_HISTOGRAM_TCP_READ_SIZE_FIRST_SLOT = 128,
+  GRPC_STATS_HISTOGRAM_TCP_READ_SIZE_BUCKETS = 64,
+  GRPC_STATS_HISTOGRAM_TCP_READ_OFFER_FIRST_SLOT = 192,
+  GRPC_STATS_HISTOGRAM_TCP_READ_OFFER_BUCKETS = 64,
+  GRPC_STATS_HISTOGRAM_TCP_READ_IOV_SIZE_FIRST_SLOT = 256,
+  GRPC_STATS_HISTOGRAM_TCP_READ_IOV_SIZE_BUCKETS = 64,
+  GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE_FIRST_SLOT = 320,
+  GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE_BUCKETS = 64,
+  GRPC_STATS_HISTOGRAM_BUCKETS = 384
+} grpc_stats_histogram_constants;
 #define GRPC_STATS_INC_CLIENT_CALLS_CREATED(exec_ctx) \
   GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_CLIENT_CALLS_CREATED)
 #define GRPC_STATS_INC_SERVER_CALLS_CREATED(exec_ctx) \
   GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SERVER_CALLS_CREATED)
-#define GRPC_STATS_INC_SYSCALL_WRITE(exec_ctx) \
-  GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SYSCALL_WRITE)
-#define GRPC_STATS_INC_SYSCALL_READ(exec_ctx) \
-  GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SYSCALL_READ)
 #define GRPC_STATS_INC_SYSCALL_POLL(exec_ctx) \
   GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SYSCALL_POLL)
 #define GRPC_STATS_INC_SYSCALL_WAIT(exec_ctx) \
   GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SYSCALL_WAIT)
-extern const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT];
+#define GRPC_STATS_INC_HISTOGRAM_SLOW_LOOKUPS(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HISTOGRAM_SLOW_LOOKUPS)
+#define GRPC_STATS_INC_SYSCALL_WRITE(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SYSCALL_WRITE)
+#define GRPC_STATS_INC_SYSCALL_READ(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SYSCALL_READ)
+#define GRPC_STATS_INC_HTTP2_OP_BATCHES(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HTTP2_OP_BATCHES)
+#define GRPC_STATS_INC_HTTP2_OP_CANCEL(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HTTP2_OP_CANCEL)
+#define GRPC_STATS_INC_HTTP2_OP_SEND_INITIAL_METADATA(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx),                            \
+                         GRPC_STATS_COUNTER_HTTP2_OP_SEND_INITIAL_METADATA)
+#define GRPC_STATS_INC_HTTP2_OP_SEND_MESSAGE(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HTTP2_OP_SEND_MESSAGE)
+#define GRPC_STATS_INC_HTTP2_OP_SEND_TRAILING_METADATA(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx),                             \
+                         GRPC_STATS_COUNTER_HTTP2_OP_SEND_TRAILING_METADATA)
+#define GRPC_STATS_INC_HTTP2_OP_RECV_INITIAL_METADATA(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx),                            \
+                         GRPC_STATS_COUNTER_HTTP2_OP_RECV_INITIAL_METADATA)
+#define GRPC_STATS_INC_HTTP2_OP_RECV_MESSAGE(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HTTP2_OP_RECV_MESSAGE)
+#define GRPC_STATS_INC_HTTP2_OP_RECV_TRAILING_METADATA(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx),                             \
+                         GRPC_STATS_COUNTER_HTTP2_OP_RECV_TRAILING_METADATA)
+#define GRPC_STATS_INC_HTTP2_PINGS_SENT(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HTTP2_PINGS_SENT)
+#define GRPC_STATS_INC_HTTP2_WRITES_BEGUN(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HTTP2_WRITES_BEGUN)
+#define GRPC_STATS_INC_COMBINER_LOCKS_INITIATED(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx),                      \
+                         GRPC_STATS_COUNTER_COMBINER_LOCKS_INITIATED)
+#define GRPC_STATS_INC_COMBINER_LOCKS_SCHEDULED_ITEMS(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx),                            \
+                         GRPC_STATS_COUNTER_COMBINER_LOCKS_SCHEDULED_ITEMS)
+#define GRPC_STATS_INC_COMBINER_LOCKS_SCHEDULED_FINAL_ITEMS(exec_ctx) \
+  GRPC_STATS_INC_COUNTER(                                             \
+      (exec_ctx), GRPC_STATS_COUNTER_COMBINER_LOCKS_SCHEDULED_FINAL_ITEMS)
+#define GRPC_STATS_INC_COMBINER_LOCKS_OFFLOADED(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx),                      \
+                         GRPC_STATS_COUNTER_COMBINER_LOCKS_OFFLOADED)
+#define GRPC_STATS_INC_EXECUTOR_SCHEDULED_ITEMS(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx),                      \
+                         GRPC_STATS_COUNTER_EXECUTOR_SCHEDULED_ITEMS)
+#define GRPC_STATS_INC_EXECUTOR_SCHEDULED_TO_SELF(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx),                        \
+                         GRPC_STATS_COUNTER_EXECUTOR_SCHEDULED_TO_SELF)
+#define GRPC_STATS_INC_EXECUTOR_WAKEUP_INITIATED(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx),                       \
+                         GRPC_STATS_COUNTER_EXECUTOR_WAKEUP_INITIATED)
+#define GRPC_STATS_INC_EXECUTOR_QUEUE_DRAINED(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_EXECUTOR_QUEUE_DRAINED)
+#define GRPC_STATS_INC_TCP_WRITE_SIZE(exec_ctx, value) \
+  grpc_stats_inc_tcp_write_size((exec_ctx), (int)(value))
+void grpc_stats_inc_tcp_write_size(grpc_exec_ctx *exec_ctx, int x);
+#define GRPC_STATS_INC_TCP_WRITE_IOV_SIZE(exec_ctx, value) \
+  grpc_stats_inc_tcp_write_iov_size((exec_ctx), (int)(value))
+void grpc_stats_inc_tcp_write_iov_size(grpc_exec_ctx *exec_ctx, int x);
+#define GRPC_STATS_INC_TCP_READ_SIZE(exec_ctx, value) \
+  grpc_stats_inc_tcp_read_size((exec_ctx), (int)(value))
+void grpc_stats_inc_tcp_read_size(grpc_exec_ctx *exec_ctx, int x);
+#define GRPC_STATS_INC_TCP_READ_OFFER(exec_ctx, value) \
+  grpc_stats_inc_tcp_read_offer((exec_ctx), (int)(value))
+void grpc_stats_inc_tcp_read_offer(grpc_exec_ctx *exec_ctx, int x);
+#define GRPC_STATS_INC_TCP_READ_IOV_SIZE(exec_ctx, value) \
+  grpc_stats_inc_tcp_read_iov_size((exec_ctx), (int)(value))
+void grpc_stats_inc_tcp_read_iov_size(grpc_exec_ctx *exec_ctx, int x);
+#define GRPC_STATS_INC_HTTP2_SEND_MESSAGE_SIZE(exec_ctx, value) \
+  grpc_stats_inc_http2_send_message_size((exec_ctx), (int)(value))
+void grpc_stats_inc_http2_send_message_size(grpc_exec_ctx *exec_ctx, int x);
+extern const int grpc_stats_histo_buckets[6];
+extern const int grpc_stats_histo_start[6];
+extern const int *const grpc_stats_histo_bucket_boundaries[6];
+extern void (*const grpc_stats_inc_histogram[6])(grpc_exec_ctx *exec_ctx,
+                                                 int x);
 
 #endif /* GRPC_CORE_LIB_DEBUG_STATS_DATA_H */

src/core/lib/debug/stats_data.yaml

@@ -1,9 +1,67 @@
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 # Stats data declaration
-# use tools/codegen/core/gen_stats_data.py to turn this into stats_data.h
+# use tools / codegen / core / gen_stats_data.py to turn this into stats_data.h
 
+# overall
 - counter: client_calls_created
 - counter: server_calls_created
-- counter: syscall_write
-- counter: syscall_read
+# polling
 - counter: syscall_poll
 - counter: syscall_wait
+# stats system
+- counter: histogram_slow_lookups
+# tcp
+- counter: syscall_write
+- counter: syscall_read
+- histogram: tcp_write_size
+  max: 16777216 # 16 meg max write tracked
+  buckets: 64
+- histogram: tcp_write_iov_size
+  max: 1024
+  buckets: 64
+- histogram: tcp_read_size
+  max: 16777216
+  buckets: 64
+- histogram: tcp_read_offer
+  max: 16777216
+  buckets: 64
+- histogram: tcp_read_iov_size
+  max: 1024
+  buckets: 64
+# chttp2
+- counter: http2_op_batches
+- counter: http2_op_cancel
+- counter: http2_op_send_initial_metadata
+- counter: http2_op_send_message
+- counter: http2_op_send_trailing_metadata
+- counter: http2_op_recv_initial_metadata
+- counter: http2_op_recv_message
+- counter: http2_op_recv_trailing_metadata
+- histogram: http2_send_message_size
+  max: 16777216
+  buckets: 64
+- counter: http2_pings_sent
+- counter: http2_writes_begun
+# combiner locks
+- counter: combiner_locks_initiated
+- counter: combiner_locks_scheduled_items
+- counter: combiner_locks_scheduled_final_items
+- counter: combiner_locks_offloaded
+# executor
+- counter: executor_scheduled_items
+- counter: executor_scheduled_to_self
+- counter: executor_wakeup_initiated
+- counter: executor_queue_drained

src/core/lib/iomgr/combiner.c

@@ -24,6 +24,7 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 
+#include "src/core/lib/debug/stats.h"
 #include "src/core/lib/iomgr/executor.h"
 #include "src/core/lib/profiling/timers.h"
 
@@ -154,6 +155,7 @@ static void push_first_on_exec_ctx(grpc_exec_ctx *exec_ctx,
 
 static void combiner_exec(grpc_exec_ctx *exec_ctx, grpc_closure *cl,
                           grpc_error *error) {
+  GRPC_STATS_INC_COMBINER_LOCKS_SCHEDULED_ITEMS(exec_ctx);
   GPR_TIMER_BEGIN("combiner.execute", 0);
   grpc_combiner *lock = COMBINER_FROM_CLOSURE_SCHEDULER(cl, scheduler);
   gpr_atm last = gpr_atm_full_fetch_add(&lock->state, STATE_ELEM_COUNT_LOW_BIT);
@@ -161,6 +163,7 @@ static void combiner_exec(grpc_exec_ctx *exec_ctx, grpc_closure *cl,
                               "C:%p grpc_combiner_execute c=%p last=%" PRIdPTR,
                               lock, cl, last));
   if (last == 1) {
+    GRPC_STATS_INC_COMBINER_LOCKS_INITIATED(exec_ctx);
     gpr_atm_no_barrier_store(&lock->initiating_exec_ctx_or_null,
                              (gpr_atm)exec_ctx);
     // first element on this list: add it to the list of combiner locks
@@ -196,6 +199,7 @@ static void offload(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
 }
 
 static void queue_offload(grpc_exec_ctx *exec_ctx, grpc_combiner *lock) {
+  GRPC_STATS_INC_COMBINER_LOCKS_OFFLOADED(exec_ctx);
   move_next(exec_ctx);
   GRPC_COMBINER_TRACE(gpr_log(GPR_DEBUG, "C:%p queue_offload", lock));
   GRPC_CLOSURE_SCHED(exec_ctx, &lock->offload, GRPC_ERROR_NONE);
@@ -326,6 +330,7 @@ static void enqueue_finally(grpc_exec_ctx *exec_ctx, void *closure,
 
 static void combiner_finally_exec(grpc_exec_ctx *exec_ctx,
                                   grpc_closure *closure, grpc_error *error) {
+  GRPC_STATS_INC_COMBINER_LOCKS_SCHEDULED_FINAL_ITEMS(exec_ctx);
   grpc_combiner *lock =
       COMBINER_FROM_CLOSURE_SCHEDULER(closure, finally_scheduler);
   GRPC_COMBINER_TRACE(gpr_log(GPR_DEBUG,

src/core/lib/iomgr/executor.c

@@ -28,6 +28,7 @@
 #include <grpc/support/tls.h>
 #include <grpc/support/useful.h>
 
+#include "src/core/lib/debug/stats.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
 #include "src/core/lib/support/spinlock.h"
 
@@ -168,6 +169,7 @@ static void executor_thread(void *arg) {
       gpr_mu_unlock(&ts->mu);
       break;
     }
+    GRPC_STATS_INC_EXECUTOR_QUEUE_DRAINED(&exec_ctx);
     grpc_closure_list exec = ts->elems;
     ts->elems = (grpc_closure_list)GRPC_CLOSURE_LIST_INIT;
     gpr_mu_unlock(&ts->mu);
@@ -183,6 +185,7 @@ static void executor_thread(void *arg) {
 static void executor_push(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
                           grpc_error *error, bool is_short) {
   bool retry_push;
+  GRPC_STATS_INC_EXECUTOR_SCHEDULED_ITEMS(exec_ctx);
   do {
     retry_push = false;
     size_t cur_thread_count = (size_t)gpr_atm_no_barrier_load(&g_cur_threads);
@@ -201,6 +204,8 @@ static void executor_push(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
     thread_state *ts = (thread_state *)gpr_tls_get(&g_this_thread_state);
     if (ts == NULL) {
       ts = &g_thread_state[GPR_HASH_POINTER(exec_ctx, cur_thread_count)];
+    } else {
+      GRPC_STATS_INC_EXECUTOR_SCHEDULED_TO_SELF(exec_ctx);
     }
     thread_state *orig_ts = ts;
 
@@ -232,6 +237,7 @@ static void executor_push(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
         continue;
       }
       if (grpc_closure_list_empty(ts->elems)) {
+        GRPC_STATS_INC_EXECUTOR_WAKEUP_INITIATED(exec_ctx);
         gpr_cv_signal(&ts->cv);
       }
       grpc_closure_list_append(&ts->elems, closure, error);

src/core/lib/iomgr/tcp_posix.c

@@ -396,6 +396,9 @@ static void tcp_do_read(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) {
   msg.msg_controllen = 0;
   msg.msg_flags = 0;
 
+  GRPC_STATS_INC_TCP_READ_OFFER(exec_ctx, tcp->incoming_buffer->length);
+  GRPC_STATS_INC_TCP_READ_IOV_SIZE(exec_ctx, tcp->incoming_buffer->count);
+
   GPR_TIMER_BEGIN("recvmsg", 0);
   do {
     GRPC_STATS_INC_SYSCALL_READ(exec_ctx);
@@ -426,6 +429,7 @@ static void tcp_do_read(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) {
             GRPC_ERROR_CREATE_FROM_STATIC_STRING("Socket closed"), tcp));
     TCP_UNREF(exec_ctx, tcp, "read");
   } else {
+    GRPC_STATS_INC_TCP_READ_SIZE(exec_ctx, read_bytes);
     add_to_estimate(tcp, (size_t)read_bytes);
     GPR_ASSERT((size_t)read_bytes <= tcp->incoming_buffer->length);
     if ((size_t)read_bytes < tcp->incoming_buffer->length) {
@@ -555,6 +559,9 @@ static bool tcp_flush(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp,
     msg.msg_controllen = 0;
     msg.msg_flags = 0;
 
+    GRPC_STATS_INC_TCP_WRITE_SIZE(exec_ctx, sending_length);
+    GRPC_STATS_INC_TCP_WRITE_IOV_SIZE(exec_ctx, iov_size);
+
     GPR_TIMER_BEGIN("sendmsg", 1);
     do {
       /* TODO(klempner): Cork if this is a partial write */

src/cpp/util/core_stats.cc

@@ -0,0 +1,90 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "src/cpp/util/core_stats.h"
+
+#include <grpc/support/log.h>
+
+using grpc::core::Bucket;
+using grpc::core::Histogram;
+using grpc::core::Metric;
+using grpc::core::Stats;
+
+namespace grpc {
+
+void CoreStatsToProto(const grpc_stats_data& core, Stats* proto) {
+  for (int i = 0; i < GRPC_STATS_COUNTER_COUNT; i++) {
+    Metric* m = proto->add_metrics();
+    m->set_name(grpc_stats_counter_name[i]);
+    m->set_count(core.counters[i]);
+  }
+  for (int i = 0; i < GRPC_STATS_HISTOGRAM_COUNT; i++) {
+    Metric* m = proto->add_metrics();
+    m->set_name(grpc_stats_histogram_name[i]);
+    Histogram* h = m->mutable_histogram();
+    for (int j = 0; j < grpc_stats_histo_buckets[i]; j++) {
+      Bucket* b = h->add_buckets();
+      b->set_start(grpc_stats_histo_bucket_boundaries[i][j]);
+      b->set_count(core.histograms[grpc_stats_histo_start[i] + j]);
+    }
+  }
+}
+
+void ProtoToCoreStats(const grpc::core::Stats& proto, grpc_stats_data* core) {
+  memset(core, 0, sizeof(*core));
+  for (const auto& m : proto.metrics()) {
+    switch (m.value_case()) {
+      case Metric::VALUE_NOT_SET:
+        break;
+      case Metric::kCount:
+        for (int i = 0; i < GRPC_STATS_COUNTER_COUNT; i++) {
+          if (m.name() == grpc_stats_counter_name[i]) {
+            core->counters[i] = m.count();
+            break;
+          }
+        }
+        break;
+      case Metric::kHistogram:
+        for (int i = 0; i < GRPC_STATS_HISTOGRAM_COUNT; i++) {
+          if (m.name() == grpc_stats_histogram_name[i]) {
+            const auto& h = m.histogram();
+            bool valid = true;
+            if (grpc_stats_histo_buckets[i] != h.buckets_size()) valid = false;
+            for (int j = 0; valid && j < h.buckets_size(); j++) {
+              if (grpc_stats_histo_bucket_boundaries[i][j] !=
+                  h.buckets(j).start()) {
+                valid = false;
+              }
+            }
+            if (!valid) {
+              gpr_log(GPR_ERROR,
+                      "Found histogram %s but shape is different from proto",
+                      m.name().c_str());
+            }
+            for (int j = 0; valid && j < h.buckets_size(); j++) {
+              core->histograms[grpc_stats_histo_start[i] + j] =
+                  h.buckets(j).count();
+            }
+          }
+        }
+        break;
+    }
+  }
+}
+
+}  // namespace grpc

src/cpp/util/core_stats.h

@@ -0,0 +1,35 @@
+/*
+ *
+ * Copyright 2016 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_INTERNAL_CPP_UTIL_CORE_STATS_H
+#define GRPC_INTERNAL_CPP_UTIL_CORE_STATS_H
+
+#include "src/proto/grpc/core/stats.pb.h"
+
+extern "C" {
+#include "src/core/lib/debug/stats.h"
+}
+
+namespace grpc {
+
+void CoreStatsToProto(const grpc_stats_data& core, grpc::core::Stats* proto);
+void ProtoToCoreStats(const grpc::core::Stats& proto, grpc_stats_data* core);
+
+}  // namespace grpc
+
+#endif  // GRPC_INTERNAL_CPP_UTIL_CORE_STATS_H

src/proto/grpc/core/BUILD

@@ -0,0 +1,24 @@
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])  # Apache v2
+
+load("//bazel:grpc_build_system.bzl", "grpc_proto_library", "grpc_package")
+
+grpc_package(name = "core", visibility = "public")
+
+grpc_proto_library(
+    name = "stats_proto",
+    srcs = ["stats.proto"],
+)

src/proto/grpc/core/stats.proto

@@ -0,0 +1,38 @@
+// Copyright 2017 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package grpc.core;
+
+message Bucket {
+  double start = 1;
+  uint64 count = 2;
+}
+
+message Histogram {
+  repeated Bucket buckets = 1;
+}
+
+message Metric {
+  string name = 1;
+  oneof value {
+    uint64 count = 10;
+    Histogram histogram = 11;
+  }
+}
+
+message Stats {
+  repeated Metric metrics = 1;
+}

src/proto/grpc/testing/BUILD

@@ -84,6 +84,9 @@ grpc_proto_library(
     name = "stats_proto",
     srcs = ["stats.proto"],
     has_services = False,
+    deps = [
+        "//src/proto/grpc/core:stats_proto",
+    ]
 )
 
 grpc_proto_library(

src/proto/grpc/testing/stats.proto

@@ -16,6 +16,8 @@ syntax = "proto3";
 
 package grpc.testing;
 
+import "src/proto/grpc/core/stats.proto";
+
 message ServerStats {
   // wall clock time change in seconds since last reset
   double time_elapsed = 1;
@@ -35,6 +37,9 @@ message ServerStats {
 
   // Number of polls called inside completion queue
   uint64 cq_poll_count = 6;
+
+  // Core library stats
+  grpc.core.Stats core_stats = 7;
 }
 
 // Histogram params based on grpc/support/histogram.c
@@ -72,4 +77,7 @@ message ClientStats {
 
   // Number of polls called inside completion queue
   uint64 cq_poll_count = 6;
+
+  // Core library stats
+  grpc.core.Stats core_stats = 7;
 }

test/core/debug/stats_test.cc

@@ -0,0 +1,123 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+extern "C" {
+#include "src/core/lib/debug/stats.h"
+}
+
+#include <grpc/grpc.h>
+#include <grpc/support/log.h>
+#include <gtest/gtest.h>
+
+namespace grpc {
+namespace testing {
+
+class Snapshot {
+ public:
+  Snapshot() { grpc_stats_collect(&begin_); }
+
+  grpc_stats_data delta() {
+    grpc_stats_data now;
+    grpc_stats_collect(&now);
+    grpc_stats_data delta;
+    grpc_stats_diff(&now, &begin_, &delta);
+    return delta;
+  }
+
+ private:
+  grpc_stats_data begin_;
+};
+
+TEST(StatsTest, IncCounters) {
+  for (int i = 0; i < GRPC_STATS_COUNTER_COUNT; i++) {
+    Snapshot snapshot;
+
+    grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+    GRPC_STATS_INC_COUNTER(&exec_ctx, (grpc_stats_counters)i);
+    grpc_exec_ctx_finish(&exec_ctx);
+
+    EXPECT_EQ(snapshot.delta().counters[i], 1);
+  }
+}
+
+TEST(StatsTest, IncSpecificCounter) {
+  Snapshot snapshot;
+
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  GRPC_STATS_INC_SYSCALL_POLL(&exec_ctx);
+  grpc_exec_ctx_finish(&exec_ctx);
+
+  EXPECT_EQ(snapshot.delta().counters[GRPC_STATS_COUNTER_SYSCALL_POLL], 1);
+}
+
+static int FindExpectedBucket(int i, int j) {
+  if (j < 0) {
+    return 0;
+  }
+  if (j >= grpc_stats_histo_bucket_boundaries[i][grpc_stats_histo_buckets[i]]) {
+    return grpc_stats_histo_buckets[i] - 1;
+  }
+  return std::upper_bound(grpc_stats_histo_bucket_boundaries[i],
+                          grpc_stats_histo_bucket_boundaries[i] +
+                              grpc_stats_histo_buckets[i],
+                          j) -
+         grpc_stats_histo_bucket_boundaries[i] - 1;
+}
+
+TEST(StatsTest, IncHistogram) {
+  for (int i = 0; i < GRPC_STATS_HISTOGRAM_COUNT; i++) {
+    std::vector<int> test_values;
+    for (int j = -1000;
+         j <
+         grpc_stats_histo_bucket_boundaries[i]
+                                           [grpc_stats_histo_buckets[i] - 1] +
+             1000;
+         j++) {
+      test_values.push_back(j);
+    }
+    std::random_shuffle(test_values.begin(), test_values.end());
+    if (test_values.size() > 10000) {
+      test_values.resize(10000);
+    }
+    for (auto j : test_values) {
+      Snapshot snapshot;
+
+      int expected_bucket = FindExpectedBucket(i, j);
+
+      grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+      grpc_stats_inc_histogram[i](&exec_ctx, j);
+      grpc_exec_ctx_finish(&exec_ctx);
+
+      auto delta = snapshot.delta();
+
+      EXPECT_EQ(delta.histograms[grpc_stats_histo_start[i] + expected_bucket],
+                1);
+    }
+  }
+}
+
+}  // namespace testing
+}  // namespace grpc
+
+int main(int argc, char** argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+  grpc_init();
+  int ret = RUN_ALL_TESTS();
+  grpc_shutdown();
+  return ret;
+}

test/cpp/microbenchmarks/helpers.cc

@@ -31,10 +31,17 @@ void TrackCounters::Finish(benchmark::State &state) {
 void TrackCounters::AddToLabel(std::ostream &out, benchmark::State &state) {
   grpc_stats_data stats_end;
   grpc_stats_collect(&stats_end);
+  grpc_stats_data stats;
+  grpc_stats_diff(&stats_end, &stats_begin_, &stats);
   for (int i = 0; i < GRPC_STATS_COUNTER_COUNT; i++) {
-    out << " " << grpc_stats_counter_name[i] << "/iter:"
-        << ((double)(stats_end.counters[i] - stats_begin_.counters[i]) /
-            (double)state.iterations());
+    out << " " << grpc_stats_counter_name[i]
+        << "/iter:" << ((double)stats.counters[i] / (double)state.iterations());
+  }
+  for (int i = 0; i < GRPC_STATS_HISTOGRAM_COUNT; i++) {
+    out << " " << grpc_stats_histogram_name[i] << "-median:"
+        << grpc_stats_histo_percentile(&stats, (grpc_stats_histograms)i, 50.0)
+        << " " << grpc_stats_histogram_name[i] << "-99p:"
+        << grpc_stats_histo_percentile(&stats, (grpc_stats_histograms)i, 99.0);
   }
 #ifdef GPR_LOW_LEVEL_COUNTERS
   grpc_memory_counters counters_at_end = grpc_memory_counters_snapshot();

test/cpp/qps/BUILD

@@ -46,6 +46,7 @@ grpc_cc_library(
         ":usage_timer",
         "//:grpc",
         "//:grpc++",
+        "//:grpc++_core_stats",
         "//src/proto/grpc/testing:control_proto",
         "//src/proto/grpc/testing:payloads_proto",
         "//src/proto/grpc/testing:services_proto",

test/cpp/qps/client.h

@@ -34,6 +34,7 @@
 #include "src/proto/grpc/testing/payloads.pb.h"
 #include "src/proto/grpc/testing/services.grpc.pb.h"
 
+#include "src/cpp/util/core_stats.h"
 #include "test/cpp/qps/histogram.h"
 #include "test/cpp/qps/interarrival.h"
 #include "test/cpp/qps/usage_timer.h"
@@ -172,6 +173,9 @@ class Client {
       timer_result = timer_->Mark();
     }
 
+    grpc_stats_data core_stats;
+    grpc_stats_collect(&core_stats);
+
     ClientStats stats;
     latencies.FillProto(stats.mutable_latencies());
     for (StatusHistogram::const_iterator it = statuses.begin();
@@ -184,6 +188,7 @@ class Client {
     stats.set_time_system(timer_result.system);
     stats.set_time_user(timer_result.user);
     stats.set_cq_poll_count(poll_count);
+    CoreStatsToProto(core_stats, stats.mutable_core_stats());
     return stats;
   }
 

test/cpp/qps/report.cc

@@ -26,6 +26,7 @@
 #include "test/cpp/qps/stats.h"
 
 #include <grpc++/client_context.h>
+#include "src/cpp/util/core_stats.h"
 #include "src/proto/grpc/testing/services.grpc.pb.h"
 
 namespace grpc {
@@ -85,6 +86,33 @@ void GprLogReporter::ReportQPS(const ScenarioResult& result) {
     gpr_log(GPR_INFO, "successful requests/second: %.1f",
             result.summary().successful_requests_per_second());
   }
+  for (int i = 0; i < result.client_stats_size(); i++) {
+    if (result.client_stats(i).has_core_stats()) {
+      ReportCoreStats("CLIENT", i, result.client_stats(i).core_stats());
+    }
+  }
+  for (int i = 0; i < result.server_stats_size(); i++) {
+    if (result.server_stats(i).has_core_stats()) {
+      ReportCoreStats("SERVER", i, result.server_stats(i).core_stats());
+    }
+  }
+}
+
+void GprLogReporter::ReportCoreStats(const char* name, int idx,
+                                     const grpc::core::Stats& stats) {
+  grpc_stats_data data;
+  ProtoToCoreStats(stats, &data);
+  for (int i = 0; i < GRPC_STATS_COUNTER_COUNT; i++) {
+    gpr_log(GPR_DEBUG, "%s[%d].%s = %" PRIdPTR, name, idx,
+            grpc_stats_counter_name[i], data.counters[i]);
+  }
+  for (int i = 0; i < GRPC_STATS_HISTOGRAM_COUNT; i++) {
+    gpr_log(GPR_DEBUG, "%s[%d].%s = %lf/%lf/%lf (50/95/99%%-ile)", name, idx,
+            grpc_stats_histogram_name[i],
+            grpc_stats_histo_percentile(&data, (grpc_stats_histograms)i, 50),
+            grpc_stats_histo_percentile(&data, (grpc_stats_histograms)i, 95),
+            grpc_stats_histo_percentile(&data, (grpc_stats_histograms)i, 99));
+  }
 }
 
 void GprLogReporter::ReportQPSPerCore(const ScenarioResult& result) {

test/cpp/qps/report.h

@@ -104,6 +104,9 @@ class GprLogReporter : public Reporter {
   void ReportCpuUsage(const ScenarioResult& result) override;
   void ReportPollCount(const ScenarioResult& result) override;
   void ReportQueriesPerCpuSec(const ScenarioResult& result) override;
+
+  void ReportCoreStats(const char* name, int idx,
+                       const grpc::core::Stats& stats);
 };
 
 /** Dumps the report to a JSON file. */

test/cpp/qps/server.h

@@ -26,6 +26,7 @@
 #include <grpc/support/log.h>
 #include <vector>
 
+#include "src/cpp/util/core_stats.h"
 #include "src/proto/grpc/testing/control.pb.h"
 #include "src/proto/grpc/testing/messages.pb.h"
 #include "test/core/end2end/data/ssl_test_data.h"
@@ -63,6 +64,9 @@ class Server {
       timer_result = timer_->Mark();
     }
 
+    grpc_stats_data core_stats;
+    grpc_stats_collect(&core_stats);
+
     ServerStats stats;
     stats.set_time_elapsed(timer_result.wall);
     stats.set_time_system(timer_result.system);
@@ -70,6 +74,7 @@ class Server {
     stats.set_total_cpu_time(timer_result.total_cpu_time);
     stats.set_idle_cpu_time(timer_result.idle_cpu_time);
     stats.set_cq_poll_count(poll_count);
+    CoreStatsToProto(core_stats, stats.mutable_core_stats());
     return stats;
   }
 

tools/codegen/core/gen_stats_data.py

@@ -15,21 +15,148 @@
 # limitations under the License.
 
 import collections
+import ctypes
+import math
 import sys
 import yaml
 
 with open('src/core/lib/debug/stats_data.yaml') as f:
   attrs = yaml.load(f.read())
 
-Counter = collections.namedtuple('Counter', 'name')
+types = (
+  (collections.namedtuple('Counter', 'name'), []),
+  (collections.namedtuple('Histogram', 'name max buckets'), []),
+)
 
-counters = []
+inst_map = dict((t[0].__name__, t[1]) for t in types)
+
+stats = []
 
 for attr in attrs:
-  if 'counter' in attr:
-    counters.append(Counter(name=attr['counter']))
+  found = False
+  for t, lst in types:
+    t_name = t.__name__.lower()
+    if t_name in attr:
+      name = attr[t_name]
+      del attr[t_name]
+      lst.append(t(name=name, **attr))
+      found = True
+      break
+  assert found, "Bad decl: %s" % attr
+
+def dbl2u64(d):
+  return ctypes.c_ulonglong.from_buffer(ctypes.c_double(d)).value
+
+def shift_works_until(mapped_bounds, shift_bits):
+  for i, ab in enumerate(zip(mapped_bounds, mapped_bounds[1:])):
+    a, b = ab
+    if (a >> shift_bits) == (b >> shift_bits):
+      return i
+  return len(mapped_bounds)
+
+def find_ideal_shift(mapped_bounds, max_size):
+  best = None
+  for shift_bits in reversed(range(0,64)):
+    n = shift_works_until(mapped_bounds, shift_bits)
+    if n == 0: continue
+    table_size = mapped_bounds[n-1] >> shift_bits
+    if table_size > max_size: continue
+    if table_size > 65535: continue
+    if best is None:
+      best = (shift_bits, n, table_size)
+    elif best[1] < n:
+      best = (shift_bits, n, table_size)
+  print best
+  return best
+
+def gen_map_table(mapped_bounds, shift_data):
+  tbl = []
+  cur = 0
+  print mapped_bounds
+  mapped_bounds = [x >> shift_data[0] for x in mapped_bounds]
+  print mapped_bounds
+  for i in range(0, mapped_bounds[shift_data[1]-1]):
+    while i > mapped_bounds[cur]:
+      cur += 1
+    tbl.append(cur)
+  return tbl
+
+static_tables = []
+
+def decl_static_table(values, type):
+  global static_tables
+  v = (type, values)
+  for i, vp in enumerate(static_tables):
+    if v == vp: return i
+  print "ADD TABLE: %s %r" % (type, values)
+  r = len(static_tables)
+  static_tables.append(v)
+  return r
+
+def type_for_uint_table(table):
+  mv = max(table)
+  if mv < 2**8:
+    return 'uint8_t'
+  elif mv < 2**16:
+    return 'uint16_t'
+  elif mv < 2**32:
+    return 'uint32_t'
   else:
-    print 'Error: bad attr %r' % attr
+    return 'uint64_t'
+
+def gen_bucket_code(histogram):
+  bounds = [0, 1]
+  done_trivial = False
+  done_unmapped = False
+  first_nontrivial = None
+  first_unmapped = None
+  while len(bounds) < histogram.buckets + 1:
+    if len(bounds) == histogram.buckets:
+      nextb = int(histogram.max)
+    else:
+      mul = math.pow(float(histogram.max) / bounds[-1],
+                     1.0 / (histogram.buckets + 1 - len(bounds)))
+      nextb = int(math.ceil(bounds[-1] * mul))
+    if nextb <= bounds[-1] + 1:
+      nextb = bounds[-1] + 1
+    elif not done_trivial:
+      done_trivial = True
+      first_nontrivial = len(bounds)
+    bounds.append(nextb)
+  bounds_idx = decl_static_table(bounds, 'int')
+  if done_trivial:
+    first_nontrivial_code = dbl2u64(first_nontrivial)
+    code_bounds = [dbl2u64(x) - first_nontrivial_code for x in bounds]
+    shift_data = find_ideal_shift(code_bounds[first_nontrivial:], 256 * histogram.buckets)
+  #print first_nontrivial, shift_data, bounds
+  #if shift_data is not None: print [hex(x >> shift_data[0]) for x in code_bounds[first_nontrivial:]]
+  code = 'value = GPR_CLAMP(value, 0, %d);\n' % histogram.max
+  map_table = gen_map_table(code_bounds[first_nontrivial:], shift_data)
+  if first_nontrivial is None:
+    code += ('GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_%s, value);\n'
+             % histogram.name.upper())
+  else:
+    code += 'if (value < %d) {\n' % first_nontrivial
+    code += ('GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_%s, value);\n'
+             % histogram.name.upper())
+    code += 'return;\n'
+    code += '}'
+    first_nontrivial_code = dbl2u64(first_nontrivial)
+    if shift_data is not None:
+      map_table_idx = decl_static_table(map_table, type_for_uint_table(map_table))
+      code += 'union { double dbl; uint64_t uint; } _val, _bkt;\n'
+      code += '_val.dbl = value;\n'
+      code += 'if (_val.uint < %dull) {\n' % ((map_table[-1] << shift_data[0]) + first_nontrivial_code)
+      code += 'int bucket = '
+      code += 'grpc_stats_table_%d[((_val.uint - %dull) >> %d)] + %d;\n' % (map_table_idx, first_nontrivial_code, shift_data[0], first_nontrivial)
+      code += '_bkt.dbl = grpc_stats_table_%d[bucket];\n' % bounds_idx
+      code += 'bucket -= (_val.uint < _bkt.uint);\n'
+      code += 'GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_%s, bucket);\n' % histogram.name.upper()
+      code += 'return;\n'
+      code += '}\n'
+    code += 'GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_%s, '% histogram.name.upper()
+    code += 'grpc_stats_histo_find_bucket_slow((exec_ctx), value, grpc_stats_table_%d, %d));\n' % (bounds_idx, histogram.buckets)
+  return (code, bounds_idx)
 
 # utility: print a big comment block into a set of files
 def put_banner(files, banner):
@@ -61,17 +188,50 @@ with open('src/core/lib/debug/stats_data.h', 'w') as H:
   print >>H, "#ifndef GRPC_CORE_LIB_DEBUG_STATS_DATA_H"
   print >>H, "#define GRPC_CORE_LIB_DEBUG_STATS_DATA_H"
   print >>H
+  print >>H, "#include <inttypes.h>"
+  print >>H, "#include \"src/core/lib/iomgr/exec_ctx.h\""
+  print >>H
+
+  for typename, instances in sorted(inst_map.items()):
+    print >>H, "typedef enum {"
+    for inst in instances:
+      print >>H, "  GRPC_STATS_%s_%s," % (typename.upper(), inst.name.upper())
+    print >>H, "  GRPC_STATS_%s_COUNT" % (typename.upper())
+    print >>H, "} grpc_stats_%ss;" % (typename.lower())
+    print >>H, "extern const char *grpc_stats_%s_name[GRPC_STATS_%s_COUNT];" % (
+        typename.lower(), typename.upper())
+
+  histo_start = []
+  histo_buckets = []
+  histo_bucket_boundaries = []
 
   print >>H, "typedef enum {"
-  for ctr in counters:
-    print >>H, "  GRPC_STATS_COUNTER_%s," % ctr.name.upper()
-  print >>H, "  GRPC_STATS_COUNTER_COUNT"
-  print >>H, "} grpc_stats_counters;"
+  first_slot = 0
+  for histogram in inst_map['Histogram']:
+    histo_start.append(first_slot)
+    histo_buckets.append(histogram.buckets)
+    print >>H, "  GRPC_STATS_HISTOGRAM_%s_FIRST_SLOT = %d," % (histogram.name.upper(), first_slot)
+    print >>H, "  GRPC_STATS_HISTOGRAM_%s_BUCKETS = %d," % (histogram.name.upper(), histogram.buckets)
+    first_slot += histogram.buckets
+  print >>H, "  GRPC_STATS_HISTOGRAM_BUCKETS = %d" % first_slot
+  print >>H, "} grpc_stats_histogram_constants;"
 
-  for ctr in counters:
-    print >>H, "#define GRPC_STATS_INC_%s(exec_ctx) GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_%s)" % (ctr.name.upper(), ctr.name.upper())
+  for ctr in inst_map['Counter']:
+    print >>H, ("#define GRPC_STATS_INC_%s(exec_ctx) " +
+                "GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_%s)") % (
+                ctr.name.upper(), ctr.name.upper())
+  for histogram in inst_map['Histogram']:
+    print >>H, "#define GRPC_STATS_INC_%s(exec_ctx, value) grpc_stats_inc_%s((exec_ctx), (int)(value))" % (
+        histogram.name.upper(), histogram.name.lower())
+    print >>H, "void grpc_stats_inc_%s(grpc_exec_ctx *exec_ctx, int x);" % histogram.name.lower()
 
-  print >>H, "extern const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT];"
+  for i, tbl in enumerate(static_tables):
+    print >>H, "extern const %s grpc_stats_table_%d[%d];" % (tbl[0], i, len(tbl[1]))
+
+  print >>H, "extern const int grpc_stats_histo_buckets[%d];" % len(inst_map['Histogram'])
+  print >>H, "extern const int grpc_stats_histo_start[%d];" % len(inst_map['Histogram'])
+  print >>H, "extern const int *const grpc_stats_histo_bucket_boundaries[%d];" % len(inst_map['Histogram'])
+  print >>H, "extern void (*const grpc_stats_inc_histogram[%d])(grpc_exec_ctx *exec_ctx, int x);" % len(inst_map['Histogram'])
 
   print >>H
   print >>H, "#endif /* GRPC_CORE_LIB_DEBUG_STATS_DATA_H */"
@@ -95,8 +255,36 @@ with open('src/core/lib/debug/stats_data.c', 'w') as C:
   put_banner([C], ["Automatically generated by tools/codegen/core/gen_stats_data.py"])
 
   print >>C, "#include \"src/core/lib/debug/stats_data.h\""
+  print >>C, "#include \"src/core/lib/debug/stats.h\""
+  print >>C, "#include \"src/core/lib/iomgr/exec_ctx.h\""
+  print >>C, "#include <grpc/support/useful.h>"
+
+  histo_code = []
+  for histogram in inst_map['Histogram']:
+    code, bounds_idx = gen_bucket_code(histogram)
+    histo_bucket_boundaries.append(bounds_idx)
+    histo_code.append(code)
+
+  for typename, instances in sorted(inst_map.items()):
+    print >>C, "const char *grpc_stats_%s_name[GRPC_STATS_%s_COUNT] = {" % (
+        typename.lower(), typename.upper())
+    for inst in instances:
+      print >>C, "  \"%s\"," % inst.name
+    print >>C, "};"
+  for i, tbl in enumerate(static_tables):
+    print >>C, "const %s grpc_stats_table_%d[%d] = {%s};" % (
+        tbl[0], i, len(tbl[1]), ','.join('%s' % x for x in tbl[1]))
+
+  for histogram, code in zip(inst_map['Histogram'], histo_code):
+    print >>C, ("void grpc_stats_inc_%s(grpc_exec_ctx *exec_ctx, int value) {%s}") % (
+                histogram.name.lower(),
+                code)
 
-  print >>C, "const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT] = {";
-  for ctr in counters:
-    print >>C, "  \"%s\"," % ctr.name
-  print >>C, "};"
+  print >>C, "const int grpc_stats_histo_buckets[%d] = {%s};" % (
+      len(inst_map['Histogram']), ','.join('%s' % x for x in histo_buckets))
+  print >>C, "const int grpc_stats_histo_start[%d] = {%s};" % (
+      len(inst_map['Histogram']), ','.join('%s' % x for x in histo_start))
+  print >>C, "const int *const grpc_stats_histo_bucket_boundaries[%d] = {%s};" % (
+      len(inst_map['Histogram']), ','.join('grpc_stats_table_%d' % x for x in histo_bucket_boundaries))
+  print >>C, "void (*const grpc_stats_inc_histogram[%d])(grpc_exec_ctx *exec_ctx, int x) = {%s};" % (
+      len(inst_map['Histogram']), ','.join('grpc_stats_inc_%s' % histogram.name.lower() for histogram in inst_map['Histogram']))

tools/run_tests/generated/sources_and_headers.json

@@ -3003,7 +3003,8 @@
       "gpr", 
       "grpc", 
       "grpc++", 
-      "grpc++_codegen_base"
+      "grpc++_codegen_base", 
+      "grpc++_core_stats"
     ], 
     "headers": [
       "src/proto/grpc/testing/control.grpc.pb.h", 
@@ -3036,7 +3037,8 @@
       "gpr", 
       "grpc", 
       "grpc++_codegen_base", 
-      "grpc++_codegen_base_src"
+      "grpc++_codegen_base_src", 
+      "grpc++_core_stats"
     ], 
     "headers": [
       "src/proto/grpc/testing/control.grpc.pb.h", 
@@ -3736,6 +3738,7 @@
       "gpr_test_util", 
       "grpc", 
       "grpc++", 
+      "grpc++_core_stats", 
       "grpc++_test_config", 
       "grpc++_test_util", 
       "grpc_test_util", 
@@ -3757,6 +3760,7 @@
       "gpr_test_util", 
       "grpc", 
       "grpc++", 
+      "grpc++_core_stats", 
       "grpc++_test_config", 
       "grpc++_test_util", 
       "grpc_test_util", 
@@ -3778,6 +3782,7 @@
       "gpr_test_util", 
       "grpc", 
       "grpc++", 
+      "grpc++_core_stats", 
       "grpc++_test_config", 
       "grpc++_test_util", 
       "grpc_test_util", 
@@ -3885,6 +3890,7 @@
       "gpr_test_util", 
       "grpc", 
       "grpc++", 
+      "grpc++_core_stats", 
       "grpc++_test_config", 
       "grpc++_test_util", 
       "grpc_test_util", 
@@ -4047,6 +4053,24 @@
     "third_party": false, 
     "type": "target"
   }, 
+  {
+    "deps": [
+      "gpr", 
+      "gpr_test_util", 
+      "grpc", 
+      "grpc++_test_util", 
+      "grpc_test_util"
+    ], 
+    "headers": [], 
+    "is_filegroup": false, 
+    "language": "c++", 
+    "name": "stats_test", 
+    "src": [
+      "test/core/debug/stats_test.cc"
+    ], 
+    "third_party": false, 
+    "type": "target"
+  }, 
   {
     "deps": [
       "gpr", 
@@ -6056,6 +6080,26 @@
     "third_party": false, 
     "type": "lib"
   }, 
+  {
+    "deps": [
+      "grpc++"
+    ], 
+    "headers": [
+      "src/cpp/util/core_stats.h", 
+      "src/proto/grpc/core/stats.grpc.pb.h", 
+      "src/proto/grpc/core/stats.pb.h", 
+      "src/proto/grpc/core/stats_mock.grpc.pb.h"
+    ], 
+    "is_filegroup": false, 
+    "language": "c++", 
+    "name": "grpc++_core_stats", 
+    "src": [
+      "src/cpp/util/core_stats.cc", 
+      "src/cpp/util/core_stats.h"
+    ], 
+    "third_party": false, 
+    "type": "lib"
+  }, 
   {
     "deps": [
       "census", 
@@ -6552,6 +6596,7 @@
     "deps": [
       "grpc", 
       "grpc++", 
+      "grpc++_core_stats", 
       "grpc++_test_util", 
       "grpc_test_util"
     ], 

tools/run_tests/generated/tests.json

@@ -3971,6 +3971,28 @@
       "windows"
     ]
   }, 
+  {
+    "args": [], 
+    "ci_platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ], 
+    "cpu_cost": 1.0, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "gtest": true, 
+    "language": "c++", 
+    "name": "stats_test", 
+    "platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ]
+  }, 
   {
     "args": [], 
     "ci_platforms": [