Merge pull request #13947 from kpayson64/less_epoll_ctl

Add fd cache to epollex

src/core/lib/debug/stats_data.cc

@@ -40,6 +40,8 @@ const char* grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT] = {
     "pollset_kick_wakeup_fd",
     "pollset_kick_wakeup_cv",
     "pollset_kick_own_thread",
+    "syscall_epoll_ctl",
+    "pollset_fd_cache_hits",
     "histogram_slow_lookups",
     "syscall_write",
     "syscall_read",
@@ -144,6 +146,9 @@ const char* grpc_stats_counter_doc[GRPC_STATS_COUNTER_COUNT] = {
     "polling wakeup (only valid for epoll1 right now)",
     "How many times could a polling wakeup be satisfied by keeping the waking "
     "thread awake? (only valid for epoll1 right now)",
+    "Number of epoll_ctl calls made (only valid for epollex right now)",
+    "Number of epoll_ctl calls skipped because the fd was cached as already "
+    "being added.  (only valid for epollex right now)",
     "Number of times histogram increments went through the slow (binary "
     "search) path",
     "Number of write syscalls (or equivalent - eg sendmsg) made by this "

src/core/lib/debug/stats_data.h

@@ -41,6 +41,8 @@ typedef enum {
   GRPC_STATS_COUNTER_POLLSET_KICK_WAKEUP_FD,
   GRPC_STATS_COUNTER_POLLSET_KICK_WAKEUP_CV,
   GRPC_STATS_COUNTER_POLLSET_KICK_OWN_THREAD,
+  GRPC_STATS_COUNTER_SYSCALL_EPOLL_CTL,
+  GRPC_STATS_COUNTER_POLLSET_FD_CACHE_HITS,
   GRPC_STATS_COUNTER_HISTOGRAM_SLOW_LOOKUPS,
   GRPC_STATS_COUNTER_SYSCALL_WRITE,
   GRPC_STATS_COUNTER_SYSCALL_READ,
@@ -203,6 +205,10 @@ typedef enum {
   GRPC_STATS_INC_COUNTER(GRPC_STATS_COUNTER_POLLSET_KICK_WAKEUP_CV)
 #define GRPC_STATS_INC_POLLSET_KICK_OWN_THREAD() \
   GRPC_STATS_INC_COUNTER(GRPC_STATS_COUNTER_POLLSET_KICK_OWN_THREAD)
+#define GRPC_STATS_INC_SYSCALL_EPOLL_CTL() \
+  GRPC_STATS_INC_COUNTER(GRPC_STATS_COUNTER_SYSCALL_EPOLL_CTL)
+#define GRPC_STATS_INC_POLLSET_FD_CACHE_HITS() \
+  GRPC_STATS_INC_COUNTER(GRPC_STATS_COUNTER_POLLSET_FD_CACHE_HITS)
 #define GRPC_STATS_INC_HISTOGRAM_SLOW_LOOKUPS() \
   GRPC_STATS_INC_COUNTER(GRPC_STATS_COUNTER_HISTOGRAM_SLOW_LOOKUPS)
 #define GRPC_STATS_INC_SYSCALL_WRITE() \
@@ -443,6 +449,8 @@ void grpc_stats_inc_server_cqs_checked(int x);
 #define GRPC_STATS_INC_POLLSET_KICK_WAKEUP_FD()
 #define GRPC_STATS_INC_POLLSET_KICK_WAKEUP_CV()
 #define GRPC_STATS_INC_POLLSET_KICK_OWN_THREAD()
+#define GRPC_STATS_INC_SYSCALL_EPOLL_CTL()
+#define GRPC_STATS_INC_POLLSET_FD_CACHE_HITS()
 #define GRPC_STATS_INC_HISTOGRAM_SLOW_LOOKUPS()
 #define GRPC_STATS_INC_SYSCALL_WRITE()
 #define GRPC_STATS_INC_SYSCALL_READ()

src/core/lib/debug/stats_data.yaml

@@ -63,6 +63,12 @@
   doc: How many times could a polling wakeup be satisfied by keeping the waking
        thread awake?
        (only valid for epoll1 right now)
+# polling
+- counter: syscall_epoll_ctl
+  doc: Number of epoll_ctl calls made (only valid for epollex right now)
+- counter: pollset_fd_cache_hits
+  doc: Number of epoll_ctl calls skipped because the fd was cached as
+       already being added.  (only valid for epollex right now)
 # stats system
 - counter: histogram_slow_lookups
   doc: Number of times histogram increments went through the slow

src/core/lib/debug/stats_data_bq_schema.sql

@@ -12,6 +12,8 @@ pollset_kicked_again_per_iteration:FLOAT,
 pollset_kick_wakeup_fd_per_iteration:FLOAT,
 pollset_kick_wakeup_cv_per_iteration:FLOAT,
 pollset_kick_own_thread_per_iteration:FLOAT,
+syscall_epoll_ctl_per_iteration:FLOAT,
+pollset_fd_cache_hits_per_iteration:FLOAT,
 histogram_slow_lookups_per_iteration:FLOAT,
 syscall_write_per_iteration:FLOAT,
 syscall_read_per_iteration:FLOAT,

src/core/lib/iomgr/ev_epollex_linux.cc

@@ -63,6 +63,7 @@
 // a keepalive ping timeout issue. We may want to revert https://github
 // .com/grpc/grpc/pull/14943 once we figure out the root cause.
 #define MAX_EPOLL_EVENTS_HANDLED_EACH_POLL_CALL 16
+#define MAX_PROBE_EPOLL_FDS 32
 
 grpc_core::DebugOnlyTraceFlag grpc_trace_pollable_refcount(false,
                                                            "pollable_refcount");
@@ -75,6 +76,12 @@ typedef enum { PO_MULTI, PO_FD, PO_EMPTY } pollable_type;
 
 typedef struct pollable pollable;
 
+typedef struct cached_fd {
+  intptr_t salt;
+  int fd;
+  uint64_t last_used;
+} cached_fd;
+
 /// A pollable is something that can be polled: it has an epoll set to poll on,
 /// and a wakeup fd for kicks
 /// There are three broad types:
@@ -103,6 +110,11 @@ struct pollable {
   int event_cursor;
   int event_count;
   struct epoll_event events[MAX_EPOLL_EVENTS];
+
+  // Maintain a LRU-eviction cache of fds in this pollable
+  cached_fd fd_cache[MAX_PROBE_EPOLL_FDS];
+  int fd_cache_size;
+  uint64_t fd_cache_counter;
 };
 
 static const char* pollable_type_string(pollable_type t) {
@@ -145,8 +157,11 @@ static void pollable_unref(pollable* p, int line, const char* reason);
  * Fd Declarations
  */
 
+static gpr_atm g_fd_salt;
+
 struct grpc_fd {
   int fd;
+  intptr_t salt;
   /* refst format:
        bit 0    : 1=Active / 0=Orphaned
        bits 1-n : refcount
@@ -354,6 +369,7 @@ static grpc_fd* fd_create(int fd, const char* name) {
   new_fd->pollable_obj = nullptr;
   gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1);
   new_fd->fd = fd;
+  new_fd->salt = gpr_atm_no_barrier_fetch_add(&g_fd_salt, 1);
   new_fd->read_closure->InitEvent();
   new_fd->write_closure->InitEvent();
   gpr_atm_no_barrier_store(&new_fd->read_notifier_pollset, (gpr_atm)NULL);
@@ -484,6 +500,8 @@ static grpc_error* pollable_create(pollable_type type, pollable** p) {
   (*p)->root_worker = nullptr;
   (*p)->event_cursor = 0;
   (*p)->event_count = 0;
+  (*p)->fd_cache_size = 0;
+  (*p)->fd_cache_counter = 0;
   return GRPC_ERROR_NONE;
 }
 
@@ -524,7 +542,36 @@ static grpc_error* pollable_add_fd(pollable* p, grpc_fd* fd) {
   grpc_error* error = GRPC_ERROR_NONE;
   static const char* err_desc = "pollable_add_fd";
   const int epfd = p->epfd;
+  gpr_mu_lock(&p->mu);
+  p->fd_cache_counter++;
+  // Handle the case of overflow for our cache counter by
+  // reseting the recency-counter on all cache objects
+  if (p->fd_cache_counter == 0) {
+    for (int i = 0; i < p->fd_cache_size; i++) {
+      p->fd_cache[i].last_used = 0;
+    }
+  }
 
+  int lru_idx = 0;
+  for (int i = 0; i < p->fd_cache_size; i++) {
+    if (p->fd_cache[i].fd == fd->fd && p->fd_cache[i].salt == fd->salt) {
+      GRPC_STATS_INC_POLLSET_FD_CACHE_HITS();
+      p->fd_cache[i].last_used = p->fd_cache_counter;
+      gpr_mu_unlock(&p->mu);
+      return GRPC_ERROR_NONE;
+    } else if (p->fd_cache[i].last_used < p->fd_cache[lru_idx].last_used) {
+      lru_idx = i;
+    }
+  }
+  // Add to cache
+  if (p->fd_cache_size < MAX_PROBE_EPOLL_FDS) {
+    lru_idx = p->fd_cache_size;
+    p->fd_cache_size++;
+  }
+  p->fd_cache[lru_idx].fd = fd->fd;
+  p->fd_cache[lru_idx].salt = fd->salt;
+  p->fd_cache[lru_idx].last_used = p->fd_cache_counter;
+  gpr_mu_unlock(&p->mu);
   if (grpc_polling_trace.enabled()) {
     gpr_log(GPR_INFO, "add fd %p (%d) to pollable %p", fd, fd->fd, p);
   }
@@ -533,6 +580,7 @@ static grpc_error* pollable_add_fd(pollable* p, grpc_fd* fd) {
   ev_fd.events =
       static_cast<uint32_t>(EPOLLET | EPOLLIN | EPOLLOUT | EPOLLEXCLUSIVE);
   ev_fd.data.ptr = fd;
+  GRPC_STATS_INC_SYSCALL_EPOLL_CTL();
   if (epoll_ctl(epfd, EPOLL_CTL_ADD, fd->fd, &ev_fd) != 0) {
     switch (errno) {
       case EEXIST:

tools/run_tests/performance/massage_qps_stats.py

@@ -19,488 +19,455 @@ import massage_qps_stats_helpers
 
 def massage_qps_stats(scenario_result):
     for stats in scenario_result["serverStats"] + scenario_result["clientStats"]:
-        if "coreStats" in stats:
-            # Get rid of the "coreStats" element and replace it by statistics
-            # that correspond to columns in the bigquery schema.
-            core_stats = stats["coreStats"]
-            del stats["coreStats"]
-            stats[
-                "core_client_calls_created"] = massage_qps_stats_helpers.counter(
-                    core_stats, "client_calls_created")
-            stats[
-                "core_server_calls_created"] = massage_qps_stats_helpers.counter(
-                    core_stats, "server_calls_created")
-            stats["core_cqs_created"] = massage_qps_stats_helpers.counter(
-                core_stats, "cqs_created")
-            stats[
-                "core_client_channels_created"] = massage_qps_stats_helpers.counter(
-                    core_stats, "client_channels_created")
-            stats[
-                "core_client_subchannels_created"] = massage_qps_stats_helpers.counter(
-                    core_stats, "client_subchannels_created")
-            stats[
-                "core_server_channels_created"] = massage_qps_stats_helpers.counter(
-                    core_stats, "server_channels_created")
-            stats["core_syscall_poll"] = massage_qps_stats_helpers.counter(
-                core_stats, "syscall_poll")
-            stats["core_syscall_wait"] = massage_qps_stats_helpers.counter(
-                core_stats, "syscall_wait")
-            stats["core_pollset_kick"] = massage_qps_stats_helpers.counter(
-                core_stats, "pollset_kick")
-            stats[
-                "core_pollset_kicked_without_poller"] = massage_qps_stats_helpers.counter(
-                    core_stats, "pollset_kicked_without_poller")
-            stats[
-                "core_pollset_kicked_again"] = massage_qps_stats_helpers.counter(
-                    core_stats, "pollset_kicked_again")
-            stats[
-                "core_pollset_kick_wakeup_fd"] = massage_qps_stats_helpers.counter(
-                    core_stats, "pollset_kick_wakeup_fd")
-            stats[
-                "core_pollset_kick_wakeup_cv"] = massage_qps_stats_helpers.counter(
-                    core_stats, "pollset_kick_wakeup_cv")
-            stats[
-                "core_pollset_kick_own_thread"] = massage_qps_stats_helpers.counter(
-                    core_stats, "pollset_kick_own_thread")
-            stats[
-                "core_histogram_slow_lookups"] = massage_qps_stats_helpers.counter(
-                    core_stats, "histogram_slow_lookups")
-            stats["core_syscall_write"] = massage_qps_stats_helpers.counter(
-                core_stats, "syscall_write")
-            stats["core_syscall_read"] = massage_qps_stats_helpers.counter(
-                core_stats, "syscall_read")
-            stats[
-                "core_tcp_backup_pollers_created"] = massage_qps_stats_helpers.counter(
-                    core_stats, "tcp_backup_pollers_created")
-            stats[
-                "core_tcp_backup_poller_polls"] = massage_qps_stats_helpers.counter(
-                    core_stats, "tcp_backup_poller_polls")
-            stats["core_http2_op_batches"] = massage_qps_stats_helpers.counter(
-                core_stats, "http2_op_batches")
-            stats["core_http2_op_cancel"] = massage_qps_stats_helpers.counter(
-                core_stats, "http2_op_cancel")
-            stats[
-                "core_http2_op_send_initial_metadata"] = massage_qps_stats_helpers.counter(
-                    core_stats, "http2_op_send_initial_metadata")
-            stats[
-                "core_http2_op_send_message"] = massage_qps_stats_helpers.counter(
-                    core_stats, "http2_op_send_message")
-            stats[
-                "core_http2_op_send_trailing_metadata"] = massage_qps_stats_helpers.counter(
-                    core_stats, "http2_op_send_trailing_metadata")
-            stats[
-                "core_http2_op_recv_initial_metadata"] = massage_qps_stats_helpers.counter(
-                    core_stats, "http2_op_recv_initial_metadata")
-            stats[
-                "core_http2_op_recv_message"] = massage_qps_stats_helpers.counter(
-                    core_stats, "http2_op_recv_message")
-            stats[
-                "core_http2_op_recv_trailing_metadata"] = massage_qps_stats_helpers.counter(
-                    core_stats, "http2_op_recv_trailing_metadata")
-            stats[
-                "core_http2_settings_writes"] = massage_qps_stats_helpers.counter(
-                    core_stats, "http2_settings_writes")
-            stats["core_http2_pings_sent"] = massage_qps_stats_helpers.counter(
-                core_stats, "http2_pings_sent")
-            stats[
-                "core_http2_writes_begun"] = massage_qps_stats_helpers.counter(
-                    core_stats, "http2_writes_begun")
-            stats[
-                "core_http2_writes_offloaded"] = massage_qps_stats_helpers.counter(
-                    core_stats, "http2_writes_offloaded")
-            stats[
-                "core_http2_writes_continued"] = massage_qps_stats_helpers.counter(
-                    core_stats, "http2_writes_continued")
-            stats[
-                "core_http2_partial_writes"] = massage_qps_stats_helpers.counter(
-                    core_stats, "http2_partial_writes")
-            stats[
-                "core_http2_initiate_write_due_to_initial_write"] = massage_qps_stats_helpers.counter(
-                    core_stats, "http2_initiate_write_due_to_initial_write")
-            stats[
-                "core_http2_initiate_write_due_to_start_new_stream"] = massage_qps_stats_helpers.counter(
-                    core_stats, "http2_initiate_write_due_to_start_new_stream")
-            stats[
-                "core_http2_initiate_write_due_to_send_message"] = massage_qps_stats_helpers.counter(
-                    core_stats, "http2_initiate_write_due_to_send_message")
-            stats[
-                "core_http2_initiate_write_due_to_send_initial_metadata"] = massage_qps_stats_helpers.counter(
-                    core_stats,
-                    "http2_initiate_write_due_to_send_initial_metadata")
-            stats[
-                "core_http2_initiate_write_due_to_send_trailing_metadata"] = massage_qps_stats_helpers.counter(
-                    core_stats,
-                    "http2_initiate_write_due_to_send_trailing_metadata")
-            stats[
-                "core_http2_initiate_write_due_to_retry_send_ping"] = massage_qps_stats_helpers.counter(
-                    core_stats, "http2_initiate_write_due_to_retry_send_ping")
-            stats[
-                "core_http2_initiate_write_due_to_continue_pings"] = massage_qps_stats_helpers.counter(
-                    core_stats, "http2_initiate_write_due_to_continue_pings")
-            stats[
-                "core_http2_initiate_write_due_to_goaway_sent"] = massage_qps_stats_helpers.counter(
-                    core_stats, "http2_initiate_write_due_to_goaway_sent")
-            stats[
-                "core_http2_initiate_write_due_to_rst_stream"] = massage_qps_stats_helpers.counter(
-                    core_stats, "http2_initiate_write_due_to_rst_stream")
-            stats[
-                "core_http2_initiate_write_due_to_close_from_api"] = massage_qps_stats_helpers.counter(
-                    core_stats, "http2_initiate_write_due_to_close_from_api")
-            stats[
-                "core_http2_initiate_write_due_to_stream_flow_control"] = massage_qps_stats_helpers.counter(
-                    core_stats,
-                    "http2_initiate_write_due_to_stream_flow_control")
-            stats[
-                "core_http2_initiate_write_due_to_transport_flow_control"] = massage_qps_stats_helpers.counter(
-                    core_stats,
-                    "http2_initiate_write_due_to_transport_flow_control")
-            stats[
-                "core_http2_initiate_write_due_to_send_settings"] = massage_qps_stats_helpers.counter(
-                    core_stats, "http2_initiate_write_due_to_send_settings")
-            stats[
-                "core_http2_initiate_write_due_to_bdp_estimator_ping"] = massage_qps_stats_helpers.counter(
-                    core_stats,
-                    "http2_initiate_write_due_to_bdp_estimator_ping")
-            stats[
-                "core_http2_initiate_write_due_to_flow_control_unstalled_by_setting"] = massage_qps_stats_helpers.counter(
-                    core_stats,
-                    "http2_initiate_write_due_to_flow_control_unstalled_by_setting"
-                )
-            stats[
-                "core_http2_initiate_write_due_to_flow_control_unstalled_by_update"] = massage_qps_stats_helpers.counter(
-                    core_stats,
-                    "http2_initiate_write_due_to_flow_control_unstalled_by_update"
-                )
-            stats[
-                "core_http2_initiate_write_due_to_application_ping"] = massage_qps_stats_helpers.counter(
-                    core_stats, "http2_initiate_write_due_to_application_ping")
-            stats[
-                "core_http2_initiate_write_due_to_keepalive_ping"] = massage_qps_stats_helpers.counter(
-                    core_stats, "http2_initiate_write_due_to_keepalive_ping")
-            stats[
-                "core_http2_initiate_write_due_to_transport_flow_control_unstalled"] = massage_qps_stats_helpers.counter(
-                    core_stats,
-                    "http2_initiate_write_due_to_transport_flow_control_unstalled"
-                )
-            stats[
-                "core_http2_initiate_write_due_to_ping_response"] = massage_qps_stats_helpers.counter(
-                    core_stats, "http2_initiate_write_due_to_ping_response")
-            stats[
-                "core_http2_initiate_write_due_to_force_rst_stream"] = massage_qps_stats_helpers.counter(
-                    core_stats, "http2_initiate_write_due_to_force_rst_stream")
-            stats[
-                "core_http2_spurious_writes_begun"] = massage_qps_stats_helpers.counter(
-                    core_stats, "http2_spurious_writes_begun")
-            stats[
-                "core_hpack_recv_indexed"] = massage_qps_stats_helpers.counter(
-                    core_stats, "hpack_recv_indexed")
-            stats[
-                "core_hpack_recv_lithdr_incidx"] = massage_qps_stats_helpers.counter(
-                    core_stats, "hpack_recv_lithdr_incidx")
-            stats[
-                "core_hpack_recv_lithdr_incidx_v"] = massage_qps_stats_helpers.counter(
-                    core_stats, "hpack_recv_lithdr_incidx_v")
-            stats[
-                "core_hpack_recv_lithdr_notidx"] = massage_qps_stats_helpers.counter(
-                    core_stats, "hpack_recv_lithdr_notidx")
-            stats[
-                "core_hpack_recv_lithdr_notidx_v"] = massage_qps_stats_helpers.counter(
-                    core_stats, "hpack_recv_lithdr_notidx_v")
-            stats[
-                "core_hpack_recv_lithdr_nvridx"] = massage_qps_stats_helpers.counter(
-                    core_stats, "hpack_recv_lithdr_nvridx")
-            stats[
-                "core_hpack_recv_lithdr_nvridx_v"] = massage_qps_stats_helpers.counter(
-                    core_stats, "hpack_recv_lithdr_nvridx_v")
-            stats[
-                "core_hpack_recv_uncompressed"] = massage_qps_stats_helpers.counter(
-                    core_stats, "hpack_recv_uncompressed")
-            stats[
-                "core_hpack_recv_huffman"] = massage_qps_stats_helpers.counter(
-                    core_stats, "hpack_recv_huffman")
-            stats["core_hpack_recv_binary"] = massage_qps_stats_helpers.counter(
-                core_stats, "hpack_recv_binary")
-            stats[
-                "core_hpack_recv_binary_base64"] = massage_qps_stats_helpers.counter(
-                    core_stats, "hpack_recv_binary_base64")
-            stats[
-                "core_hpack_send_indexed"] = massage_qps_stats_helpers.counter(
-                    core_stats, "hpack_send_indexed")
-            stats[
-                "core_hpack_send_lithdr_incidx"] = massage_qps_stats_helpers.counter(
-                    core_stats, "hpack_send_lithdr_incidx")
-            stats[
-                "core_hpack_send_lithdr_incidx_v"] = massage_qps_stats_helpers.counter(
-                    core_stats, "hpack_send_lithdr_incidx_v")
-            stats[
-                "core_hpack_send_lithdr_notidx"] = massage_qps_stats_helpers.counter(
-                    core_stats, "hpack_send_lithdr_notidx")
-            stats[
-                "core_hpack_send_lithdr_notidx_v"] = massage_qps_stats_helpers.counter(
-                    core_stats, "hpack_send_lithdr_notidx_v")
-            stats[
-                "core_hpack_send_lithdr_nvridx"] = massage_qps_stats_helpers.counter(
-                    core_stats, "hpack_send_lithdr_nvridx")
-            stats[
-                "core_hpack_send_lithdr_nvridx_v"] = massage_qps_stats_helpers.counter(
-                    core_stats, "hpack_send_lithdr_nvridx_v")
-            stats[
-                "core_hpack_send_uncompressed"] = massage_qps_stats_helpers.counter(
-                    core_stats, "hpack_send_uncompressed")
-            stats[
-                "core_hpack_send_huffman"] = massage_qps_stats_helpers.counter(
-                    core_stats, "hpack_send_huffman")
-            stats["core_hpack_send_binary"] = massage_qps_stats_helpers.counter(
-                core_stats, "hpack_send_binary")
-            stats[
-                "core_hpack_send_binary_base64"] = massage_qps_stats_helpers.counter(
-                    core_stats, "hpack_send_binary_base64")
-            stats[
-                "core_combiner_locks_initiated"] = massage_qps_stats_helpers.counter(
-                    core_stats, "combiner_locks_initiated")
-            stats[
-                "core_combiner_locks_scheduled_items"] = massage_qps_stats_helpers.counter(
-                    core_stats, "combiner_locks_scheduled_items")
-            stats[
-                "core_combiner_locks_scheduled_final_items"] = massage_qps_stats_helpers.counter(
-                    core_stats, "combiner_locks_scheduled_final_items")
-            stats[
-                "core_combiner_locks_offloaded"] = massage_qps_stats_helpers.counter(
-                    core_stats, "combiner_locks_offloaded")
-            stats[
-                "core_call_combiner_locks_initiated"] = massage_qps_stats_helpers.counter(
-                    core_stats, "call_combiner_locks_initiated")
-            stats[
-                "core_call_combiner_locks_scheduled_items"] = massage_qps_stats_helpers.counter(
-                    core_stats, "call_combiner_locks_scheduled_items")
-            stats[
-                "core_call_combiner_set_notify_on_cancel"] = massage_qps_stats_helpers.counter(
-                    core_stats, "call_combiner_set_notify_on_cancel")
-            stats[
-                "core_call_combiner_cancelled"] = massage_qps_stats_helpers.counter(
-                    core_stats, "call_combiner_cancelled")
-            stats[
-                "core_executor_scheduled_short_items"] = massage_qps_stats_helpers.counter(
-                    core_stats, "executor_scheduled_short_items")
-            stats[
-                "core_executor_scheduled_long_items"] = massage_qps_stats_helpers.counter(
-                    core_stats, "executor_scheduled_long_items")
-            stats[
-                "core_executor_scheduled_to_self"] = massage_qps_stats_helpers.counter(
-                    core_stats, "executor_scheduled_to_self")
-            stats[
-                "core_executor_wakeup_initiated"] = massage_qps_stats_helpers.counter(
-                    core_stats, "executor_wakeup_initiated")
-            stats[
-                "core_executor_queue_drained"] = massage_qps_stats_helpers.counter(
-                    core_stats, "executor_queue_drained")
-            stats[
-                "core_executor_push_retries"] = massage_qps_stats_helpers.counter(
-                    core_stats, "executor_push_retries")
-            stats[
-                "core_server_requested_calls"] = massage_qps_stats_helpers.counter(
-                    core_stats, "server_requested_calls")
-            stats[
-                "core_server_slowpath_requests_queued"] = massage_qps_stats_helpers.counter(
-                    core_stats, "server_slowpath_requests_queued")
-            stats[
-                "core_cq_ev_queue_trylock_failures"] = massage_qps_stats_helpers.counter(
-                    core_stats, "cq_ev_queue_trylock_failures")
-            stats[
-                "core_cq_ev_queue_trylock_successes"] = massage_qps_stats_helpers.counter(
-                    core_stats, "cq_ev_queue_trylock_successes")
-            stats[
-                "core_cq_ev_queue_transient_pop_failures"] = massage_qps_stats_helpers.counter(
-                    core_stats, "cq_ev_queue_transient_pop_failures")
-            h = massage_qps_stats_helpers.histogram(core_stats,
-                                                    "call_initial_size")
-            stats["core_call_initial_size"] = ",".join(
-                "%f" % x for x in h.buckets)
-            stats["core_call_initial_size_bkts"] = ",".join(
-                "%f" % x for x in h.boundaries)
-            stats[
-                "core_call_initial_size_50p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 50, h.boundaries)
-            stats[
-                "core_call_initial_size_95p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 95, h.boundaries)
-            stats[
-                "core_call_initial_size_99p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 99, h.boundaries)
-            h = massage_qps_stats_helpers.histogram(core_stats,
-                                                    "poll_events_returned")
-            stats["core_poll_events_returned"] = ",".join(
-                "%f" % x for x in h.buckets)
-            stats["core_poll_events_returned_bkts"] = ",".join(
-                "%f" % x for x in h.boundaries)
-            stats[
-                "core_poll_events_returned_50p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 50, h.boundaries)
-            stats[
-                "core_poll_events_returned_95p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 95, h.boundaries)
-            stats[
-                "core_poll_events_returned_99p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 99, h.boundaries)
-            h = massage_qps_stats_helpers.histogram(core_stats,
-                                                    "tcp_write_size")
-            stats["core_tcp_write_size"] = ",".join("%f" % x for x in h.buckets)
-            stats["core_tcp_write_size_bkts"] = ",".join(
-                "%f" % x for x in h.boundaries)
-            stats[
-                "core_tcp_write_size_50p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 50, h.boundaries)
-            stats[
-                "core_tcp_write_size_95p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 95, h.boundaries)
-            stats[
-                "core_tcp_write_size_99p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 99, h.boundaries)
-            h = massage_qps_stats_helpers.histogram(core_stats,
-                                                    "tcp_write_iov_size")
-            stats["core_tcp_write_iov_size"] = ",".join(
-                "%f" % x for x in h.buckets)
-            stats["core_tcp_write_iov_size_bkts"] = ",".join(
-                "%f" % x for x in h.boundaries)
-            stats[
-                "core_tcp_write_iov_size_50p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 50, h.boundaries)
-            stats[
-                "core_tcp_write_iov_size_95p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 95, h.boundaries)
-            stats[
-                "core_tcp_write_iov_size_99p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 99, h.boundaries)
-            h = massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size")
-            stats["core_tcp_read_size"] = ",".join("%f" % x for x in h.buckets)
-            stats["core_tcp_read_size_bkts"] = ",".join(
-                "%f" % x for x in h.boundaries)
-            stats[
-                "core_tcp_read_size_50p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 50, h.boundaries)
-            stats[
-                "core_tcp_read_size_95p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 95, h.boundaries)
-            stats[
-                "core_tcp_read_size_99p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 99, h.boundaries)
-            h = massage_qps_stats_helpers.histogram(core_stats,
-                                                    "tcp_read_offer")
-            stats["core_tcp_read_offer"] = ",".join("%f" % x for x in h.buckets)
-            stats["core_tcp_read_offer_bkts"] = ",".join(
-                "%f" % x for x in h.boundaries)
-            stats[
-                "core_tcp_read_offer_50p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 50, h.boundaries)
-            stats[
-                "core_tcp_read_offer_95p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 95, h.boundaries)
-            stats[
-                "core_tcp_read_offer_99p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 99, h.boundaries)
-            h = massage_qps_stats_helpers.histogram(core_stats,
-                                                    "tcp_read_offer_iov_size")
-            stats["core_tcp_read_offer_iov_size"] = ",".join(
-                "%f" % x for x in h.buckets)
-            stats["core_tcp_read_offer_iov_size_bkts"] = ",".join(
-                "%f" % x for x in h.boundaries)
-            stats[
-                "core_tcp_read_offer_iov_size_50p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 50, h.boundaries)
-            stats[
-                "core_tcp_read_offer_iov_size_95p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 95, h.boundaries)
-            stats[
-                "core_tcp_read_offer_iov_size_99p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 99, h.boundaries)
-            h = massage_qps_stats_helpers.histogram(core_stats,
-                                                    "http2_send_message_size")
-            stats["core_http2_send_message_size"] = ",".join(
-                "%f" % x for x in h.buckets)
-            stats["core_http2_send_message_size_bkts"] = ",".join(
-                "%f" % x for x in h.boundaries)
-            stats[
-                "core_http2_send_message_size_50p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 50, h.boundaries)
-            stats[
-                "core_http2_send_message_size_95p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 95, h.boundaries)
-            stats[
-                "core_http2_send_message_size_99p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 99, h.boundaries)
-            h = massage_qps_stats_helpers.histogram(
-                core_stats, "http2_send_initial_metadata_per_write")
-            stats["core_http2_send_initial_metadata_per_write"] = ",".join(
-                "%f" % x for x in h.buckets)
-            stats["core_http2_send_initial_metadata_per_write_bkts"] = ",".join(
-                "%f" % x for x in h.boundaries)
-            stats[
-                "core_http2_send_initial_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 50, h.boundaries)
-            stats[
-                "core_http2_send_initial_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 95, h.boundaries)
-            stats[
-                "core_http2_send_initial_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 99, h.boundaries)
-            h = massage_qps_stats_helpers.histogram(
-                core_stats, "http2_send_message_per_write")
-            stats["core_http2_send_message_per_write"] = ",".join(
-                "%f" % x for x in h.buckets)
-            stats["core_http2_send_message_per_write_bkts"] = ",".join(
-                "%f" % x for x in h.boundaries)
-            stats[
-                "core_http2_send_message_per_write_50p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 50, h.boundaries)
-            stats[
-                "core_http2_send_message_per_write_95p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 95, h.boundaries)
-            stats[
-                "core_http2_send_message_per_write_99p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 99, h.boundaries)
-            h = massage_qps_stats_helpers.histogram(
-                core_stats, "http2_send_trailing_metadata_per_write")
-            stats["core_http2_send_trailing_metadata_per_write"] = ",".join(
-                "%f" % x for x in h.buckets)
-            stats[
-                "core_http2_send_trailing_metadata_per_write_bkts"] = ",".join(
-                    "%f" % x for x in h.boundaries)
-            stats[
-                "core_http2_send_trailing_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 50, h.boundaries)
-            stats[
-                "core_http2_send_trailing_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 95, h.boundaries)
-            stats[
-                "core_http2_send_trailing_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 99, h.boundaries)
-            h = massage_qps_stats_helpers.histogram(
-                core_stats, "http2_send_flowctl_per_write")
-            stats["core_http2_send_flowctl_per_write"] = ",".join(
-                "%f" % x for x in h.buckets)
-            stats["core_http2_send_flowctl_per_write_bkts"] = ",".join(
-                "%f" % x for x in h.boundaries)
-            stats[
-                "core_http2_send_flowctl_per_write_50p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 50, h.boundaries)
-            stats[
-                "core_http2_send_flowctl_per_write_95p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 95, h.boundaries)
-            stats[
-                "core_http2_send_flowctl_per_write_99p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 99, h.boundaries)
-            h = massage_qps_stats_helpers.histogram(core_stats,
-                                                    "server_cqs_checked")
-            stats["core_server_cqs_checked"] = ",".join(
-                "%f" % x for x in h.buckets)
-            stats["core_server_cqs_checked_bkts"] = ",".join(
-                "%f" % x for x in h.boundaries)
-            stats[
-                "core_server_cqs_checked_50p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 50, h.boundaries)
-            stats[
-                "core_server_cqs_checked_95p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 95, h.boundaries)
-            stats[
-                "core_server_cqs_checked_99p"] = massage_qps_stats_helpers.percentile(
-                    h.buckets, 99, h.boundaries)
+        if "coreStats" not in stats: return
+        core_stats = stats["coreStats"]
+        del stats["coreStats"]
+        stats["core_client_calls_created"] = massage_qps_stats_helpers.counter(
+            core_stats, "client_calls_created")
+        stats["core_server_calls_created"] = massage_qps_stats_helpers.counter(
+            core_stats, "server_calls_created")
+        stats["core_cqs_created"] = massage_qps_stats_helpers.counter(
+            core_stats, "cqs_created")
+        stats[
+            "core_client_channels_created"] = massage_qps_stats_helpers.counter(
+                core_stats, "client_channels_created")
+        stats[
+            "core_client_subchannels_created"] = massage_qps_stats_helpers.counter(
+                core_stats, "client_subchannels_created")
+        stats[
+            "core_server_channels_created"] = massage_qps_stats_helpers.counter(
+                core_stats, "server_channels_created")
+        stats["core_syscall_poll"] = massage_qps_stats_helpers.counter(
+            core_stats, "syscall_poll")
+        stats["core_syscall_wait"] = massage_qps_stats_helpers.counter(
+            core_stats, "syscall_wait")
+        stats["core_pollset_kick"] = massage_qps_stats_helpers.counter(
+            core_stats, "pollset_kick")
+        stats[
+            "core_pollset_kicked_without_poller"] = massage_qps_stats_helpers.counter(
+                core_stats, "pollset_kicked_without_poller")
+        stats["core_pollset_kicked_again"] = massage_qps_stats_helpers.counter(
+            core_stats, "pollset_kicked_again")
+        stats[
+            "core_pollset_kick_wakeup_fd"] = massage_qps_stats_helpers.counter(
+                core_stats, "pollset_kick_wakeup_fd")
+        stats[
+            "core_pollset_kick_wakeup_cv"] = massage_qps_stats_helpers.counter(
+                core_stats, "pollset_kick_wakeup_cv")
+        stats[
+            "core_pollset_kick_own_thread"] = massage_qps_stats_helpers.counter(
+                core_stats, "pollset_kick_own_thread")
+        stats["core_syscall_epoll_ctl"] = massage_qps_stats_helpers.counter(
+            core_stats, "syscall_epoll_ctl")
+        stats["core_pollset_fd_cache_hits"] = massage_qps_stats_helpers.counter(
+            core_stats, "pollset_fd_cache_hits")
+        stats[
+            "core_histogram_slow_lookups"] = massage_qps_stats_helpers.counter(
+                core_stats, "histogram_slow_lookups")
+        stats["core_syscall_write"] = massage_qps_stats_helpers.counter(
+            core_stats, "syscall_write")
+        stats["core_syscall_read"] = massage_qps_stats_helpers.counter(
+            core_stats, "syscall_read")
+        stats[
+            "core_tcp_backup_pollers_created"] = massage_qps_stats_helpers.counter(
+                core_stats, "tcp_backup_pollers_created")
+        stats[
+            "core_tcp_backup_poller_polls"] = massage_qps_stats_helpers.counter(
+                core_stats, "tcp_backup_poller_polls")
+        stats["core_http2_op_batches"] = massage_qps_stats_helpers.counter(
+            core_stats, "http2_op_batches")
+        stats["core_http2_op_cancel"] = massage_qps_stats_helpers.counter(
+            core_stats, "http2_op_cancel")
+        stats[
+            "core_http2_op_send_initial_metadata"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_op_send_initial_metadata")
+        stats["core_http2_op_send_message"] = massage_qps_stats_helpers.counter(
+            core_stats, "http2_op_send_message")
+        stats[
+            "core_http2_op_send_trailing_metadata"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_op_send_trailing_metadata")
+        stats[
+            "core_http2_op_recv_initial_metadata"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_op_recv_initial_metadata")
+        stats["core_http2_op_recv_message"] = massage_qps_stats_helpers.counter(
+            core_stats, "http2_op_recv_message")
+        stats[
+            "core_http2_op_recv_trailing_metadata"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_op_recv_trailing_metadata")
+        stats["core_http2_settings_writes"] = massage_qps_stats_helpers.counter(
+            core_stats, "http2_settings_writes")
+        stats["core_http2_pings_sent"] = massage_qps_stats_helpers.counter(
+            core_stats, "http2_pings_sent")
+        stats["core_http2_writes_begun"] = massage_qps_stats_helpers.counter(
+            core_stats, "http2_writes_begun")
+        stats[
+            "core_http2_writes_offloaded"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_writes_offloaded")
+        stats[
+            "core_http2_writes_continued"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_writes_continued")
+        stats["core_http2_partial_writes"] = massage_qps_stats_helpers.counter(
+            core_stats, "http2_partial_writes")
+        stats[
+            "core_http2_initiate_write_due_to_initial_write"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_initiate_write_due_to_initial_write")
+        stats[
+            "core_http2_initiate_write_due_to_start_new_stream"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_initiate_write_due_to_start_new_stream")
+        stats[
+            "core_http2_initiate_write_due_to_send_message"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_initiate_write_due_to_send_message")
+        stats[
+            "core_http2_initiate_write_due_to_send_initial_metadata"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_initiate_write_due_to_send_initial_metadata")
+        stats[
+            "core_http2_initiate_write_due_to_send_trailing_metadata"] = massage_qps_stats_helpers.counter(
+                core_stats,
+                "http2_initiate_write_due_to_send_trailing_metadata")
+        stats[
+            "core_http2_initiate_write_due_to_retry_send_ping"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_initiate_write_due_to_retry_send_ping")
+        stats[
+            "core_http2_initiate_write_due_to_continue_pings"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_initiate_write_due_to_continue_pings")
+        stats[
+            "core_http2_initiate_write_due_to_goaway_sent"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_initiate_write_due_to_goaway_sent")
+        stats[
+            "core_http2_initiate_write_due_to_rst_stream"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_initiate_write_due_to_rst_stream")
+        stats[
+            "core_http2_initiate_write_due_to_close_from_api"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_initiate_write_due_to_close_from_api")
+        stats[
+            "core_http2_initiate_write_due_to_stream_flow_control"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_initiate_write_due_to_stream_flow_control")
+        stats[
+            "core_http2_initiate_write_due_to_transport_flow_control"] = massage_qps_stats_helpers.counter(
+                core_stats,
+                "http2_initiate_write_due_to_transport_flow_control")
+        stats[
+            "core_http2_initiate_write_due_to_send_settings"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_initiate_write_due_to_send_settings")
+        stats[
+            "core_http2_initiate_write_due_to_bdp_estimator_ping"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_initiate_write_due_to_bdp_estimator_ping")
+        stats[
+            "core_http2_initiate_write_due_to_flow_control_unstalled_by_setting"] = massage_qps_stats_helpers.counter(
+                core_stats,
+                "http2_initiate_write_due_to_flow_control_unstalled_by_setting")
+        stats[
+            "core_http2_initiate_write_due_to_flow_control_unstalled_by_update"] = massage_qps_stats_helpers.counter(
+                core_stats,
+                "http2_initiate_write_due_to_flow_control_unstalled_by_update")
+        stats[
+            "core_http2_initiate_write_due_to_application_ping"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_initiate_write_due_to_application_ping")
+        stats[
+            "core_http2_initiate_write_due_to_keepalive_ping"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_initiate_write_due_to_keepalive_ping")
+        stats[
+            "core_http2_initiate_write_due_to_transport_flow_control_unstalled"] = massage_qps_stats_helpers.counter(
+                core_stats,
+                "http2_initiate_write_due_to_transport_flow_control_unstalled")
+        stats[
+            "core_http2_initiate_write_due_to_ping_response"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_initiate_write_due_to_ping_response")
+        stats[
+            "core_http2_initiate_write_due_to_force_rst_stream"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_initiate_write_due_to_force_rst_stream")
+        stats[
+            "core_http2_spurious_writes_begun"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_spurious_writes_begun")
+        stats["core_hpack_recv_indexed"] = massage_qps_stats_helpers.counter(
+            core_stats, "hpack_recv_indexed")
+        stats[
+            "core_hpack_recv_lithdr_incidx"] = massage_qps_stats_helpers.counter(
+                core_stats, "hpack_recv_lithdr_incidx")
+        stats[
+            "core_hpack_recv_lithdr_incidx_v"] = massage_qps_stats_helpers.counter(
+                core_stats, "hpack_recv_lithdr_incidx_v")
+        stats[
+            "core_hpack_recv_lithdr_notidx"] = massage_qps_stats_helpers.counter(
+                core_stats, "hpack_recv_lithdr_notidx")
+        stats[
+            "core_hpack_recv_lithdr_notidx_v"] = massage_qps_stats_helpers.counter(
+                core_stats, "hpack_recv_lithdr_notidx_v")
+        stats[
+            "core_hpack_recv_lithdr_nvridx"] = massage_qps_stats_helpers.counter(
+                core_stats, "hpack_recv_lithdr_nvridx")
+        stats[
+            "core_hpack_recv_lithdr_nvridx_v"] = massage_qps_stats_helpers.counter(
+                core_stats, "hpack_recv_lithdr_nvridx_v")
+        stats[
+            "core_hpack_recv_uncompressed"] = massage_qps_stats_helpers.counter(
+                core_stats, "hpack_recv_uncompressed")
+        stats["core_hpack_recv_huffman"] = massage_qps_stats_helpers.counter(
+            core_stats, "hpack_recv_huffman")
+        stats["core_hpack_recv_binary"] = massage_qps_stats_helpers.counter(
+            core_stats, "hpack_recv_binary")
+        stats[
+            "core_hpack_recv_binary_base64"] = massage_qps_stats_helpers.counter(
+                core_stats, "hpack_recv_binary_base64")
+        stats["core_hpack_send_indexed"] = massage_qps_stats_helpers.counter(
+            core_stats, "hpack_send_indexed")
+        stats[
+            "core_hpack_send_lithdr_incidx"] = massage_qps_stats_helpers.counter(
+                core_stats, "hpack_send_lithdr_incidx")
+        stats[
+            "core_hpack_send_lithdr_incidx_v"] = massage_qps_stats_helpers.counter(
+                core_stats, "hpack_send_lithdr_incidx_v")
+        stats[
+            "core_hpack_send_lithdr_notidx"] = massage_qps_stats_helpers.counter(
+                core_stats, "hpack_send_lithdr_notidx")
+        stats[
+            "core_hpack_send_lithdr_notidx_v"] = massage_qps_stats_helpers.counter(
+                core_stats, "hpack_send_lithdr_notidx_v")
+        stats[
+            "core_hpack_send_lithdr_nvridx"] = massage_qps_stats_helpers.counter(
+                core_stats, "hpack_send_lithdr_nvridx")
+        stats[
+            "core_hpack_send_lithdr_nvridx_v"] = massage_qps_stats_helpers.counter(
+                core_stats, "hpack_send_lithdr_nvridx_v")
+        stats[
+            "core_hpack_send_uncompressed"] = massage_qps_stats_helpers.counter(
+                core_stats, "hpack_send_uncompressed")
+        stats["core_hpack_send_huffman"] = massage_qps_stats_helpers.counter(
+            core_stats, "hpack_send_huffman")
+        stats["core_hpack_send_binary"] = massage_qps_stats_helpers.counter(
+            core_stats, "hpack_send_binary")
+        stats[
+            "core_hpack_send_binary_base64"] = massage_qps_stats_helpers.counter(
+                core_stats, "hpack_send_binary_base64")
+        stats[
+            "core_combiner_locks_initiated"] = massage_qps_stats_helpers.counter(
+                core_stats, "combiner_locks_initiated")
+        stats[
+            "core_combiner_locks_scheduled_items"] = massage_qps_stats_helpers.counter(
+                core_stats, "combiner_locks_scheduled_items")
+        stats[
+            "core_combiner_locks_scheduled_final_items"] = massage_qps_stats_helpers.counter(
+                core_stats, "combiner_locks_scheduled_final_items")
+        stats[
+            "core_combiner_locks_offloaded"] = massage_qps_stats_helpers.counter(
+                core_stats, "combiner_locks_offloaded")
+        stats[
+            "core_call_combiner_locks_initiated"] = massage_qps_stats_helpers.counter(
+                core_stats, "call_combiner_locks_initiated")
+        stats[
+            "core_call_combiner_locks_scheduled_items"] = massage_qps_stats_helpers.counter(
+                core_stats, "call_combiner_locks_scheduled_items")
+        stats[
+            "core_call_combiner_set_notify_on_cancel"] = massage_qps_stats_helpers.counter(
+                core_stats, "call_combiner_set_notify_on_cancel")
+        stats[
+            "core_call_combiner_cancelled"] = massage_qps_stats_helpers.counter(
+                core_stats, "call_combiner_cancelled")
+        stats[
+            "core_executor_scheduled_short_items"] = massage_qps_stats_helpers.counter(
+                core_stats, "executor_scheduled_short_items")
+        stats[
+            "core_executor_scheduled_long_items"] = massage_qps_stats_helpers.counter(
+                core_stats, "executor_scheduled_long_items")
+        stats[
+            "core_executor_scheduled_to_self"] = massage_qps_stats_helpers.counter(
+                core_stats, "executor_scheduled_to_self")
+        stats[
+            "core_executor_wakeup_initiated"] = massage_qps_stats_helpers.counter(
+                core_stats, "executor_wakeup_initiated")
+        stats[
+            "core_executor_queue_drained"] = massage_qps_stats_helpers.counter(
+                core_stats, "executor_queue_drained")
+        stats["core_executor_push_retries"] = massage_qps_stats_helpers.counter(
+            core_stats, "executor_push_retries")
+        stats[
+            "core_server_requested_calls"] = massage_qps_stats_helpers.counter(
+                core_stats, "server_requested_calls")
+        stats[
+            "core_server_slowpath_requests_queued"] = massage_qps_stats_helpers.counter(
+                core_stats, "server_slowpath_requests_queued")
+        stats[
+            "core_cq_ev_queue_trylock_failures"] = massage_qps_stats_helpers.counter(
+                core_stats, "cq_ev_queue_trylock_failures")
+        stats[
+            "core_cq_ev_queue_trylock_successes"] = massage_qps_stats_helpers.counter(
+                core_stats, "cq_ev_queue_trylock_successes")
+        stats[
+            "core_cq_ev_queue_transient_pop_failures"] = massage_qps_stats_helpers.counter(
+                core_stats, "cq_ev_queue_transient_pop_failures")
+        h = massage_qps_stats_helpers.histogram(core_stats, "call_initial_size")
+        stats["core_call_initial_size"] = ",".join("%f" % x for x in h.buckets)
+        stats["core_call_initial_size_bkts"] = ",".join(
+            "%f" % x for x in h.boundaries)
+        stats[
+            "core_call_initial_size_50p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 50, h.boundaries)
+        stats[
+            "core_call_initial_size_95p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 95, h.boundaries)
+        stats[
+            "core_call_initial_size_99p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 99, h.boundaries)
+        h = massage_qps_stats_helpers.histogram(core_stats,
+                                                "poll_events_returned")
+        stats["core_poll_events_returned"] = ",".join(
+            "%f" % x for x in h.buckets)
+        stats["core_poll_events_returned_bkts"] = ",".join(
+            "%f" % x for x in h.boundaries)
+        stats[
+            "core_poll_events_returned_50p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 50, h.boundaries)
+        stats[
+            "core_poll_events_returned_95p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 95, h.boundaries)
+        stats[
+            "core_poll_events_returned_99p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 99, h.boundaries)
+        h = massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size")
+        stats["core_tcp_write_size"] = ",".join("%f" % x for x in h.buckets)
+        stats["core_tcp_write_size_bkts"] = ",".join(
+            "%f" % x for x in h.boundaries)
+        stats["core_tcp_write_size_50p"] = massage_qps_stats_helpers.percentile(
+            h.buckets, 50, h.boundaries)
+        stats["core_tcp_write_size_95p"] = massage_qps_stats_helpers.percentile(
+            h.buckets, 95, h.boundaries)
+        stats["core_tcp_write_size_99p"] = massage_qps_stats_helpers.percentile(
+            h.buckets, 99, h.boundaries)
+        h = massage_qps_stats_helpers.histogram(core_stats,
+                                                "tcp_write_iov_size")
+        stats["core_tcp_write_iov_size"] = ",".join("%f" % x for x in h.buckets)
+        stats["core_tcp_write_iov_size_bkts"] = ",".join(
+            "%f" % x for x in h.boundaries)
+        stats[
+            "core_tcp_write_iov_size_50p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 50, h.boundaries)
+        stats[
+            "core_tcp_write_iov_size_95p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 95, h.boundaries)
+        stats[
+            "core_tcp_write_iov_size_99p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 99, h.boundaries)
+        h = massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size")
+        stats["core_tcp_read_size"] = ",".join("%f" % x for x in h.buckets)
+        stats["core_tcp_read_size_bkts"] = ",".join(
+            "%f" % x for x in h.boundaries)
+        stats["core_tcp_read_size_50p"] = massage_qps_stats_helpers.percentile(
+            h.buckets, 50, h.boundaries)
+        stats["core_tcp_read_size_95p"] = massage_qps_stats_helpers.percentile(
+            h.buckets, 95, h.boundaries)
+        stats["core_tcp_read_size_99p"] = massage_qps_stats_helpers.percentile(
+            h.buckets, 99, h.boundaries)
+        h = massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer")
+        stats["core_tcp_read_offer"] = ",".join("%f" % x for x in h.buckets)
+        stats["core_tcp_read_offer_bkts"] = ",".join(
+            "%f" % x for x in h.boundaries)
+        stats["core_tcp_read_offer_50p"] = massage_qps_stats_helpers.percentile(
+            h.buckets, 50, h.boundaries)
+        stats["core_tcp_read_offer_95p"] = massage_qps_stats_helpers.percentile(
+            h.buckets, 95, h.boundaries)
+        stats["core_tcp_read_offer_99p"] = massage_qps_stats_helpers.percentile(
+            h.buckets, 99, h.boundaries)
+        h = massage_qps_stats_helpers.histogram(core_stats,
+                                                "tcp_read_offer_iov_size")
+        stats["core_tcp_read_offer_iov_size"] = ",".join(
+            "%f" % x for x in h.buckets)
+        stats["core_tcp_read_offer_iov_size_bkts"] = ",".join(
+            "%f" % x for x in h.boundaries)
+        stats[
+            "core_tcp_read_offer_iov_size_50p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 50, h.boundaries)
+        stats[
+            "core_tcp_read_offer_iov_size_95p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 95, h.boundaries)
+        stats[
+            "core_tcp_read_offer_iov_size_99p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 99, h.boundaries)
+        h = massage_qps_stats_helpers.histogram(core_stats,
+                                                "http2_send_message_size")
+        stats["core_http2_send_message_size"] = ",".join(
+            "%f" % x for x in h.buckets)
+        stats["core_http2_send_message_size_bkts"] = ",".join(
+            "%f" % x for x in h.boundaries)
+        stats[
+            "core_http2_send_message_size_50p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 50, h.boundaries)
+        stats[
+            "core_http2_send_message_size_95p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 95, h.boundaries)
+        stats[
+            "core_http2_send_message_size_99p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 99, h.boundaries)
+        h = massage_qps_stats_helpers.histogram(
+            core_stats, "http2_send_initial_metadata_per_write")
+        stats["core_http2_send_initial_metadata_per_write"] = ",".join(
+            "%f" % x for x in h.buckets)
+        stats["core_http2_send_initial_metadata_per_write_bkts"] = ",".join(
+            "%f" % x for x in h.boundaries)
+        stats[
+            "core_http2_send_initial_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 50, h.boundaries)
+        stats[
+            "core_http2_send_initial_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 95, h.boundaries)
+        stats[
+            "core_http2_send_initial_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 99, h.boundaries)
+        h = massage_qps_stats_helpers.histogram(core_stats,
+                                                "http2_send_message_per_write")
+        stats["core_http2_send_message_per_write"] = ",".join(
+            "%f" % x for x in h.buckets)
+        stats["core_http2_send_message_per_write_bkts"] = ",".join(
+            "%f" % x for x in h.boundaries)
+        stats[
+            "core_http2_send_message_per_write_50p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 50, h.boundaries)
+        stats[
+            "core_http2_send_message_per_write_95p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 95, h.boundaries)
+        stats[
+            "core_http2_send_message_per_write_99p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 99, h.boundaries)
+        h = massage_qps_stats_helpers.histogram(
+            core_stats, "http2_send_trailing_metadata_per_write")
+        stats["core_http2_send_trailing_metadata_per_write"] = ",".join(
+            "%f" % x for x in h.buckets)
+        stats["core_http2_send_trailing_metadata_per_write_bkts"] = ",".join(
+            "%f" % x for x in h.boundaries)
+        stats[
+            "core_http2_send_trailing_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 50, h.boundaries)
+        stats[
+            "core_http2_send_trailing_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 95, h.boundaries)
+        stats[
+            "core_http2_send_trailing_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 99, h.boundaries)
+        h = massage_qps_stats_helpers.histogram(core_stats,
+                                                "http2_send_flowctl_per_write")
+        stats["core_http2_send_flowctl_per_write"] = ",".join(
+            "%f" % x for x in h.buckets)
+        stats["core_http2_send_flowctl_per_write_bkts"] = ",".join(
+            "%f" % x for x in h.boundaries)
+        stats[
+            "core_http2_send_flowctl_per_write_50p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 50, h.boundaries)
+        stats[
+            "core_http2_send_flowctl_per_write_95p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 95, h.boundaries)
+        stats[
+            "core_http2_send_flowctl_per_write_99p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 99, h.boundaries)
+        h = massage_qps_stats_helpers.histogram(core_stats,
+                                                "server_cqs_checked")
+        stats["core_server_cqs_checked"] = ",".join("%f" % x for x in h.buckets)
+        stats["core_server_cqs_checked_bkts"] = ",".join(
+            "%f" % x for x in h.boundaries)
+        stats[
+            "core_server_cqs_checked_50p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 50, h.boundaries)
+        stats[
+            "core_server_cqs_checked_95p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 95, h.boundaries)
+        stats[
+            "core_server_cqs_checked_99p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 99, h.boundaries)

tools/run_tests/performance/scenario_result_schema.json

@@ -180,6 +180,16 @@
         "name": "core_pollset_kick_own_thread", 
         "type": "INTEGER"
       }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_syscall_epoll_ctl", 
+        "type": "INTEGER"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_pollset_fd_cache_hits", 
+        "type": "INTEGER"
+      }, 
       {
         "mode": "NULLABLE", 
         "name": "core_histogram_slow_lookups", 
@@ -1012,6 +1022,16 @@
         "name": "core_pollset_kick_own_thread", 
         "type": "INTEGER"
       }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_syscall_epoll_ctl", 
+        "type": "INTEGER"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_pollset_fd_cache_hits", 
+        "type": "INTEGER"
+      }, 
       {
         "mode": "NULLABLE", 
         "name": "core_histogram_slow_lookups",