Optimize framing a little

- rely on the fact that data-to-come holds a reference to
data-being-written, so there's no need to add a ref for every frame
written
- provide an 'inlined' version of grpc_slice_malloc (via a #define) that
gives the compiler more information about small allocations to enable
better optimization

include/grpc/slice.h

@@ -75,6 +75,13 @@ GPRAPI grpc_slice grpc_slice_new_with_len(void *p, size_t len,
    call.
    Aborts if malloc() fails. */
 GPRAPI grpc_slice grpc_slice_malloc(size_t length);
+GPRAPI grpc_slice grpc_slice_malloc_large(size_t length);
+
+#define GRPC_SLICE_MALLOC(len)                                    \
+  ((len) <= GRPC_SLICE_INLINED_SIZE                               \
+       ? (grpc_slice){.refcount = NULL,                           \
+                      .data.inlined = {.length = (uint8_t)(len)}} \
+       : grpc_slice_malloc_large((len)))
 
 /* Intern a slice:
 
@@ -117,6 +124,9 @@ GPRAPI grpc_slice grpc_slice_sub_no_ref(grpc_slice s, size_t begin, size_t end);
    Requires s intialized, split <= s.length */
 GPRAPI grpc_slice grpc_slice_split_tail(grpc_slice *s, size_t split);
 
+/* The same as grpc_slice_split_tail, but without altering refcounts */
+GPRAPI grpc_slice grpc_slice_split_tail_no_ref(grpc_slice *s, size_t split);
+
 /* Splits s into two: modifies s to be s[split:s.length], and returns a new
    slice, sharing a refcount with s, that contains s[0:split].
    Requires s intialized, split <= s.length */

include/grpc/slice_buffer.h

@@ -77,6 +77,10 @@ GPRAPI void grpc_slice_buffer_trim_end(grpc_slice_buffer *src, size_t n,
 /* move the first n bytes of src into dst */
 GPRAPI void grpc_slice_buffer_move_first(grpc_slice_buffer *src, size_t n,
                                          grpc_slice_buffer *dst);
+/* move the first n bytes of src into dst without adding references */
+GPRAPI void grpc_slice_buffer_move_first_no_ref(grpc_slice_buffer *src,
+                                                size_t n,
+                                                grpc_slice_buffer *dst);
 /* move the first n bytes of src into dst (copying them) */
 GPRAPI void grpc_slice_buffer_move_first_into_buffer(grpc_exec_ctx *exec_ctx,
                                                      grpc_slice_buffer *src,

src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.c

@@ -98,7 +98,7 @@ grpc_slice grpc_grpclb_request_encode(const grpc_grpclb_request *request) {
   pb_encode(&sizestream, grpc_lb_v1_LoadBalanceRequest_fields, request);
   encoded_length = sizestream.bytes_written;
 
-  slice = grpc_slice_malloc(encoded_length);
+  slice = GRPC_SLICE_MALLOC(encoded_length);
   outputstream =
       pb_ostream_from_buffer(GRPC_SLICE_START_PTR(slice), encoded_length);
   GPR_ASSERT(pb_encode(&outputstream, grpc_lb_v1_LoadBalanceRequest_fields,

src/core/ext/transport/chttp2/transport/bin_decoder.c

@@ -169,7 +169,7 @@ grpc_slice grpc_chttp2_base64_decode(grpc_exec_ctx *exec_ctx,
       }
     }
   }
-  output = grpc_slice_malloc(output_length);
+  output = GRPC_SLICE_MALLOC(output_length);
 
   ctx.input_cur = GRPC_SLICE_START_PTR(input);
   ctx.input_end = GRPC_SLICE_END_PTR(input);
@@ -193,7 +193,7 @@ grpc_slice grpc_chttp2_base64_decode_with_length(grpc_exec_ctx *exec_ctx,
                                                  grpc_slice input,
                                                  size_t output_length) {
   size_t input_length = GRPC_SLICE_LENGTH(input);
-  grpc_slice output = grpc_slice_malloc(output_length);
+  grpc_slice output = GRPC_SLICE_MALLOC(output_length);
   struct grpc_base64_decode_context ctx;
 
   // The length of a base64 string cannot be 4 * n + 1

src/core/ext/transport/chttp2/transport/bin_encoder.c

@@ -66,7 +66,7 @@ grpc_slice grpc_chttp2_base64_encode(grpc_slice input) {
   size_t input_triplets = input_length / 3;
   size_t tail_case = input_length % 3;
   size_t output_length = input_triplets * 4 + tail_xtra[tail_case];
-  grpc_slice output = grpc_slice_malloc(output_length);
+  grpc_slice output = GRPC_SLICE_MALLOC(output_length);
   uint8_t *in = GRPC_SLICE_START_PTR(input);
   char *out = (char *)GRPC_SLICE_START_PTR(output);
   size_t i;
@@ -119,7 +119,7 @@ grpc_slice grpc_chttp2_huffman_compress(grpc_slice input) {
     nbits += grpc_chttp2_huffsyms[*in].length;
   }
 
-  output = grpc_slice_malloc(nbits / 8 + (nbits % 8 != 0));
+  output = GRPC_SLICE_MALLOC(nbits / 8 + (nbits % 8 != 0));
   out = GRPC_SLICE_START_PTR(output);
   for (in = GRPC_SLICE_START_PTR(input); in != GRPC_SLICE_END_PTR(input);
        ++in) {
@@ -184,7 +184,7 @@ grpc_slice grpc_chttp2_base64_encode_and_huffman_compress(grpc_slice input) {
   size_t output_syms = input_triplets * 4 + tail_xtra[tail_case];
   size_t max_output_bits = 11 * output_syms;
   size_t max_output_length = max_output_bits / 8 + (max_output_bits % 8 != 0);
-  grpc_slice output = grpc_slice_malloc(max_output_length);
+  grpc_slice output = GRPC_SLICE_MALLOC(max_output_length);
   uint8_t *in = GRPC_SLICE_START_PTR(input);
   uint8_t *start_out = GRPC_SLICE_START_PTR(output);
   huff_out out;

src/core/ext/transport/chttp2/transport/chttp2_transport.c

@@ -1906,7 +1906,7 @@ static void close_from_api(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
      the time we got around to sending this, so instead we ignore HPACK
      compression and just write the uncompressed bytes onto the wire. */
   if (!s->sent_initial_metadata) {
-    http_status_hdr = grpc_slice_malloc(13);
+    http_status_hdr = GRPC_SLICE_MALLOC(13);
     p = GRPC_SLICE_START_PTR(http_status_hdr);
     *p++ = 0x00;
     *p++ = 7;
@@ -1925,7 +1925,7 @@ static void close_from_api(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
     len += (uint32_t)GRPC_SLICE_LENGTH(http_status_hdr);
   }
 
-  status_hdr = grpc_slice_malloc(15 + (grpc_status >= 10));
+  status_hdr = GRPC_SLICE_MALLOC(15 + (grpc_status >= 10));
   p = GRPC_SLICE_START_PTR(status_hdr);
   *p++ = 0x00; /* literal header, not indexed */
   *p++ = 11;   /* len(grpc-status) */
@@ -1954,7 +1954,7 @@ static void close_from_api(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
   size_t msg_len = GRPC_SLICE_LENGTH(slice);
   GPR_ASSERT(msg_len <= UINT32_MAX);
   uint32_t msg_len_len = GRPC_CHTTP2_VARINT_LENGTH((uint32_t)msg_len, 1);
-  message_pfx = grpc_slice_malloc(14 + msg_len_len);
+  message_pfx = GRPC_SLICE_MALLOC(14 + msg_len_len);
   p = GRPC_SLICE_START_PTR(message_pfx);
   *p++ = 0x00; /* literal header, not indexed */
   *p++ = 12;   /* len(grpc-message) */
@@ -1976,7 +1976,7 @@ static void close_from_api(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
   len += (uint32_t)GRPC_SLICE_LENGTH(message_pfx);
   len += (uint32_t)msg_len;
 
-  hdr = grpc_slice_malloc(9);
+  hdr = GRPC_SLICE_MALLOC(9);
   p = GRPC_SLICE_START_PTR(hdr);
   *p++ = (uint8_t)(len >> 16);
   *p++ = (uint8_t)(len >> 8);

src/core/ext/transport/chttp2/transport/frame_data.c

@@ -123,7 +123,7 @@ void grpc_chttp2_encode_data(uint32_t id, grpc_slice_buffer *inbuf,
   uint8_t *p;
   static const size_t header_size = 9;
 
-  hdr = grpc_slice_malloc(header_size);
+  hdr = GRPC_SLICE_MALLOC(header_size);
   p = GRPC_SLICE_START_PTR(hdr);
   GPR_ASSERT(write_bytes < (1 << 24));
   *p++ = (uint8_t)(write_bytes >> 16);
@@ -137,7 +137,7 @@ void grpc_chttp2_encode_data(uint32_t id, grpc_slice_buffer *inbuf,
   *p++ = (uint8_t)(id);
   grpc_slice_buffer_add(outbuf, hdr);
 
-  grpc_slice_buffer_move_first(inbuf, write_bytes, outbuf);
+  grpc_slice_buffer_move_first_no_ref(inbuf, write_bytes, outbuf);
 
   stats->framing_bytes += header_size;
   stats->data_bytes += write_bytes;

src/core/ext/transport/chttp2/transport/frame_goaway.c

@@ -163,7 +163,7 @@ grpc_error *grpc_chttp2_goaway_parser_parse(grpc_exec_ctx *exec_ctx,
 void grpc_chttp2_goaway_append(uint32_t last_stream_id, uint32_t error_code,
                                grpc_slice debug_data,
                                grpc_slice_buffer *slice_buffer) {
-  grpc_slice header = grpc_slice_malloc(9 + 4 + 4);
+  grpc_slice header = GRPC_SLICE_MALLOC(9 + 4 + 4);
   uint8_t *p = GRPC_SLICE_START_PTR(header);
   uint32_t frame_length;
   GPR_ASSERT(GRPC_SLICE_LENGTH(debug_data) < UINT32_MAX - 4 - 4);

src/core/ext/transport/chttp2/transport/frame_ping.c

@@ -43,7 +43,7 @@
 static bool g_disable_ping_ack = false;
 
 grpc_slice grpc_chttp2_ping_create(uint8_t ack, uint64_t opaque_8bytes) {
-  grpc_slice slice = grpc_slice_malloc(9 + 8);
+  grpc_slice slice = GRPC_SLICE_MALLOC(9 + 8);
   uint8_t *p = GRPC_SLICE_START_PTR(slice);
 
   *p++ = 0;

src/core/ext/transport/chttp2/transport/frame_rst_stream.c

@@ -44,7 +44,7 @@
 grpc_slice grpc_chttp2_rst_stream_create(uint32_t id, uint32_t code,
                                          grpc_transport_one_way_stats *stats) {
   static const size_t frame_size = 13;
-  grpc_slice slice = grpc_slice_malloc(frame_size);
+  grpc_slice slice = GRPC_SLICE_MALLOC(frame_size);
   stats->framing_bytes += frame_size;
   uint8_t *p = GRPC_SLICE_START_PTR(slice);
 

src/core/ext/transport/chttp2/transport/frame_settings.c

@@ -93,7 +93,7 @@ grpc_slice grpc_chttp2_settings_create(uint32_t *old, const uint32_t *new,
     n += (new[i] != old[i] || (force_mask & (1u << i)) != 0);
   }
 
-  output = grpc_slice_malloc(9 + 6 * n);
+  output = GRPC_SLICE_MALLOC(9 + 6 * n);
   p = fill_header(GRPC_SLICE_START_PTR(output), 6 * n, 0);
 
   for (i = 0; i < count; i++) {
@@ -115,7 +115,7 @@ grpc_slice grpc_chttp2_settings_create(uint32_t *old, const uint32_t *new,
 }
 
 grpc_slice grpc_chttp2_settings_ack_create(void) {
-  grpc_slice output = grpc_slice_malloc(9);
+  grpc_slice output = GRPC_SLICE_MALLOC(9);
   fill_header(GRPC_SLICE_START_PTR(output), 0, GRPC_CHTTP2_FLAG_ACK);
   return output;
 }

src/core/ext/transport/chttp2/transport/frame_window_update.c

@@ -41,7 +41,7 @@
 grpc_slice grpc_chttp2_window_update_create(
     uint32_t id, uint32_t window_update, grpc_transport_one_way_stats *stats) {
   static const size_t frame_size = 13;
-  grpc_slice slice = grpc_slice_malloc(frame_size);
+  grpc_slice slice = GRPC_SLICE_MALLOC(frame_size);
   stats->header_bytes += frame_size;
   uint8_t *p = GRPC_SLICE_START_PTR(slice);
 

src/core/ext/transport/chttp2/transport/hpack_encoder.c

@@ -122,7 +122,7 @@ static void finish_frame(framer_state *st, int is_header_boundary,
    output before beginning */
 static void begin_frame(framer_state *st) {
   st->header_idx =
-      grpc_slice_buffer_add_indexed(st->output, grpc_slice_malloc(9));
+      grpc_slice_buffer_add_indexed(st->output, GRPC_SLICE_MALLOC(9));
   st->output_length_at_start_of_frame = st->output->length;
 }
 

src/core/ext/transport/cronet/transport/cronet_transport.c

@@ -1165,7 +1165,7 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx,
     } else if (stream_state->rs.remaining_bytes == 0) {
       CRONET_LOG(GPR_DEBUG, "read operation complete");
       grpc_slice read_data_slice =
-          grpc_slice_malloc((uint32_t)stream_state->rs.length_field);
+          GRPC_SLICE_MALLOC((uint32_t)stream_state->rs.length_field);
       uint8_t *dst_p = GRPC_SLICE_START_PTR(read_data_slice);
       memcpy(dst_p, stream_state->rs.read_buffer,
              (size_t)stream_state->rs.length_field);

src/core/lib/channel/http_client_filter.c

@@ -312,7 +312,7 @@ static grpc_error *hc_mutate_op(grpc_exec_ctx *exec_ctx,
             op->payload->send_message.send_message->length, k_url_safe,
             k_multi_line);
         estimated_len += 1; /* for the trailing 0 */
-        grpc_slice path_with_query_slice = grpc_slice_malloc(estimated_len);
+        grpc_slice path_with_query_slice = GRPC_SLICE_MALLOC(estimated_len);
 
         /* memcopy individual pieces into this slice */
         uint8_t *write_ptr =

src/core/lib/compression/message_compress.c

@@ -50,7 +50,7 @@ static int zlib_body(grpc_exec_ctx* exec_ctx, z_stream* zs,
   int r;
   int flush;
   size_t i;
-  grpc_slice outbuf = grpc_slice_malloc(OUTPUT_BLOCK_SIZE);
+  grpc_slice outbuf = GRPC_SLICE_MALLOC(OUTPUT_BLOCK_SIZE);
   const uInt uint_max = ~(uInt)0;
 
   GPR_ASSERT(GRPC_SLICE_LENGTH(outbuf) <= uint_max);
@@ -65,7 +65,7 @@ static int zlib_body(grpc_exec_ctx* exec_ctx, z_stream* zs,
     do {
       if (zs->avail_out == 0) {
         grpc_slice_buffer_add_indexed(output, outbuf);
-        outbuf = grpc_slice_malloc(OUTPUT_BLOCK_SIZE);
+        outbuf = GRPC_SLICE_MALLOC(OUTPUT_BLOCK_SIZE);
         GPR_ASSERT(GRPC_SLICE_LENGTH(outbuf) <= uint_max);
         zs->avail_out = (uInt)GRPC_SLICE_LENGTH(outbuf);
         zs->next_out = GRPC_SLICE_START_PTR(outbuf);

src/core/lib/iomgr/tcp_windows.c

@@ -219,7 +219,7 @@ static void win_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
   tcp->read_slices = read_slices;
   grpc_slice_buffer_reset_and_unref_internal(exec_ctx, read_slices);
 
-  tcp->read_slice = grpc_slice_malloc(8192);
+  tcp->read_slice = GRPC_SLICE_MALLOC(8192);
 
   buffer.len = (ULONG)GRPC_SLICE_LENGTH(
       tcp->read_slice);  // we know slice size fits in 32bit.

src/core/lib/security/transport/secure_endpoint.c

@@ -130,7 +130,7 @@ static void secure_endpoint_ref(secure_endpoint *ep) { gpr_ref(&ep->ref); }
 static void flush_read_staging_buffer(secure_endpoint *ep, uint8_t **cur,
                                       uint8_t **end) {
   grpc_slice_buffer_add(ep->read_buffer, ep->read_staging_buffer);
-  ep->read_staging_buffer = grpc_slice_malloc(STAGING_BUFFER_SIZE);
+  ep->read_staging_buffer = GRPC_SLICE_MALLOC(STAGING_BUFFER_SIZE);
   *cur = GRPC_SLICE_START_PTR(ep->read_staging_buffer);
   *end = GRPC_SLICE_END_PTR(ep->read_staging_buffer);
 }
@@ -252,7 +252,7 @@ static void endpoint_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *secure_ep,
 static void flush_write_staging_buffer(secure_endpoint *ep, uint8_t **cur,
                                        uint8_t **end) {
   grpc_slice_buffer_add(&ep->output_buffer, ep->write_staging_buffer);
-  ep->write_staging_buffer = grpc_slice_malloc(STAGING_BUFFER_SIZE);
+  ep->write_staging_buffer = GRPC_SLICE_MALLOC(STAGING_BUFFER_SIZE);
   *cur = GRPC_SLICE_START_PTR(ep->write_staging_buffer);
   *end = GRPC_SLICE_END_PTR(ep->write_staging_buffer);
 }
@@ -415,8 +415,8 @@ grpc_endpoint *grpc_secure_endpoint_create(
     grpc_slice_buffer_add(&ep->leftover_bytes,
                           grpc_slice_ref_internal(leftover_slices[i]));
   }
-  ep->write_staging_buffer = grpc_slice_malloc(STAGING_BUFFER_SIZE);
-  ep->read_staging_buffer = grpc_slice_malloc(STAGING_BUFFER_SIZE);
+  ep->write_staging_buffer = GRPC_SLICE_MALLOC(STAGING_BUFFER_SIZE);
+  ep->read_staging_buffer = GRPC_SLICE_MALLOC(STAGING_BUFFER_SIZE);
   grpc_slice_buffer_init(&ep->output_buffer);
   grpc_slice_buffer_init(&ep->source_buffer);
   ep->read_buffer = NULL;

src/core/lib/slice/b64.c

@@ -202,7 +202,7 @@ static int decode_group(const unsigned char *codes, size_t num_codes,
 
 grpc_slice grpc_base64_decode_with_len(grpc_exec_ctx *exec_ctx, const char *b64,
                                        size_t b64_len, int url_safe) {
-  grpc_slice result = grpc_slice_malloc(b64_len);
+  grpc_slice result = GRPC_SLICE_MALLOC(b64_len);
   unsigned char *current = GRPC_SLICE_START_PTR(result);
   size_t result_size = 0;
   unsigned char codes[4];

src/core/lib/slice/percent_encoding.c

@@ -71,7 +71,7 @@ grpc_slice grpc_percent_encode_slice(grpc_slice slice,
     return grpc_slice_ref_internal(slice);
   }
   // second pass: actually encode
-  grpc_slice out = grpc_slice_malloc(output_length);
+  grpc_slice out = GRPC_SLICE_MALLOC(output_length);
   uint8_t *q = GRPC_SLICE_START_PTR(out);
   for (p = slice_start; p < slice_end; p++) {
     if (is_unreserved_character(*p, unreserved_bytes)) {
@@ -125,7 +125,7 @@ bool grpc_strict_percent_decode_slice(grpc_slice slice_in,
     return true;
   }
   p = GRPC_SLICE_START_PTR(slice_in);
-  *slice_out = grpc_slice_malloc(out_length);
+  *slice_out = GRPC_SLICE_MALLOC(out_length);
   uint8_t *q = GRPC_SLICE_START_PTR(*slice_out);
   while (p != in_end) {
     if (*p == '%') {
@@ -163,7 +163,7 @@ grpc_slice grpc_permissive_percent_decode_slice(grpc_slice slice_in) {
     return grpc_slice_ref_internal(slice_in);
   }
   p = GRPC_SLICE_START_PTR(slice_in);
-  grpc_slice out = grpc_slice_malloc(out_length);
+  grpc_slice out = GRPC_SLICE_MALLOC(out_length);
   uint8_t *q = GRPC_SLICE_START_PTR(out);
   while (p != in_end) {
     if (*p == '%') {

src/core/lib/slice/slice.c

@@ -197,7 +197,7 @@ grpc_slice grpc_slice_new_with_len(void *p, size_t len,
 }
 
 grpc_slice grpc_slice_from_copied_buffer(const char *source, size_t length) {
-  grpc_slice slice = grpc_slice_malloc(length);
+  grpc_slice slice = GRPC_SLICE_MALLOC(length);
   memcpy(GRPC_SLICE_START_PTR(slice), source, length);
   return slice;
 }
@@ -227,35 +227,42 @@ static const grpc_slice_refcount_vtable malloc_vtable = {
     malloc_ref, malloc_unref, grpc_slice_default_eq_impl,
     grpc_slice_default_hash_impl};
 
+grpc_slice grpc_slice_malloc_large(size_t length) {
+  grpc_slice slice;
+
+  /* Memory layout used by the slice created here:
+
+     +-----------+----------------------------------------------------------+
+     | refcount  | bytes                                                    |
+     +-----------+----------------------------------------------------------+
+
+     refcount is a malloc_refcount
+     bytes is an array of bytes of the requested length
+     Both parts are placed in the same allocation returned from gpr_malloc */
+  malloc_refcount *rc = gpr_malloc(sizeof(malloc_refcount) + length);
+
+  /* Initial refcount on rc is 1 - and it's up to the caller to release
+     this reference. */
+  gpr_ref_init(&rc->refs, 1);
+
+  rc->base.vtable = &malloc_vtable;
+  rc->base.sub_refcount = &rc->base;
+
+  /* Build up the slice to be returned. */
+  /* The slices refcount points back to the allocated block. */
+  slice.refcount = &rc->base;
+  /* The data bytes are placed immediately after the refcount struct */
+  slice.data.refcounted.bytes = (uint8_t *)(rc + 1);
+  /* And the length of the block is set to the requested length */
+  slice.data.refcounted.length = length;
+  return slice;
+}
+
 grpc_slice grpc_slice_malloc(size_t length) {
   grpc_slice slice;
 
   if (length > sizeof(slice.data.inlined.bytes)) {
-    /* Memory layout used by the slice created here:
-
-       +-----------+----------------------------------------------------------+
-       | refcount  | bytes                                                    |
-       +-----------+----------------------------------------------------------+
-
-       refcount is a malloc_refcount
-       bytes is an array of bytes of the requested length
-       Both parts are placed in the same allocation returned from gpr_malloc */
-    malloc_refcount *rc = gpr_malloc(sizeof(malloc_refcount) + length);
-
-    /* Initial refcount on rc is 1 - and it's up to the caller to release
-       this reference. */
-    gpr_ref_init(&rc->refs, 1);
-
-    rc->base.vtable = &malloc_vtable;
-    rc->base.sub_refcount = &rc->base;
-
-    /* Build up the slice to be returned. */
-    /* The slices refcount points back to the allocated block. */
-    slice.refcount = &rc->base;
-    /* The data bytes are placed immediately after the refcount struct */
-    slice.data.refcounted.bytes = (uint8_t *)(rc + 1);
-    /* And the length of the block is set to the requested length */
-    slice.data.refcounted.length = length;
+    return grpc_slice_malloc_large(length);
   } else {
     /* small slice: just inline the data */
     slice.refcount = NULL;
@@ -341,6 +348,40 @@ grpc_slice grpc_slice_split_tail(grpc_slice *source, size_t split) {
   return tail;
 }
 
+grpc_slice grpc_slice_split_tail_no_ref(grpc_slice *source, size_t split) {
+  grpc_slice tail;
+
+  if (source->refcount == NULL) {
+    /* inlined data, copy it out */
+    GPR_ASSERT(source->data.inlined.length >= split);
+    tail.refcount = NULL;
+    tail.data.inlined.length = (uint8_t)(source->data.inlined.length - split);
+    memcpy(tail.data.inlined.bytes, source->data.inlined.bytes + split,
+           tail.data.inlined.length);
+    source->data.inlined.length = (uint8_t)split;
+  } else {
+    size_t tail_length = source->data.refcounted.length - split;
+    GPR_ASSERT(source->data.refcounted.length >= split);
+    if (tail_length < sizeof(tail.data.inlined.bytes)) {
+      /* Copy out the bytes - it'll be cheaper than refcounting */
+      tail.refcount = NULL;
+      tail.data.inlined.length = (uint8_t)tail_length;
+      memcpy(tail.data.inlined.bytes, source->data.refcounted.bytes + split,
+             tail_length);
+    } else {
+      /* Build the result */
+      tail.refcount = &noop_refcount;
+      /* Point into the source array */
+      tail.data.refcounted.bytes = source->data.refcounted.bytes + split;
+      tail.data.refcounted.length = tail_length;
+    }
+    source->refcount = source->refcount->sub_refcount;
+    source->data.refcounted.length = split;
+  }
+
+  return tail;
+}
+
 grpc_slice grpc_slice_split_head(grpc_slice *source, size_t split) {
   grpc_slice head;
 
@@ -457,7 +498,7 @@ int grpc_slice_slice(grpc_slice haystack, grpc_slice needle) {
 }
 
 grpc_slice grpc_slice_dup(grpc_slice a) {
-  grpc_slice copy = grpc_slice_malloc(GRPC_SLICE_LENGTH(a));
+  grpc_slice copy = GRPC_SLICE_MALLOC(GRPC_SLICE_LENGTH(a));
   memcpy(GRPC_SLICE_START_PTR(copy), GRPC_SLICE_START_PTR(a),
          GRPC_SLICE_LENGTH(a));
   return copy;

src/core/lib/slice/slice_buffer.c

@@ -255,14 +255,47 @@ void grpc_slice_buffer_move_into(grpc_slice_buffer *src,
 
 void grpc_slice_buffer_move_first(grpc_slice_buffer *src, size_t n,
                                   grpc_slice_buffer *dst) {
+  GPR_ASSERT(src->length >= n);
+  if (src->length == n) {
+    grpc_slice_buffer_move_into(src, dst);
+    return;
+  }
+
   size_t output_len = dst->length + n;
   size_t new_input_len = src->length - n;
+
+  while (src->count > 0) {
+    grpc_slice slice = grpc_slice_buffer_take_first(src);
+    size_t slice_len = GRPC_SLICE_LENGTH(slice);
+    if (n > slice_len) {
+      grpc_slice_buffer_add(dst, slice);
+      n -= slice_len;
+    } else if (n == slice_len) {
+      grpc_slice_buffer_add(dst, slice);
+      break;
+    } else { /* n < slice_len */
+      grpc_slice_buffer_undo_take_first(src, grpc_slice_split_tail(&slice, n));
+      GPR_ASSERT(GRPC_SLICE_LENGTH(slice) == n);
+      grpc_slice_buffer_add(dst, slice);
+      break;
+    }
+  }
+  GPR_ASSERT(dst->length == output_len);
+  GPR_ASSERT(src->length == new_input_len);
+  GPR_ASSERT(src->count > 0);
+}
+
+void grpc_slice_buffer_move_first_no_ref(grpc_slice_buffer *src, size_t n,
+                                         grpc_slice_buffer *dst) {
   GPR_ASSERT(src->length >= n);
   if (src->length == n) {
     grpc_slice_buffer_move_into(src, dst);
     return;
   }
 
+  size_t output_len = dst->length + n;
+  size_t new_input_len = src->length - n;
+
   while (src->count > 0) {
     grpc_slice slice = grpc_slice_buffer_take_first(src);
     size_t slice_len = GRPC_SLICE_LENGTH(slice);
@@ -273,7 +306,8 @@ void grpc_slice_buffer_move_first(grpc_slice_buffer *src, size_t n,
       grpc_slice_buffer_add(dst, slice);
       break;
     } else { /* n < slice_len */
-      grpc_slice_buffer_undo_take_first(src, grpc_slice_split_tail(&slice, n));
+      grpc_slice_buffer_undo_take_first(
+          src, grpc_slice_split_tail_no_ref(&slice, n));
       GPR_ASSERT(GRPC_SLICE_LENGTH(slice) == n);
       grpc_slice_buffer_add(dst, slice);
       break;

src/core/lib/surface/byte_buffer_reader.c

@@ -124,7 +124,7 @@ grpc_slice grpc_byte_buffer_reader_readall(grpc_byte_buffer_reader *reader) {
   grpc_slice in_slice;
   size_t bytes_read = 0;
   const size_t input_size = grpc_byte_buffer_length(reader->buffer_out);
-  grpc_slice out_slice = grpc_slice_malloc(input_size);
+  grpc_slice out_slice = GRPC_SLICE_MALLOC(input_size);
   uint8_t *const outbuf = GRPC_SLICE_START_PTR(out_slice); /* just an alias */
 
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;