Merge github.com:grpc/grpc into rollfwd

grpc.def

@@ -177,7 +177,9 @@ EXPORTS
     grpc_slice_buffer_move_into
     grpc_slice_buffer_trim_end
     grpc_slice_buffer_move_first
+    grpc_slice_buffer_move_first_into_buffer
     grpc_slice_buffer_take_first
+    grpc_slice_buffer_undo_take_first
     gpr_malloc
     gpr_free
     gpr_realloc

include/grpc/impl/codegen/slice.h

@@ -106,11 +106,16 @@ struct grpc_slice {
 /* Represents an expandable array of slices, to be interpreted as a
    single item. */
 typedef struct {
-  /* slices in the array */
+  /* This is for internal use only. External users (i.e any code outside grpc
+   * core) MUST NOT use this field */
+  grpc_slice *base_slices;
+
+  /* slices in the array (Points to the first valid grpc_slice in the array) */
   grpc_slice *slices;
   /* the number of slices in the array */
   size_t count;
-  /* the number of slices allocated in the array */
+  /* the number of slices allocated in the array. External users (i.e any code
+   * outside grpc core) MUST NOT use this field */
   size_t capacity;
   /* the combined length of all slices in the array */
   size_t length;

include/grpc/slice_buffer.h

@@ -77,8 +77,15 @@ GPRAPI void grpc_slice_buffer_trim_end(grpc_slice_buffer *src, size_t n,
 /* move the first n bytes of src into dst */
 GPRAPI void grpc_slice_buffer_move_first(grpc_slice_buffer *src, size_t n,
                                          grpc_slice_buffer *dst);
+/* move the first n bytes of src into dst (copying them) */
+GPRAPI void grpc_slice_buffer_move_first_into_buffer(grpc_exec_ctx *exec_ctx,
+                                                     grpc_slice_buffer *src,
+                                                     size_t n, void *dst);
 /* take the first slice in the slice buffer */
 GPRAPI grpc_slice grpc_slice_buffer_take_first(grpc_slice_buffer *src);
+/* undo the above with (a possibly different) \a slice */
+GPRAPI void grpc_slice_buffer_undo_take_first(grpc_slice_buffer *src,
+                                              grpc_slice slice);
 
 #ifdef __cplusplus
 }

src/core/lib/iomgr/ev_epoll_linux.c

@@ -796,7 +796,7 @@ static polling_island *polling_island_merge(polling_island *p,
     gpr_atm_rel_store(&p->merged_to, (gpr_atm)q);
     PI_ADD_REF(q, "pi_merge"); /* To account for the new incoming ref from p */
 
-    workqueue_move_items_to_parent(q);
+    workqueue_move_items_to_parent(p);
   }
   /* else if p == q, nothing needs to be done */
 

src/core/lib/iomgr/ev_poll_posix.c

@@ -413,9 +413,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                       const char *reason) {
   fd->on_done_closure = on_done;
   fd->released = release_fd != NULL;
-  if (!fd->released) {
-    shutdown(fd->fd, SHUT_RDWR);
-  } else {
+  if (fd->released) {
     *release_fd = fd->fd;
   }
   gpr_mu_lock(&fd->mu);

src/core/lib/slice/slice_buffer.c

@@ -46,15 +46,27 @@
 #define GROW(x) (3 * (x) / 2)
 
 static void maybe_embiggen(grpc_slice_buffer *sb) {
-  if (sb->count == sb->capacity) {
+  if (sb->base_slices != sb->slices) {
+    memmove(sb->base_slices, sb->slices, sb->count * sizeof(grpc_slice));
+    sb->slices = sb->base_slices;
+  }
+
+  /* How far away from sb->base_slices is sb->slices pointer */
+  size_t slice_offset = (size_t)(sb->slices - sb->base_slices);
+  size_t slice_count = sb->count + slice_offset;
+
+  if (slice_count == sb->capacity) {
     sb->capacity = GROW(sb->capacity);
-    GPR_ASSERT(sb->capacity > sb->count);
-    if (sb->slices == sb->inlined) {
-      sb->slices = gpr_malloc(sb->capacity * sizeof(grpc_slice));
-      memcpy(sb->slices, sb->inlined, sb->count * sizeof(grpc_slice));
+    GPR_ASSERT(sb->capacity > slice_count);
+    if (sb->base_slices == sb->inlined) {
+      sb->base_slices = gpr_malloc(sb->capacity * sizeof(grpc_slice));
+      memcpy(sb->base_slices, sb->inlined, slice_count * sizeof(grpc_slice));
     } else {
-      sb->slices = gpr_realloc(sb->slices, sb->capacity * sizeof(grpc_slice));
+      sb->base_slices =
+          gpr_realloc(sb->base_slices, sb->capacity * sizeof(grpc_slice));
     }
+
+    sb->slices = sb->base_slices + slice_offset;
   }
 }
 
@@ -62,14 +74,14 @@ void grpc_slice_buffer_init(grpc_slice_buffer *sb) {
   sb->count = 0;
   sb->length = 0;
   sb->capacity = GRPC_SLICE_BUFFER_INLINE_ELEMENTS;
-  sb->slices = sb->inlined;
+  sb->base_slices = sb->slices = sb->inlined;
 }
 
 void grpc_slice_buffer_destroy_internal(grpc_exec_ctx *exec_ctx,
                                         grpc_slice_buffer *sb) {
   grpc_slice_buffer_reset_and_unref_internal(exec_ctx, sb);
-  if (sb->slices != sb->inlined) {
-    gpr_free(sb->slices);
+  if (sb->base_slices != sb->inlined) {
+    gpr_free(sb->base_slices);
   }
 }
 
@@ -166,7 +178,6 @@ void grpc_slice_buffer_pop(grpc_slice_buffer *sb) {
 void grpc_slice_buffer_reset_and_unref_internal(grpc_exec_ctx *exec_ctx,
                                                 grpc_slice_buffer *sb) {
   size_t i;
-
   for (i = 0; i < sb->count; i++) {
     grpc_slice_unref_internal(exec_ctx, sb->slices[i]);
   }
@@ -182,32 +193,45 @@ void grpc_slice_buffer_reset_and_unref(grpc_slice_buffer *sb) {
 }
 
 void grpc_slice_buffer_swap(grpc_slice_buffer *a, grpc_slice_buffer *b) {
-  GPR_SWAP(size_t, a->count, b->count);
-  GPR_SWAP(size_t, a->capacity, b->capacity);
-  GPR_SWAP(size_t, a->length, b->length);
+  size_t a_offset = (size_t)(a->slices - a->base_slices);
+  size_t b_offset = (size_t)(b->slices - b->base_slices);
 
-  if (a->slices == a->inlined) {
-    if (b->slices == b->inlined) {
+  size_t a_count = a->count + a_offset;
+  size_t b_count = b->count + b_offset;
+
+  if (a->base_slices == a->inlined) {
+    if (b->base_slices == b->inlined) {
       /* swap contents of inlined buffer */
       grpc_slice temp[GRPC_SLICE_BUFFER_INLINE_ELEMENTS];
-      memcpy(temp, a->slices, b->count * sizeof(grpc_slice));
-      memcpy(a->slices, b->slices, a->count * sizeof(grpc_slice));
-      memcpy(b->slices, temp, b->count * sizeof(grpc_slice));
+      memcpy(temp, a->base_slices, a_count * sizeof(grpc_slice));
+      memcpy(a->base_slices, b->base_slices, b_count * sizeof(grpc_slice));
+      memcpy(b->base_slices, temp, a_count * sizeof(grpc_slice));
     } else {
       /* a is inlined, b is not - copy a inlined into b, fix pointers */
-      a->slices = b->slices;
-      b->slices = b->inlined;
-      memcpy(b->slices, a->inlined, b->count * sizeof(grpc_slice));
+      a->base_slices = b->base_slices;
+      b->base_slices = b->inlined;
+      memcpy(b->base_slices, a->inlined, a_count * sizeof(grpc_slice));
     }
-  } else if (b->slices == b->inlined) {
+  } else if (b->base_slices == b->inlined) {
     /* b is inlined, a is not - copy b inlined int a, fix pointers */
-    b->slices = a->slices;
-    a->slices = a->inlined;
-    memcpy(a->slices, b->inlined, a->count * sizeof(grpc_slice));
+    b->base_slices = a->base_slices;
+    a->base_slices = a->inlined;
+    memcpy(a->base_slices, b->inlined, b_count * sizeof(grpc_slice));
   } else {
     /* no inlining: easy swap */
-    GPR_SWAP(grpc_slice *, a->slices, b->slices);
+    GPR_SWAP(grpc_slice *, a->base_slices, b->base_slices);
   }
+
+  /* Update the slices pointers (cannot do a GPR_SWAP on slices fields here).
+   * Also note that since the base_slices pointers are already swapped we need
+   * use 'b_offset' for 'a->base_slices' and vice versa */
+  a->slices = a->base_slices + b_offset;
+  b->slices = b->base_slices + a_offset;
+
+  /* base_slices and slices fields are correctly set. Swap all other fields */
+  GPR_SWAP(size_t, a->count, b->count);
+  GPR_SWAP(size_t, a->capacity, b->capacity);
+  GPR_SWAP(size_t, a->length, b->length);
 }
 
 void grpc_slice_buffer_move_into(grpc_slice_buffer *src,
@@ -229,7 +253,6 @@ void grpc_slice_buffer_move_into(grpc_slice_buffer *src,
 
 void grpc_slice_buffer_move_first(grpc_slice_buffer *src, size_t n,
                                   grpc_slice_buffer *dst) {
-  size_t src_idx;
   size_t output_len = dst->length + n;
   size_t new_input_len = src->length - n;
   GPR_ASSERT(src->length >= n);
@@ -237,34 +260,55 @@ void grpc_slice_buffer_move_first(grpc_slice_buffer *src, size_t n,
     grpc_slice_buffer_move_into(src, dst);
     return;
   }
-  src_idx = 0;
-  while (src_idx < src->capacity) {
-    grpc_slice slice = src->slices[src_idx];
+
+  while (src->count > 0) {
+    grpc_slice slice = grpc_slice_buffer_take_first(src);
     size_t slice_len = GRPC_SLICE_LENGTH(slice);
     if (n > slice_len) {
       grpc_slice_buffer_add(dst, slice);
       n -= slice_len;
-      src_idx++;
     } else if (n == slice_len) {
       grpc_slice_buffer_add(dst, slice);
-      src_idx++;
       break;
     } else { /* n < slice_len */
-      src->slices[src_idx] = grpc_slice_split_tail(&slice, n);
+      grpc_slice_buffer_undo_take_first(src, grpc_slice_split_tail(&slice, n));
       GPR_ASSERT(GRPC_SLICE_LENGTH(slice) == n);
-      GPR_ASSERT(GRPC_SLICE_LENGTH(src->slices[src_idx]) == slice_len - n);
       grpc_slice_buffer_add(dst, slice);
       break;
     }
   }
   GPR_ASSERT(dst->length == output_len);
-  memmove(src->slices, src->slices + src_idx,
-          sizeof(grpc_slice) * (src->count - src_idx));
-  src->count -= src_idx;
-  src->length = new_input_len;
+  GPR_ASSERT(src->length == new_input_len);
   GPR_ASSERT(src->count > 0);
 }
 
+void grpc_slice_buffer_move_first_into_buffer(grpc_exec_ctx *exec_ctx,
+                                              grpc_slice_buffer *src, size_t n,
+                                              void *dst) {
+  char *dstp = dst;
+  GPR_ASSERT(src->length >= n);
+
+  while (n > 0) {
+    grpc_slice slice = grpc_slice_buffer_take_first(src);
+    size_t slice_len = GRPC_SLICE_LENGTH(slice);
+    if (slice_len > n) {
+      memcpy(dstp, GRPC_SLICE_START_PTR(slice), n);
+      grpc_slice_buffer_undo_take_first(
+          src, grpc_slice_sub_no_ref(slice, n, slice_len));
+      n = 0;
+    } else if (slice_len == n) {
+      memcpy(dstp, GRPC_SLICE_START_PTR(slice), n);
+      grpc_slice_unref_internal(exec_ctx, slice);
+      n = 0;
+    } else {
+      memcpy(dstp, GRPC_SLICE_START_PTR(slice), slice_len);
+      dstp += slice_len;
+      n -= slice_len;
+      grpc_slice_unref_internal(exec_ctx, slice);
+    }
+  }
+}
+
 void grpc_slice_buffer_trim_end(grpc_slice_buffer *sb, size_t n,
                                 grpc_slice_buffer *garbage) {
   GPR_ASSERT(n <= sb->length);
@@ -293,8 +337,17 @@ grpc_slice grpc_slice_buffer_take_first(grpc_slice_buffer *sb) {
   grpc_slice slice;
   GPR_ASSERT(sb->count > 0);
   slice = sb->slices[0];
-  memmove(&sb->slices[0], &sb->slices[1], (sb->count - 1) * sizeof(grpc_slice));
+  sb->slices++;
   sb->count--;
   sb->length -= GRPC_SLICE_LENGTH(slice);
+
   return slice;
 }
+
+void grpc_slice_buffer_undo_take_first(grpc_slice_buffer *sb,
+                                       grpc_slice slice) {
+  sb->slices--;
+  sb->slices[0] = slice;
+  sb->count++;
+  sb->length += GRPC_SLICE_LENGTH(slice);
+}

src/node/ext/call_credentials.cc

@@ -35,7 +35,7 @@
 #include <nan.h>
 #include <uv.h>
 
-#include <list>
+#include <queue>
 
 #include "grpc/grpc.h"
 #include "grpc/grpc_security.h"
@@ -170,7 +170,7 @@ NAN_METHOD(CallCredentials::CreateFromPlugin) {
   grpc_metadata_credentials_plugin plugin;
   plugin_state *state = new plugin_state;
   state->callback = new Nan::Callback(info[0].As<Function>());
-  state->pending_callbacks = new std::list<plugin_callback_data*>();
+  state->pending_callbacks = new std::queue<plugin_callback_data*>();
   uv_mutex_init(&state->plugin_mutex);
   uv_async_init(uv_default_loop(),
                 &state->plugin_async,
@@ -231,13 +231,13 @@ NAN_METHOD(PluginCallback) {
 NAUV_WORK_CB(SendPluginCallback) {
   Nan::HandleScope scope;
   plugin_state *state = reinterpret_cast<plugin_state*>(async->data);
-  std::list<plugin_callback_data*> callbacks;
+  std::queue<plugin_callback_data*> callbacks;
   uv_mutex_lock(&state->plugin_mutex);
-  callbacks.splice(callbacks.begin(), *state->pending_callbacks);
+  state->pending_callbacks->swap(callbacks);
   uv_mutex_unlock(&state->plugin_mutex);
   while (!callbacks.empty()) {
     plugin_callback_data *data = callbacks.front();
-    callbacks.pop_front();
+    callbacks.pop();
     Local<Object> callback_data = Nan::New<Object>();
     Nan::Set(callback_data, Nan::New("cb").ToLocalChecked(),
              Nan::New<v8::External>(reinterpret_cast<void*>(data->cb)));
@@ -266,7 +266,7 @@ void plugin_get_metadata(void *state, grpc_auth_metadata_context context,
   data->user_data = user_data;
 
   uv_mutex_lock(&p_state->plugin_mutex);
-  p_state->pending_callbacks->push_back(data);
+  p_state->pending_callbacks->push(data);
   uv_mutex_unlock(&p_state->plugin_mutex);
 
   uv_async_send(&p_state->plugin_async);

src/node/ext/call_credentials.h

@@ -34,7 +34,7 @@
 #ifndef GRPC_NODE_CALL_CREDENTIALS_H_
 #define GRPC_NODE_CALL_CREDENTIALS_H_
 
-#include <list>
+#include <queue>
 
 #include <node.h>
 #include <nan.h>
@@ -84,7 +84,7 @@ typedef struct plugin_callback_data {
 
 typedef struct plugin_state {
   Nan::Callback *callback;
-  std::list<plugin_callback_data*> *pending_callbacks;
+  std::queue<plugin_callback_data*> *pending_callbacks;
   uv_mutex_t plugin_mutex;
   // async.data == this
   uv_async_t plugin_async;

src/node/ext/node_grpc.cc

@@ -31,7 +31,7 @@
  *
  */
 
-#include <list>
+#include <queue>
 
 #include <node.h>
 #include <nan.h>
@@ -77,7 +77,7 @@ typedef struct log_args {
 
 typedef struct logger_state {
   Nan::Callback *callback;
-  std::list<log_args *> *pending_args;
+  std::queue<log_args *> *pending_args;
   uv_mutex_t mutex;
   uv_async_t async;
   // Indicates that a logger has been set
@@ -338,14 +338,14 @@ NAN_METHOD(SetDefaultRootsPem) {
 
 NAUV_WORK_CB(LogMessagesCallback) {
   Nan::HandleScope scope;
-  std::list<log_args *> args;
+  std::queue<log_args *> args;
   uv_mutex_lock(&grpc_logger_state.mutex);
-  args.splice(args.begin(), *grpc_logger_state.pending_args);
+  grpc_logger_state.pending_args->swap(args);
   uv_mutex_unlock(&grpc_logger_state.mutex);
   /* Call the callback with each log message */
   while (!args.empty()) {
     log_args *arg = args.front();
-    args.pop_front();
+    args.pop();
     Local<Value> file = Nan::New(arg->core_args.file).ToLocalChecked();
     Local<Value> line = Nan::New<Uint32, uint32_t>(arg->core_args.line);
     Local<Value> severity = Nan::New(
@@ -372,7 +372,7 @@ void node_log_func(gpr_log_func_args *args) {
   args_copy->timestamp = gpr_now(GPR_CLOCK_REALTIME);
 
   uv_mutex_lock(&grpc_logger_state.mutex);
-  grpc_logger_state.pending_args->push_back(args_copy);
+  grpc_logger_state.pending_args->push(args_copy);
   uv_mutex_unlock(&grpc_logger_state.mutex);
 
   uv_async_send(&grpc_logger_state.async);
@@ -380,7 +380,7 @@ void node_log_func(gpr_log_func_args *args) {
 
 void init_logger() {
   memset(&grpc_logger_state, 0, sizeof(logger_state));
-  grpc_logger_state.pending_args = new std::list<log_args *>();
+  grpc_logger_state.pending_args = new std::queue<log_args *>();
   uv_mutex_init(&grpc_logger_state.mutex);
   uv_async_init(uv_default_loop(),
                 &grpc_logger_state.async,

src/python/grpcio/commands.py

@@ -271,12 +271,12 @@ class BuildExt(build_ext.build_ext):
         compiler = self.compiler.compiler_type
         if compiler in BuildExt.C_OPTIONS:
             for extension in self.extensions:
-                extension.extra_compile_args += list(BuildExt.C_OPTIONS[
-                    compiler])
+                extension.extra_compile_args += list(
+                    BuildExt.C_OPTIONS[compiler])
         if compiler in BuildExt.LINK_OPTIONS:
             for extension in self.extensions:
-                extension.extra_link_args += list(BuildExt.LINK_OPTIONS[
-                    compiler])
+                extension.extra_link_args += list(
+                    BuildExt.LINK_OPTIONS[compiler])
         if not check_and_update_cythonization(self.extensions):
             self.extensions = try_cythonize(self.extensions)
         try:
@@ -284,8 +284,8 @@ class BuildExt(build_ext.build_ext):
         except Exception as error:
             formatted_exception = traceback.format_exc()
             support.diagnose_build_ext_error(self, error, formatted_exception)
-            raise CommandError("Failed `build_ext` step:\n{}".format(
-                formatted_exception))
+            raise CommandError(
+                "Failed `build_ext` step:\n{}".format(formatted_exception))
 
 
 class Gather(setuptools.Command):

src/python/grpcio/grpc/_channel.py

@@ -842,8 +842,8 @@ def _poll_connectivity(state, channel, initial_try_to_connect):
     connectivity = channel.check_connectivity_state(try_to_connect)
     with state.lock:
         state.connectivity = (
-            _common.CYGRPC_CONNECTIVITY_STATE_TO_CHANNEL_CONNECTIVITY[
-                connectivity])
+            _common.
+            CYGRPC_CONNECTIVITY_STATE_TO_CHANNEL_CONNECTIVITY[connectivity])
         callbacks = tuple(callback
                           for callback, unused_but_known_to_be_none_connectivity
                           in state.callbacks_and_connectivities)

src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi

@@ -29,6 +29,8 @@
 
 cimport cpython
 
+import traceback
+
 
 cdef class ChannelCredentials:
 
@@ -138,15 +140,22 @@ cdef class AuthMetadataContext:
 cdef void plugin_get_metadata(
     void *state, grpc_auth_metadata_context context,
     grpc_credentials_plugin_metadata_cb cb, void *user_data) with gil:
+  called_flag = [False]
   def python_callback(
       Metadata metadata, grpc_status_code status,
       bytes error_details):
     cb(user_data, metadata.c_metadata_array.metadata,
        metadata.c_metadata_array.count, status, error_details)
+    called_flag[0] = True
   cdef CredentialsMetadataPlugin self = <CredentialsMetadataPlugin>state
   cdef AuthMetadataContext cy_context = AuthMetadataContext()
   cy_context.context = context
-  self.plugin_callback(cy_context, python_callback)
+  try:
+    self.plugin_callback(cy_context, python_callback)
+  except Exception as error:
+    if not called_flag[0]:
+      cb(user_data, Metadata([]).c_metadata_array.metadata,
+         0, StatusCode.unknown, traceback.format_exc().encode())
 
 cdef void plugin_destroy_c_plugin_state(void *state) with gil:
   cpython.Py_DECREF(<CredentialsMetadataPlugin>state)

src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi

@@ -194,6 +194,25 @@ cdef class Timespec:
   def infinite_past():
     return Timespec(float("-inf"))
 
+  def __richcmp__(Timespec self not None, Timespec other not None, int op):
+    cdef gpr_timespec self_c_time = self.c_time
+    cdef gpr_timespec other_c_time = other.c_time
+    cdef int result = gpr_time_cmp(self_c_time, other_c_time)
+    if op == 0:  # <
+      return result < 0
+    elif op == 2:  # ==
+      return result == 0
+    elif op == 4:  # >
+      return result > 0
+    elif op == 1:  # <=
+      return result <= 0
+    elif op == 3:  # !=
+      return result != 0
+    elif op == 5:  # >=
+      return result >= 0
+    else:
+      raise ValueError('__richcmp__ `op` contract violated')
+
 
 cdef class CallDetails:
 

src/python/grpcio_tests/commands.py

@@ -121,8 +121,8 @@ class BuildProtoModules(setuptools.Command):
                 '--grpc_python_out={}'.format(PROTO_STEM),
             ] + [path]
             if protoc.main(command) != 0:
-                sys.stderr.write('warning: Command:\n{}\nFailed'.format(
-                    command))
+                sys.stderr.write(
+                    'warning: Command:\n{}\nFailed'.format(command))
 
         # Generated proto directories dont include __init__.py, but
         # these are needed for python package resolution

src/python/grpcio_tests/tests/_loader.py

@@ -116,5 +116,5 @@ def iterate_suite_cases(suite):
         elif isinstance(item, unittest.TestCase):
             yield item
         else:
-            raise ValueError('unexpected suite item of type {}'.format(
-                type(item)))
+            raise ValueError(
+                'unexpected suite item of type {}'.format(type(item)))

src/python/grpcio_tests/tests/_runner.py

@@ -196,8 +196,8 @@ class Runner(object):
         # Run the tests
         result.startTestRun()
         for augmented_case in augmented_cases:
-            sys.stdout.write('Running       {}\n'.format(augmented_case.case.id(
-            )))
+            sys.stdout.write(
+                'Running       {}\n'.format(augmented_case.case.id()))
             sys.stdout.flush()
             case_thread = threading.Thread(
                 target=augmented_case.case.run, args=(result,))

src/python/grpcio_tests/tests/interop/methods.py

@@ -428,8 +428,8 @@ def _compute_engine_creds(stub, args):
 
 
 def _oauth2_auth_token(stub, args):
-    json_key_filename = os.environ[
-        oauth2client_client.GOOGLE_APPLICATION_CREDENTIALS]
+    json_key_filename = os.environ[oauth2client_client.
+                                   GOOGLE_APPLICATION_CREDENTIALS]
     wanted_email = json.load(open(json_key_filename, 'rb'))['client_email']
     response = _large_unary_common_behavior(stub, True, True, None)
     if wanted_email != response.username:
@@ -441,8 +441,8 @@ def _oauth2_auth_token(stub, args):
 
 
 def _jwt_token_creds(stub, args):
-    json_key_filename = os.environ[
-        oauth2client_client.GOOGLE_APPLICATION_CREDENTIALS]
+    json_key_filename = os.environ[oauth2client_client.
+                                   GOOGLE_APPLICATION_CREDENTIALS]
     wanted_email = json.load(open(json_key_filename, 'rb'))['client_email']
     response = _large_unary_common_behavior(stub, True, False, None)
     if wanted_email != response.username:
@@ -451,11 +451,10 @@ def _jwt_token_creds(stub, args):
 
 
 def _per_rpc_creds(stub, args):
-    json_key_filename = os.environ[
-        oauth2client_client.GOOGLE_APPLICATION_CREDENTIALS]
+    json_key_filename = os.environ[oauth2client_client.
+                                   GOOGLE_APPLICATION_CREDENTIALS]
     wanted_email = json.load(open(json_key_filename, 'rb'))['client_email']
-    credentials = oauth2client_client.GoogleCredentials.get_application_default(
-    )
+    credentials = oauth2client_client.GoogleCredentials.get_application_default()
     scoped_credentials = credentials.create_scoped([args.oauth_scope])
     # TODO(https://github.com/grpc/grpc/issues/6799): Eliminate this last
     # remaining use of the Beta API.

src/python/grpcio_tests/tests/qps/worker_server.py

@@ -102,8 +102,8 @@ class WorkerServer(services_pb2.WorkerServiceServicer):
                 'grpc.testing.BenchmarkService', method_implementations)
             server.add_generic_rpc_handlers((handler,))
         else:
-            raise Exception('Unsupported server type {}'.format(
-                config.server_type))
+            raise Exception(
+                'Unsupported server type {}'.format(config.server_type))
 
         if config.HasField('security_params'):  # Use SSL
             server_creds = grpc.ssl_server_credentials((
@@ -171,8 +171,8 @@ class WorkerServer(services_pb2.WorkerServiceServicer):
             else:
                 raise Exception('Async streaming client not supported')
         else:
-            raise Exception('Unsupported client type {}'.format(
-                config.client_type))
+            raise Exception(
+                'Unsupported client type {}'.format(config.client_type))
 
         # In multi-channel tests, we split the load across all channels
         load_factor = float(config.client_channels)

src/python/grpcio_tests/tests/unit/_cython/cygrpc_test.py

@@ -95,8 +95,18 @@ class TypeSmokeTest(unittest.TestCase):
 
     def testTimespec(self):
         now = time.time()
-        timespec = cygrpc.Timespec(now)
-        self.assertAlmostEqual(now, float(timespec), places=8)
+        now_timespec_a = cygrpc.Timespec(now)
+        now_timespec_b = cygrpc.Timespec(now)
+        self.assertAlmostEqual(now, float(now_timespec_a), places=8)
+        self.assertEqual(now_timespec_a, now_timespec_b)
+        self.assertLess(cygrpc.Timespec(now - 1), cygrpc.Timespec(now))
+        self.assertGreater(cygrpc.Timespec(now + 1), cygrpc.Timespec(now))
+        self.assertGreaterEqual(cygrpc.Timespec(now + 1), cygrpc.Timespec(now))
+        self.assertGreaterEqual(cygrpc.Timespec(now), cygrpc.Timespec(now))
+        self.assertLessEqual(cygrpc.Timespec(now - 1), cygrpc.Timespec(now))
+        self.assertLessEqual(cygrpc.Timespec(now), cygrpc.Timespec(now))
+        self.assertNotEqual(cygrpc.Timespec(now - 1), cygrpc.Timespec(now))
+        self.assertNotEqual(cygrpc.Timespec(now + 1), cygrpc.Timespec(now))
 
     def testCompletionQueueUpDown(self):
         completion_queue = cygrpc.CompletionQueue()
@@ -204,8 +214,8 @@ class ServerClientMixin(object):
                 self.assertTrue(event.success)
                 self.assertIs(tag, event.tag)
             except Exception as error:
-                raise Exception("Error in '{}': {}".format(description,
-                                                           error.message))
+                raise Exception(
+                    "Error in '{}': {}".format(description, error.message))
             return event
 
         return test_utilities.SimpleFuture(performer)

src/python/grpcio_tests/tests/unit/_empty_message_test.py

@@ -122,13 +122,13 @@ class EmptyMessageTest(unittest.TestCase):
                                  list(response_iterator))
 
     def testStreamUnary(self):
-        response = self._channel.stream_unary(_STREAM_UNARY)(iter(
-            [_REQUEST] * test_constants.STREAM_LENGTH))
+        response = self._channel.stream_unary(_STREAM_UNARY)(
+            iter([_REQUEST] * test_constants.STREAM_LENGTH))
         self.assertEqual(_RESPONSE, response)
 
     def testStreamStream(self):
-        response_iterator = self._channel.stream_stream(_STREAM_STREAM)(iter(
-            [_REQUEST] * test_constants.STREAM_LENGTH))
+        response_iterator = self._channel.stream_stream(_STREAM_STREAM)(
+            iter([_REQUEST] * test_constants.STREAM_LENGTH))
         self.assertSequenceEqual([_RESPONSE] * test_constants.STREAM_LENGTH,
                                  list(response_iterator))
 

src/ruby/ext/grpc/rb_grpc_imports.generated.c

@@ -215,7 +215,9 @@ grpc_slice_buffer_swap_type grpc_slice_buffer_swap_import;
 grpc_slice_buffer_move_into_type grpc_slice_buffer_move_into_import;
 grpc_slice_buffer_trim_end_type grpc_slice_buffer_trim_end_import;
 grpc_slice_buffer_move_first_type grpc_slice_buffer_move_first_import;
+grpc_slice_buffer_move_first_into_buffer_type grpc_slice_buffer_move_first_into_buffer_import;
 grpc_slice_buffer_take_first_type grpc_slice_buffer_take_first_import;
+grpc_slice_buffer_undo_take_first_type grpc_slice_buffer_undo_take_first_import;
 gpr_malloc_type gpr_malloc_import;
 gpr_free_type gpr_free_import;
 gpr_realloc_type gpr_realloc_import;
@@ -504,7 +506,9 @@ void grpc_rb_load_imports(HMODULE library) {
   grpc_slice_buffer_move_into_import = (grpc_slice_buffer_move_into_type) GetProcAddress(library, "grpc_slice_buffer_move_into");
   grpc_slice_buffer_trim_end_import = (grpc_slice_buffer_trim_end_type) GetProcAddress(library, "grpc_slice_buffer_trim_end");
   grpc_slice_buffer_move_first_import = (grpc_slice_buffer_move_first_type) GetProcAddress(library, "grpc_slice_buffer_move_first");
+  grpc_slice_buffer_move_first_into_buffer_import = (grpc_slice_buffer_move_first_into_buffer_type) GetProcAddress(library, "grpc_slice_buffer_move_first_into_buffer");
   grpc_slice_buffer_take_first_import = (grpc_slice_buffer_take_first_type) GetProcAddress(library, "grpc_slice_buffer_take_first");
+  grpc_slice_buffer_undo_take_first_import = (grpc_slice_buffer_undo_take_first_type) GetProcAddress(library, "grpc_slice_buffer_undo_take_first");
   gpr_malloc_import = (gpr_malloc_type) GetProcAddress(library, "gpr_malloc");
   gpr_free_import = (gpr_free_type) GetProcAddress(library, "gpr_free");
   gpr_realloc_import = (gpr_realloc_type) GetProcAddress(library, "gpr_realloc");

src/ruby/ext/grpc/rb_grpc_imports.generated.h

@@ -596,9 +596,15 @@ extern grpc_slice_buffer_trim_end_type grpc_slice_buffer_trim_end_import;
 typedef void(*grpc_slice_buffer_move_first_type)(grpc_slice_buffer *src, size_t n, grpc_slice_buffer *dst);
 extern grpc_slice_buffer_move_first_type grpc_slice_buffer_move_first_import;
 #define grpc_slice_buffer_move_first grpc_slice_buffer_move_first_import
+typedef void(*grpc_slice_buffer_move_first_into_buffer_type)(grpc_exec_ctx *exec_ctx, grpc_slice_buffer *src, size_t n, void *dst);
+extern grpc_slice_buffer_move_first_into_buffer_type grpc_slice_buffer_move_first_into_buffer_import;
+#define grpc_slice_buffer_move_first_into_buffer grpc_slice_buffer_move_first_into_buffer_import
 typedef grpc_slice(*grpc_slice_buffer_take_first_type)(grpc_slice_buffer *src);
 extern grpc_slice_buffer_take_first_type grpc_slice_buffer_take_first_import;
 #define grpc_slice_buffer_take_first grpc_slice_buffer_take_first_import
+typedef void(*grpc_slice_buffer_undo_take_first_type)(grpc_slice_buffer *src, grpc_slice slice);
+extern grpc_slice_buffer_undo_take_first_type grpc_slice_buffer_undo_take_first_import;
+#define grpc_slice_buffer_undo_take_first grpc_slice_buffer_undo_take_first_import
 typedef void *(*gpr_malloc_type)(size_t size);
 extern gpr_malloc_type gpr_malloc_import;
 #define gpr_malloc gpr_malloc_import

src/ruby/tools/bin/grpc_tools_ruby_protoc

@@ -30,7 +30,7 @@
 
 require 'rbconfig'
 
-require_relative '../os_check'
+require_relative '../platform_check'
 
 ext = RbConfig::CONFIG['EXEEXT']
 
@@ -39,7 +39,7 @@ protoc_name = 'protoc' + ext
 plugin_name = 'grpc_ruby_plugin' + ext
 
 protoc_dir = File.join(File.dirname(__FILE__),
-                       RbConfig::CONFIG['host_cpu'] + '-' + OS.os_name)
+                       PLATFORM.architecture + '-' + PLATFORM.os_name)
 
 protoc_path = File.join(protoc_dir, protoc_name)
 

src/ruby/tools/bin/grpc_tools_ruby_protoc_plugin

@@ -30,12 +30,12 @@
 
 require 'rbconfig'
 
-require_relative '../os_check'
+require_relative '../platform_check'
 
 plugin_name = 'grpc_ruby_plugin' + RbConfig::CONFIG['EXEEXT']
 
 plugin_path = File.join(File.dirname(__FILE__),
-                        RbConfig::CONFIG['host_cpu'] + '-' + OS.os_name,
+                        PLATFORM.architecture + '-' + PLATFORM.os_name,
                         plugin_name)
 
 exec([ plugin_path, plugin_path ], *ARGV)

src/ruby/tools/grpc-tools.gemspec

@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
   s.description = 'protoc and the Ruby gRPC protoc plugin'
   s.license = 'BSD-3-Clause'
 
-  s.files = %w( version.rb os_check.rb README.md )
+  s.files = %w( version.rb platform_check.rb README.md )
   s.files += Dir.glob('bin/**/*')
 
   s.bindir = 'bin'

src/ruby/tools/os_check.rb → src/ruby/tools/platform_check.rb

@@ -27,19 +27,28 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-# This is based on http://stackoverflow.com/a/171011/159388 by Aaron Hinni
-
 require 'rbconfig'
 
-module OS
-  def OS.os_name
+# This is based on http://stackoverflow.com/a/171011/159388 by Aaron Hinni
+
+module PLATFORM
+  def PLATFORM.os_name
     case RbConfig::CONFIG['host_os']
-    when /cygwin|mswin|mingw|bccwin|wince|emx/
-      'windows'
-    when /darwin/
-      'macos'
-    else
-      'linux'
+      when /cygwin|mswin|mingw|bccwin|wince|emx/
+        'windows'
+      when /darwin/
+        'macos'
+      else
+        'linux'
+    end
+  end
+
+  def PLATFORM.architecture
+    case RbConfig::CONFIG['host_cpu']
+      when /x86_64/
+        'x86_64'
+      else
+        'x86'
     end
   end
 end

templates/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template

@@ -33,6 +33,8 @@
   
   <%include file="../../go_path.include"/>
   <%include file="../../python_deps.include"/>
+  RUN pip install twisted h2
+
   # Define the default command.
   CMD ["bash"]
   

test/core/iomgr/ev_epoll_linux_test.c

@@ -45,6 +45,7 @@
 #include <grpc/support/log.h>
 
 #include "src/core/lib/iomgr/iomgr.h"
+#include "src/core/lib/iomgr/workqueue.h"
 #include "test/core/util/test_config.h"
 
 typedef struct test_pollset {
@@ -60,6 +61,22 @@ typedef struct test_fd {
 /* num_fds should be an even number */
 static void test_fd_init(test_fd *tfds, int *fds, int num_fds) {
   int i;
+  int r;
+
+  /* Create some dummy file descriptors. Currently using pipe file descriptors
+   * for this test but we could use any other type of file descriptors. Also,
+   * since pipe() used in this test creates two fds in each call, num_fds should
+   * be an even number */
+  GPR_ASSERT((num_fds % 2) == 0);
+  for (i = 0; i < num_fds; i = i + 2) {
+    r = pipe(fds + i);
+    if (r != 0) {
+      gpr_log(GPR_ERROR, "Error in creating pipe. %d (%s)", errno,
+              strerror(errno));
+      return;
+    }
+  }
+
   for (i = 0; i < num_fds; i++) {
     tfds[i].inner_fd = fds[i];
     tfds[i].fd = grpc_fd_create(fds[i], "test_fd");
@@ -111,8 +128,80 @@ static void test_pollset_cleanup(grpc_exec_ctx *exec_ctx,
   }
 }
 
-#define NUM_FDS 8
-#define NUM_POLLSETS 4
+static void increment(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
+  ++*(int *)arg;
+}
+
+/*
+ * Validate that merging two workqueues preserves the closures in each queue.
+ * This is a regression test for a bug in
+ * polling_island_merge()[ev_epoll_linux.c], where the parent relationship was
+ * inverted.
+ */
+static void test_pollset_queue_merge_items() {
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  const int num_fds = 2;
+  const int num_pollsets = 2;
+  const int num_closures = 4;
+  test_fd tfds[num_fds];
+  int fds[num_fds];
+  test_pollset pollsets[num_pollsets];
+  grpc_closure closures[num_closures];
+  int i;
+  int result = 0;
+
+  test_fd_init(tfds, fds, num_fds);
+  test_pollset_init(pollsets, num_pollsets);
+
+  /* Two distinct polling islands, each with their own FD and pollset. */
+  for (i = 0; i < num_fds; i++) {
+    grpc_pollset_add_fd(&exec_ctx, pollsets[i].pollset, tfds[i].fd);
+    grpc_exec_ctx_flush(&exec_ctx);
+  }
+
+  /* Enqeue the closures, 3 to polling island 0 and 1 to polling island 1. */
+  grpc_closure_init(
+      closures, increment, &result,
+      grpc_workqueue_scheduler(grpc_fd_get_polling_island(tfds[0].fd)));
+  grpc_closure_init(
+      closures + 1, increment, &result,
+      grpc_workqueue_scheduler(grpc_fd_get_polling_island(tfds[0].fd)));
+  grpc_closure_init(
+      closures + 2, increment, &result,
+      grpc_workqueue_scheduler(grpc_fd_get_polling_island(tfds[0].fd)));
+  grpc_closure_init(
+      closures + 3, increment, &result,
+      grpc_workqueue_scheduler(grpc_fd_get_polling_island(tfds[1].fd)));
+  for (i = 0; i < num_closures; ++i) {
+    grpc_closure_sched(&exec_ctx, closures + i, GRPC_ERROR_NONE);
+  }
+
+  /* Merge the two polling islands. */
+  grpc_pollset_add_fd(&exec_ctx, pollsets[0].pollset, tfds[1].fd);
+  grpc_exec_ctx_flush(&exec_ctx);
+
+  /*
+   * Execute the closures, verify we see each one execute when executing work on
+   * the merged polling island.
+   */
+  grpc_pollset_worker *worker = NULL;
+  for (i = 0; i < num_closures; ++i) {
+    const gpr_timespec deadline = gpr_time_add(
+        gpr_now(GPR_CLOCK_MONOTONIC), gpr_time_from_seconds(2, GPR_TIMESPAN));
+    gpr_mu_lock(pollsets[1].mu);
+    GRPC_LOG_IF_ERROR(
+        "grpc_pollset_work",
+        grpc_pollset_work(&exec_ctx, pollsets[1].pollset, &worker,
+                          gpr_now(GPR_CLOCK_MONOTONIC), deadline));
+    gpr_mu_unlock(pollsets[1].mu);
+  }
+  GPR_ASSERT(result == num_closures);
+
+  test_fd_cleanup(&exec_ctx, tfds, num_fds);
+  test_pollset_cleanup(&exec_ctx, pollsets, num_pollsets);
+  grpc_exec_ctx_finish(&exec_ctx);
+}
+
 /*
  * Cases to test:
  *  case 1) Polling islands of both fd and pollset are NULL
@@ -125,28 +214,16 @@ static void test_pollset_cleanup(grpc_exec_ctx *exec_ctx,
  * */
 static void test_add_fd_to_pollset() {
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  test_fd tfds[NUM_FDS];
-  int fds[NUM_FDS];
-  test_pollset pollsets[NUM_POLLSETS];
+  const int num_fds = 8;
+  const int num_pollsets = 4;
+  test_fd tfds[num_fds];
+  int fds[num_fds];
+  test_pollset pollsets[num_pollsets];
   void *expected_pi = NULL;
   int i;
-  int r;
 
-  /* Create some dummy file descriptors. Currently using pipe file descriptors
-   * for this test but we could use any other type of file descriptors. Also,
-   * since pipe() used in this test creates two fds in each call, NUM_FDS should
-   * be an even number */
-  for (i = 0; i < NUM_FDS; i = i + 2) {
-    r = pipe(fds + i);
-    if (r != 0) {
-      gpr_log(GPR_ERROR, "Error in creating pipe. %d (%s)", errno,
-              strerror(errno));
-      return;
-    }
-  }
-
-  test_fd_init(tfds, fds, NUM_FDS);
-  test_pollset_init(pollsets, NUM_POLLSETS);
+  test_fd_init(tfds, fds, num_fds);
+  test_pollset_init(pollsets, num_pollsets);
 
   /*Step 1.
    * Create three polling islands (This will exercise test case 1 and 2) with
@@ -207,19 +284,19 @@ static void test_add_fd_to_pollset() {
 
   /* Compare Fd:0's polling island with that of all other Fds */
   expected_pi = grpc_fd_get_polling_island(tfds[0].fd);
-  for (i = 1; i < NUM_FDS; i++) {
+  for (i = 1; i < num_fds; i++) {
     GPR_ASSERT(grpc_are_polling_islands_equal(
         expected_pi, grpc_fd_get_polling_island(tfds[i].fd)));
   }
 
   /* Compare Fd:0's polling island with that of all other pollsets */
-  for (i = 0; i < NUM_POLLSETS; i++) {
+  for (i = 0; i < num_pollsets; i++) {
     GPR_ASSERT(grpc_are_polling_islands_equal(
         expected_pi, grpc_pollset_get_polling_island(pollsets[i].pollset)));
   }
 
-  test_fd_cleanup(&exec_ctx, tfds, NUM_FDS);
-  test_pollset_cleanup(&exec_ctx, pollsets, NUM_POLLSETS);
+  test_fd_cleanup(&exec_ctx, tfds, num_fds);
+  test_pollset_cleanup(&exec_ctx, pollsets, num_pollsets);
   grpc_exec_ctx_finish(&exec_ctx);
 }
 
@@ -231,6 +308,7 @@ int main(int argc, char **argv) {
   poll_strategy = grpc_get_poll_strategy_name();
   if (poll_strategy != NULL && strcmp(poll_strategy, "epoll") == 0) {
     test_add_fd_to_pollset();
+    test_pollset_queue_merge_items();
   } else {
     gpr_log(GPR_INFO,
             "Skipping the test. The test is only relevant for 'epoll' "

tools/distrib/python/docgen.py

@@ -28,11 +28,14 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+from __future__ import print_function
+
 import argparse
 import os
 import os.path
 import shutil
 import subprocess
+import sys
 import tempfile
 
 parser = argparse.ArgumentParser()
@@ -99,6 +102,7 @@ if args.submit:
   python_doc_dir = os.path.join(repo_dir, 'python')
   doc_branch = args.doc_branch
 
+  print('Cloning your repository...')
   subprocess.check_call([
           'git', 'clone', 'https://{}@github.com/{}/grpc'.format(
               github_user, github_repository_owner)
@@ -110,13 +114,20 @@ if args.submit:
   subprocess.check_call([
           'git', 'checkout', 'upstream/gh-pages', '-b', doc_branch
       ], cwd=repo_dir)
+  print('Updating documentation...')
   shutil.rmtree(python_doc_dir, ignore_errors=True)
   shutil.copytree(DOC_PATH, python_doc_dir)
-  subprocess.check_call(['git', 'add', '--all'], cwd=repo_dir)
-  subprocess.check_call([
-          'git', 'commit', '-m', 'Auto-update Python documentation'
-      ], cwd=repo_dir)
-  subprocess.check_call([
-          'git', 'push', '--set-upstream', 'origin', doc_branch
-      ], cwd=repo_dir)
+  print('Attempting to push documentation...')
+  try:
+    subprocess.check_call(['git', 'add', '--all'], cwd=repo_dir)
+    subprocess.check_call([
+            'git', 'commit', '-m', 'Auto-update Python documentation'
+        ], cwd=repo_dir)
+    subprocess.check_call([
+            'git', 'push', '--set-upstream', 'origin', doc_branch
+        ], cwd=repo_dir)
+  except subprocess.CalledProcessError:
+    print('Failed to push documentation. Examine this directory and push '
+          'manually: {}'.format(repo_parent_dir))
+    sys.exit(1)
   shutil.rmtree(repo_parent_dir)

tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile

@@ -47,5 +47,7 @@ RUN pip install pip --upgrade
 RUN pip install virtualenv
 RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0
 
+RUN pip install twisted h2
+
 # Define the default command.
 CMD ["bash"]

tools/run_tests/helper_scripts/build_python.sh

@@ -163,22 +163,18 @@ pip_install_dir() {
   PWD=`pwd`
   cd $1
   ($VENV_PYTHON setup.py build_ext -c $TOOLCHAIN || true)
-  # install the dependencies
-  $VENV_PYTHON -m pip install --upgrade .
-  # ensure that we've reinstalled the test packages
-  $VENV_PYTHON -m pip install --upgrade --force-reinstall --no-deps .
+  $VENV_PYTHON -m pip install --no-deps .
   cd $PWD
 }
 
 $VENV_PYTHON -m pip install --upgrade pip
 $VENV_PYTHON -m pip install setuptools
 $VENV_PYTHON -m pip install cython
+$VENV_PYTHON -m pip install six enum34 protobuf futures
 pip_install_dir $ROOT
+
 $VENV_PYTHON $ROOT/tools/distrib/python/make_grpcio_tools.py
 pip_install_dir $ROOT/tools/distrib/python/grpcio_tools
-# TODO(atash) figure out namespace packages and grpcio-tools and auditwheel
-# etc...
-pip_install_dir $ROOT
 
 # Build/install health checking
 $VENV_PYTHON $ROOT/src/python/grpcio_health_checking/setup.py preprocess
@@ -191,6 +187,7 @@ $VENV_PYTHON $ROOT/src/python/grpcio_reflection/setup.py build_package_protos
 pip_install_dir $ROOT/src/python/grpcio_reflection
 
 # Build/install tests
+$VENV_PYTHON -m pip install coverage oauth2client
 $VENV_PYTHON $ROOT/src/python/grpcio_tests/setup.py preprocess
 $VENV_PYTHON $ROOT/src/python/grpcio_tests/setup.py build_package_protos
 pip_install_dir $ROOT/src/python/grpcio_tests