Merge branch 'master' into grpc_tools_docker_pregenerate_ssh_key

.gitignore

@@ -0,0 +1,6 @@
+bins
+deps
+libs
+objs
+*.pyc
+

build.json

@@ -1341,6 +1341,7 @@
         "test/cpp/end2end/sync_client_async_server_test.cc"
       ],
       "deps": [
+        "grpc++_test_util",
         "grpc_test_util",
         "grpc++",
         "grpc",
@@ -1353,9 +1354,7 @@
       "build": "test",
       "c++": true,
       "src": [
-        "test/cpp/interop/empty.proto",
-        "test/cpp/interop/messages.proto",
-        "test/cpp/interop/test.proto",
+        "test/cpp/qps/qpstest.proto",
         "test/cpp/qps/client.cc"
       ],
       "deps": [
@@ -1372,9 +1371,7 @@
       "build": "test",
       "c++": true,
       "src": [
-        "test/cpp/interop/empty.proto",
-        "test/cpp/interop/messages.proto",
-        "test/cpp/interop/test.proto",
+        "test/cpp/qps/qpstest.proto",
         "test/cpp/qps/server.cc"
       ],
       "deps": [
@@ -1451,7 +1448,8 @@
       ],
       "deps": [
         "grpc++",
-        "grpc"
+        "grpc",
+        "gpr"
       ]
     },
     {
@@ -1463,7 +1461,8 @@
       ],
       "deps": [
         "grpc++",
-        "grpc"
+        "grpc",
+        "gpr"
       ]
     },
     {

src/compiler/go_generator.cc

@@ -31,8 +31,6 @@
  *
  */
 
-using namespace std;
-
 #include "src/compiler/go_generator.h"
 
 #include <cctype>
@@ -42,6 +40,8 @@ using namespace std;
 #include <google/protobuf/descriptor.pb.h>
 #include <google/protobuf/descriptor.h>
 
+using namespace std;
+
 namespace grpc_go_generator {
 
 bool NoStreaming(const google::protobuf::MethodDescriptor* method) {

src/compiler/go_generator.h

@@ -44,7 +44,7 @@ class FileDescriptor;
 
 namespace grpc_go_generator {
 
-string GetServices(const google::protobuf::FileDescriptor* file);
+std::string GetServices(const google::protobuf::FileDescriptor* file);
 
 }  // namespace grpc_go_generator
 

src/core/iomgr/pollset.h

@@ -35,6 +35,7 @@
 #define __GRPC_INTERNAL_IOMGR_POLLSET_H_
 
 #include <grpc/support/port_platform.h>
+#include <grpc/support/time.h>
 
 /* A grpc_pollset is a set of file descriptors that a higher level item is
    interested in. For example:

templates/Makefile.template

@@ -2,6 +2,7 @@
 # This currently builds C and C++ code.
 <%!
   import re
+  import os
 
   proto_re = re.compile('(.*)\\.proto')
 
@@ -355,7 +356,7 @@ buildtests_c: bins_dep_c privatelibs_c\
 buildtests_cxx: bins_dep_cxx privatelibs_cxx\
 % for tgt in targets:
 % if tgt.build == 'test' and tgt.get('c++', False):
- bins/${tgt.name}\
+ bins/$(CONFIG)/${tgt.name}\
 % endif
 % endfor
 
@@ -687,6 +688,7 @@ libs/$(CONFIG)/lib${lib.name}.a: $(LIB${lib.name.upper()}_OBJS)
 	$(Q) $(AR) rcs libs/$(CONFIG)/lib${lib.name}.a $(LIB${lib.name.upper()}_OBJS)
 % if lib.get('baselib', False):
 % if lib.get('secure', True):
+	$(Q) rm -rf tmp-merge
 	$(Q) mkdir tmp-merge
 	$(Q) ( cd tmp-merge ; $(AR) x ../libs/$(CONFIG)/lib${lib.name}.a )
 	$(Q) for l in $(OPENSSL_MERGE_LIBS) ; do ( cd tmp-merge ; <%text>ar x ../$${l}</%text> ) ; done
@@ -760,6 +762,18 @@ endif
 endif
 % endif
 
+% for src in lib.src:
+% if not proto_re.match(src):
+objs/$(CONFIG)/${os.path.splitext(src)[0]}.o: \
+% for src2 in lib.src:
+% if proto_re.match(src2):
+    ${proto_to_cc(src2)}\
+% endif
+% endfor
+% endif
+
+% endfor
+
 clean_lib${lib.name}:
 	$(E) "[CLEAN]   Cleaning lib${lib.name} files"
 	$(Q) $(RM) $(LIB${lib.name.upper()}_OBJS)
@@ -836,6 +850,14 @@ bins/$(CONFIG)/${tgt.name}: $(${tgt.name.upper()}_OBJS)\
 endif
 % endif
 
+% for src in tgt.src:
+objs/$(CONFIG)/${os.path.splitext(src)[0]}.o: \
+% for dep in tgt.deps:
+ libs/$(CONFIG)/lib${dep}.a\
+% endfor
+
+% endfor
+
 deps_${tgt.name}: $(${tgt.name.upper()}_DEPS)
 
 % if tgt.get('secure', True):

test/core/util/port.h

@@ -34,6 +34,10 @@
 #ifndef __GRPC_TEST_UTIL_PORT_H__
 #define __GRPC_TEST_UTIL_PORT_H__
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 /* pick a port number that is currently unused by either tcp or udp. return
    0 on failure. */
 int grpc_pick_unused_port();
@@ -41,4 +45,8 @@ int grpc_pick_unused_port();
    on failure. */
 int grpc_pick_unused_port_or_die();
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif  /* __GRPC_TEST_UTIL_PORT_H__ */

test/cpp/client/channel_arguments_test.cc

@@ -69,15 +69,12 @@ TEST_F(ChannelArgumentsTest, SetInt) {
   key = "";
   SetChannelArgs(channel_args, &args);
   EXPECT_EQ(2, args.num_args);
-  bool found[2] = {false, false};
   // We do not enforce order on the arguments.
-  for (int i = 0; i < args.num_args; i++) {
+  for (size_t i = 0; i < args.num_args; i++) {
     EXPECT_EQ(GRPC_ARG_INTEGER, args.args[i].type);
     if (grpc::string(args.args[i].key) == "key0") {
-      found[0] = true;
       EXPECT_EQ(0, args.args[i].value.integer);
     } else if (grpc::string(args.args[i].key) == "key1") {
-      found[1] = true;
       EXPECT_EQ(1, args.args[i].value.integer);
     }
   }
@@ -107,15 +104,12 @@ TEST_F(ChannelArgumentsTest, SetString) {
   channel_args.SetString(key, val);
   SetChannelArgs(channel_args, &args);
   EXPECT_EQ(2, args.num_args);
-  bool found[2] = {false, false};
   // We do not enforce order on the arguments.
-  for (int i = 0; i < args.num_args; i++) {
+  for (size_t i = 0; i < args.num_args; i++) {
     EXPECT_EQ(GRPC_ARG_STRING, args.args[i].type);
     if (grpc::string(args.args[i].key) == "key0") {
-      found[0] = true;
       EXPECT_STREQ("val0", args.args[i].value.string);
     } else if (grpc::string(args.args[i].key) == "key1") {
-      found[1] = true;
       EXPECT_STREQ("val1", args.args[i].value.string);
     }
   }
@@ -125,6 +119,6 @@ TEST_F(ChannelArgumentsTest, SetString) {
 }  // namespace grpc
 
 int main(int argc, char** argv) {
-
+  ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

test/cpp/client/credentials_test.cc

@@ -65,7 +65,7 @@ TEST_F(CredentialsTest, InvalidServiceAccountCreds) {
 }  // namespace grpc
 
 int main(int argc, char **argv) {
-
+  ::testing::InitGoogleTest(&argc, argv);
   grpc_init();
   int ret = RUN_ALL_TESTS();
   grpc_shutdown();

test/cpp/end2end/end2end_test.cc

@@ -34,7 +34,7 @@
 #include <chrono>
 #include <thread>
 
-#include "net/grpc/cpp/echo_duplicate_proto_cc.pb.h"
+#include "test/cpp/util/echo_duplicate.pb.h"
 #include "test/cpp/util/echo.pb.h"
 #include "src/cpp/util/time.h"
 #include <grpc++/channel_arguments.h>
@@ -47,7 +47,7 @@
 #include <grpc++/server_context.h>
 #include <grpc++/status.h>
 #include <grpc++/stream.h>
-#include "net/util/netutil.h"
+#include "test/core/util/port.h"
 #include <gtest/gtest.h>
 
 #include <grpc/grpc.h>
@@ -141,7 +141,7 @@ class TestServiceImplDupPkg
 class End2endTest : public ::testing::Test {
  protected:
   void SetUp() override {
-    int port = PickUnusedPortOrDie();
+    int port = grpc_pick_unused_port_or_die();
     server_address_ << "localhost:" << port;
     // Setup server
     ServerBuilder builder;
@@ -189,7 +189,7 @@ TEST_F(End2endTest, SimpleRpc) {
 
 TEST_F(End2endTest, MultipleRpcs) {
   ResetStub();
-  vector<std::thread*> threads;
+  std::vector<std::thread*> threads;
   for (int i = 0; i < 10; ++i) {
     threads.push_back(new std::thread(SendRpc, stub_.get(), 10));
   }

test/cpp/end2end/sync_client_async_server_test.cc

@@ -49,7 +49,7 @@
 #include <grpc++/status.h>
 #include <grpc++/stream.h>
 #include "test/cpp/end2end/async_test_server.h"
-#include "net/util/netutil.h"
+#include "test/core/util/port.h"
 #include <gtest/gtest.h>
 
 using grpc::cpp::test::util::EchoRequest;
@@ -73,7 +73,7 @@ void ServerLoop(void* s) {
 class End2endTest : public ::testing::Test {
  protected:
   void SetUp() override {
-    int port = PickUnusedPortOrDie();
+    int port = grpc_pick_unused_port_or_die();
     // TODO(yangg) protobuf has a StringPrintf, maybe use that
     std::ostringstream oss;
     oss << "[::]:" << port;

test/cpp/qps/qpstest.proto

@@ -4,10 +4,6 @@ syntax = "proto2";
 
 package grpc.testing;
 
-option java_api_version = 2;
-option cc_api_version = 2;
-option java_package = "com.google.net.stubby.testing.integration";
-
 enum PayloadType {
   // Compressable text format.
   COMPRESSABLE= 1;

tools/run_tests/jobset.py

@@ -39,6 +39,36 @@ _RUNNING = object()
 _KILLED = object()
 
 
+_COLORS = {
+    'red': 31,
+    'green': 32,
+    'yellow': 33,
+    }
+
+
+_BEGINNING_OF_LINE = '\x1b[0G'
+_CLEAR_LINE = '\x1b[2K'
+
+
+_TAG_COLOR = {
+    'FAILED': 'red',
+    'PASSED': 'green',
+    'START': 'yellow',
+    'WAITING': 'yellow',
+    }
+
+
+def message(tag, message, explanatory_text=None):
+  sys.stdout.write('%s%s\x1b[%dm%s\x1b[0m: %s%s' % (
+      _BEGINNING_OF_LINE,
+      _CLEAR_LINE,
+      _COLORS[_TAG_COLOR[tag]],
+      tag,
+      message,
+      '\n%s\n' % explanatory_text if explanatory_text is not None else ''))
+  sys.stdout.flush()
+
+
 class Job(object):
   """Manages one job."""
 
@@ -49,9 +79,7 @@ class Job(object):
                                      stderr=subprocess.STDOUT,
                                      stdout=self._tempfile)
     self._state = _RUNNING
-    sys.stdout.write('\x1b[0G\x1b[2K\x1b[33mSTART\x1b[0m: %s' %
-                     self._cmdline)
-    sys.stdout.flush()
+    message('START', self._cmdline)
 
   def state(self):
     """Poll current state of the job. Prints messages at completion."""
@@ -60,16 +88,10 @@ class Job(object):
         self._state = _FAILURE
         self._tempfile.seek(0)
         stdout = self._tempfile.read()
-        sys.stdout.write('\x1b[0G\x1b[2K\x1b[31mFAILED\x1b[0m: %s'
-                         ' [ret=%d]\n'
-                         '%s\n' % (
-                             self._cmdline, self._process.returncode, stdout))
-        sys.stdout.flush()
+        message('FAILED', '%s [ret=%d]' % (self._cmdline, self._process.returncode), stdout)
       else:
         self._state = _SUCCESS
-        sys.stdout.write('\x1b[0G\x1b[2K\x1b[32mPASSED\x1b[0m: %s' %
-                         self._cmdline)
-        sys.stdout.flush()
+        message('PASSED', '%s' % self._cmdline)
     return self._state
 
   def kill(self):
@@ -86,6 +108,7 @@ class Jobset(object):
     self._check_cancelled = check_cancelled
     self._cancelled = False
     self._failures = 0
+    self._completed = 0
     self._maxjobs = maxjobs
 
   def start(self, cmdline):
@@ -107,8 +130,11 @@ class Jobset(object):
         if st == _FAILURE: self._failures += 1
         dead.add(job)
       for job in dead:
+        self._completed += 1
         self._running.remove(job)
-      if not dead: return
+      if dead: return
+      message('WAITING', '%d jobs running, %d complete' % (
+          len(self._running), self._completed))
       time.sleep(0.1)
 
   def cancelled(self):

tools/run_tests/run_tests.py

@@ -11,15 +11,44 @@ import time
 import jobset
 import watch_dirs
 
-# flags required for make for each configuration
-_CONFIGS = ['dbg', 'opt', 'tsan', 'msan', 'asan']
+# SimpleConfig: just compile with CONFIG=config, and run the binary to test
+class SimpleConfig(object):
+  def __init__(self, config):
+    self.build_config = config
+
+  def run_command(self, binary):
+    return [binary]
+
+
+# ValgrindConfig: compile with some CONFIG=config, but use valgrind to run
+class ValgrindConfig(object):
+  def __init__(self, config):
+    self.build_config = config
+
+  def run_command(self, binary):
+    return ['valgrind', binary]
+
+
+# different configurations we can run under
+_CONFIGS = {
+  'dbg': SimpleConfig('dbg'),
+  'opt': SimpleConfig('opt'),
+  'tsan': SimpleConfig('tsan'),
+  'msan': SimpleConfig('msan'),
+  'asan': SimpleConfig('asan'),
+  'valgrind': ValgrindConfig('dbg'),
+  }
+
+
+_DEFAULT = ['dbg', 'opt']
+_MAKE_TEST_TARGETS = ['buildtests_c', 'buildtests_cxx']
 
 # parse command line
 argp = argparse.ArgumentParser(description='Run grpc tests.')
 argp.add_argument('-c', '--config',
-                  choices=['all'] + _CONFIGS,
+                  choices=['all'] + sorted(_CONFIGS.keys()),
                   nargs='+',
-                  default=['all'])
+                  default=_DEFAULT)
 argp.add_argument('-t', '--test-filter', nargs='*', default=['*'])
 argp.add_argument('-n', '--runs_per_test', default=1, type=int)
 argp.add_argument('-f', '--forever',
@@ -29,10 +58,11 @@ argp.add_argument('-f', '--forever',
 args = argp.parse_args()
 
 # grab config
-configs = [cfg
-           for cfg in itertools.chain.from_iterable(
-               _CONFIGS if x == 'all' else [x]
-               for x in args.config)]
+run_configs = set(_CONFIGS[cfg]
+                  for cfg in itertools.chain.from_iterable(
+                      _CONFIGS.iterkeys() if x == 'all' else [x]
+                      for x in args.config))
+build_configs = set(cfg.build_config for cfg in run_configs)
 filters = args.test_filter
 runs_per_test = args.runs_per_test
 forever = args.forever
@@ -44,19 +74,22 @@ def _build_and_run(check_cancelled):
   if not jobset.run(
       (['make',
         '-j', '%d' % (multiprocessing.cpu_count() + 1),
-        'buildtests_c',
+        target,
         'CONFIG=%s' % cfg]
-       for cfg in configs), check_cancelled, maxjobs=1):
+       for cfg in build_configs
+       for target in _MAKE_TEST_TARGETS),
+      check_cancelled, maxjobs=1):
     sys.exit(1)
 
   # run all the tests
-  jobset.run(([x]
-              for x in itertools.chain.from_iterable(
-                  itertools.chain.from_iterable(itertools.repeat(
-                      glob.glob('bins/%s/%s_test' % (config, filt)),
-                      runs_per_test))
-                  for config in configs
-                  for filt in filters)), check_cancelled)
+  jobset.run((
+      config.run_command(x)
+      for config in run_configs
+      for filt in filters
+      for x in itertools.chain.from_iterable(itertools.repeat(
+          glob.glob('bins/%s/%s_test' % (
+              config.build_config, filt)),
+          runs_per_test))), check_cancelled)
 
 
 if forever: