Split the benchmarks into individual jobspecs

build.yaml

@@ -3466,6 +3466,7 @@ targets:
   - gpr
   args:
   - --benchmark_min_time=0
+  benchmark: true
   defaults: benchmark
   platforms:
   - mac
@@ -3487,6 +3488,7 @@ targets:
   - gpr
   args:
   - --benchmark_min_time=0
+  benchmark: true
   defaults: benchmark
   platforms:
   - mac
@@ -3508,6 +3510,7 @@ targets:
   - gpr
   args:
   - --benchmark_min_time=0
+  benchmark: true
   defaults: benchmark
   platforms:
   - mac
@@ -3529,6 +3532,7 @@ targets:
   - gpr
   args:
   - --benchmark_min_time=0
+  benchmark: true
   defaults: benchmark
   platforms:
   - mac
@@ -3550,6 +3554,7 @@ targets:
   - gpr
   args:
   - --benchmark_min_time=0
+  benchmark: true
   defaults: benchmark
   platforms:
   - mac
@@ -3571,6 +3576,7 @@ targets:
   - gpr
   args:
   - --benchmark_min_time=0
+  benchmark: true
   defaults: benchmark
   platforms:
   - mac
@@ -3592,6 +3598,7 @@ targets:
   - gpr
   args:
   - --benchmark_min_time=4
+  benchmark: true
   defaults: benchmark
   platforms:
   - mac
@@ -3613,6 +3620,7 @@ targets:
   - gpr
   args:
   - --benchmark_min_time=0
+  benchmark: true
   defaults: benchmark
   platforms:
   - mac
@@ -3636,6 +3644,7 @@ targets:
   - gpr
   args:
   - --benchmark_min_time=0
+  benchmark: true
   defaults: benchmark
   excluded_poll_engines:
   - poll
@@ -3663,6 +3672,7 @@ targets:
   - gpr
   args:
   - --benchmark_min_time=0
+  benchmark: true
   defaults: benchmark
   excluded_poll_engines:
   - poll
@@ -3689,6 +3699,7 @@ targets:
   - grpc++_test_config
   args:
   - --benchmark_min_time=0
+  benchmark: true
   defaults: benchmark
   excluded_poll_engines:
   - poll
@@ -3716,6 +3727,7 @@ targets:
   - gpr
   args:
   - --benchmark_min_time=0
+  benchmark: true
   defaults: benchmark
   excluded_poll_engines:
   - poll
@@ -3741,6 +3753,7 @@ targets:
   - gpr
   args:
   - --benchmark_min_time=0
+  benchmark: true
   defaults: benchmark
   platforms:
   - mac
@@ -3762,6 +3775,7 @@ targets:
   - gpr
   args:
   - --benchmark_min_time=0
+  benchmark: true
   defaults: benchmark
   platforms:
   - mac

tools/run_tests/run_tests.py

@@ -323,13 +323,19 @@ class CLanguage(object):
         if cpu_cost == 'capacity':
           cpu_cost = multiprocessing.cpu_count()
         if os.path.isfile(binary):
+          test_prefix = None
           if 'gtest' in target and target['gtest']:
-            # here we parse the output of --gtest_list_tests to build up a
-            # complete list of the tests contained in a binary
-            # for each test, we then add a job to run, filtering for just that
-            # test
+            test_prefix = 'gtest'
+          elif 'benchmark' in target and target['benchmark']:
+            test_prefix = 'benchmark'
+
+          if test_prefix:
+            # here we parse the output of --gtest_list_tests (or 
+            # --benchmark_list_tests)to build up a complete list of 
+            # the tests contained in a binary for each test, we then 
+            # add a job to run, filtering for just that test.
             with open(os.devnull, 'w') as fnull:
-              tests = subprocess.check_output([binary, '--gtest_list_tests'],
+              tests = subprocess.check_output([binary, '--%s_list_tests' % test_prefix],
                                               stderr=fnull)
             base = None
             for line in tests.split('\n'):
@@ -342,7 +348,7 @@ class CLanguage(object):
                 assert base is not None
                 assert line[1] == ' '
                 test = base + line.strip()
-                cmdline = [binary, '--gtest_filter=%s' % test] + target['args']
+                cmdline = [binary, '--%s_filter=%s' % (test_prefix, test)] + target['args']
                 out.append(self.config.job_spec(cmdline,
                                                 shortname='%s %s' % (' '.join(cmdline), shortname_ext),
                                                 cpu_cost=cpu_cost,