Disable counters for trickle, inc timeout

tools/jenkins/run_performance.sh

@@ -20,4 +20,4 @@ set -ex
 cd $(dirname $0)/../..
 
 tools/run_tests/start_port_server.py
-tools/profiling/microbenchmarks/bm_diff/bm_main.py -d origin/$ghprbTargetBranch -b bm_fullstack_trickle -l 4 -t cli_transport_stalls cli_stream_stalls svr_transport_stalls svr_stream_stalls
+tools/profiling/microbenchmarks/bm_diff/bm_main.py -d origin/$ghprbTargetBranch -b bm_fullstack_trickle -l 4 -t cli_transport_stalls cli_stream_stalls svr_transport_stalls svr_stream_stalls --no-counters

tools/profiling/microbenchmarks/bm_diff/bm_build.py

@@ -46,6 +46,12 @@ def _args():
     type=str,
     help='Unique name of this build. To be used as a handle to pass to the other bm* scripts'
   )
+  argp.add_argument(
+    '-c',
+    '--counters',
+    type=bool,
+    default=True,
+    help='Whether or not to run and diff a counters build')
   args = argp.parse_args()
   assert args.name
   return args
@@ -55,16 +61,18 @@ def _make_cmd(cfg, benchmarks, jobs):
   return ['make'] + benchmarks + ['CONFIG=%s' % cfg, '-j', '%d' % jobs]
 
 
-def build(name, benchmarks, jobs):
+def build(name, benchmarks, jobs, counters):
   shutil.rmtree('bm_diff_%s' % name, ignore_errors=True)
   subprocess.check_call(['git', 'submodule', 'update'])
   try:
     subprocess.check_call(_make_cmd('opt', benchmarks, jobs))
-    subprocess.check_call(_make_cmd('counters', benchmarks, jobs))
+    if counters:
+      subprocess.check_call(_make_cmd('counters', benchmarks, jobs))
   except subprocess.CalledProcessError, e:
     subprocess.check_call(['make', 'clean'])
     subprocess.check_call(_make_cmd('opt', benchmarks, jobs))
-    subprocess.check_call(_make_cmd('counters', benchmarks, jobs))
+    if counters:
+      subprocess.check_call(_make_cmd('counters', benchmarks, jobs))
   os.rename(
     'bins',
     'bm_diff_%s' % name,)
@@ -72,4 +80,4 @@ def build(name, benchmarks, jobs):
 
 if __name__ == '__main__':
   args = _args()
-  build(args.name, args.benchmarks, args.jobs)
+  build(args.name, args.benchmarks, args.jobs, args.counters)

tools/profiling/microbenchmarks/bm_diff/bm_diff.py

@@ -67,6 +67,12 @@ def _args():
     default=20,
     help='Number of times to loops the benchmarks. Must match what was passed to bm_run.py'
   )
+  argp.add_argument(
+    '-c',
+    '--counters',
+    type=bool,
+    default=True,
+    help='Whether or not to run and diff a counters build')
   argp.add_argument('-n', '--new', type=str, help='New benchmark name')
   argp.add_argument('-o', '--old', type=str, help='Old benchmark name')
   argp.add_argument(
@@ -121,7 +127,8 @@ def _read_json(filename, badjson_files, nonexistant_files):
   stripped = ".".join(filename.split(".")[:-2])
   try:
     with open(filename) as f:
-      return json.loads(f.read())
+      r = f.read();
+      return json.loads(r)
   except IOError, e:
     if stripped in nonexistant_files:
       nonexistant_files[stripped] += 1
@@ -129,14 +136,17 @@ def _read_json(filename, badjson_files, nonexistant_files):
       nonexistant_files[stripped] = 1
     return None
   except ValueError, e:
+    print r
     if stripped in badjson_files:
       badjson_files[stripped] += 1
     else:
       badjson_files[stripped] = 1
     return None
 
+def fmt_dict(d):
+  return ''.join(["    " + k + ": " + str(d[k]) + "\n" for k in d])
 
-def diff(bms, loops, track, old, new):
+def diff(bms, loops, track, old, new, counters):
   benchmarks = collections.defaultdict(Benchmark)
 
   badjson_files = {}
@@ -148,18 +158,22 @@ def diff(bms, loops, track, old, new):
          '--benchmark_list_tests']).splitlines():
         stripped_line = line.strip().replace("/", "_").replace(
           "<", "_").replace(">", "_").replace(", ", "_")
-        js_new_ctr = _read_json('%s.%s.counters.%s.%d.json' %
-                    (bm, stripped_line, new, loop),
-                    badjson_files, nonexistant_files)
         js_new_opt = _read_json('%s.%s.opt.%s.%d.json' %
                     (bm, stripped_line, new, loop),
                     badjson_files, nonexistant_files)
-        js_old_ctr = _read_json('%s.%s.counters.%s.%d.json' %
-                    (bm, stripped_line, old, loop),
-                    badjson_files, nonexistant_files)
         js_old_opt = _read_json('%s.%s.opt.%s.%d.json' %
                     (bm, stripped_line, old, loop),
                     badjson_files, nonexistant_files)
+        if counters:
+          js_new_ctr = _read_json('%s.%s.counters.%s.%d.json' %
+                      (bm, stripped_line, new, loop),
+                      badjson_files, nonexistant_files)
+          js_old_ctr = _read_json('%s.%s.counters.%s.%d.json' %
+                      (bm, stripped_line, old, loop),
+                      badjson_files, nonexistant_files)
+        else:
+          js_new_ctr = None
+          js_old_ctr = None
 
         if js_new_ctr:
           for row in bm_json.expand_json(js_new_ctr, js_new_opt):
@@ -187,12 +201,12 @@ def diff(bms, loops, track, old, new):
     rows.append([name] + benchmarks[name].row(fields))
   note = None
   if len(badjson_files):
-    note = 'Corrupt JSON data (indicates timeout or crash) = %s' % str(badjson_files)
+    note = 'Corrupt JSON data (indicates timeout or crash): \n%s' % fmt_dict(badjson_files)
   if len(nonexistant_files):
     if note:
-      note += '\n\nMissing files (indicates new benchmark) = %s' % str(nonexistant_files)
+      note += '\n\nMissing files (indicates new benchmark): \n%s' % fmt_dict(nonexistant_files)
     else:
-      note = '\n\nMissing files (indicates new benchmark) = %s' % str(nonexistant_files)
+      note = '\n\nMissing files (indicates new benchmark): \n%s' % fmt_dict(nonexistant_files)
   if rows:
     return tabulate.tabulate(rows, headers=headers, floatfmt='+.2f'), note
   else:
@@ -202,5 +216,5 @@ def diff(bms, loops, track, old, new):
 if __name__ == '__main__':
   args = _args()
   diff, note = diff(args.benchmarks, args.loops, args.track, args.old,
-            args.new)
+            args.new, args.counters)
   print('%s\n%s' % (note, diff if diff else "No performance differences"))

tools/profiling/microbenchmarks/bm_diff/bm_main.py

@@ -80,6 +80,9 @@ def _args():
     type=int,
     default=multiprocessing.cpu_count(),
     help='Number of CPUs to use')
+  argp.add_argument('--counters', dest='counters', action='store_true')
+  argp.add_argument('--no-counters', dest='counters', action='store_false')
+  argp.set_defaults(counters=True)
   args = argp.parse_args()
   assert args.diff_base or args.old, "One of diff_base or old must be set!"
   if args.loops < 3:
@@ -103,7 +106,7 @@ def eintr_be_gone(fn):
 
 def main(args):
 
-  bm_build.build('new', args.benchmarks, args.jobs)
+  bm_build.build('new', args.benchmarks, args.jobs, args.counters)
 
   old = args.old
   if args.diff_base:
@@ -112,16 +115,16 @@ def main(args):
       ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip()
     subprocess.check_call(['git', 'checkout', args.diff_base])
     try:
-      bm_build.build('old', args.benchmarks, args.jobs)
+      bm_build.build(old, args.benchmarks, args.jobs, args.counters)
     finally:
       subprocess.check_call(['git', 'checkout', where_am_i])
       subprocess.check_call(['git', 'submodule', 'update'])
 
-  bm_run.run('new', args.benchmarks, args.jobs, args.loops, args.repetitions)
-  bm_run.run(old, args.benchmarks, args.jobs, args.loops, args.repetitions)
+  bm_run.run('new', args.benchmarks, args.jobs, args.loops, args.repetitions, args.counters)
+  bm_run.run(old, args.benchmarks, args.jobs, args.loops, args.repetitions, args.counters)
 
   diff, note = bm_diff.diff(args.benchmarks, args.loops, args.track, old,
-                'new')
+                'new', args.counters)
   if diff:
     text = 'Performance differences noted:\n' + diff
   else:

tools/profiling/microbenchmarks/bm_diff/bm_run.py

@@ -67,6 +67,12 @@ def _args():
     default=20,
     help='Number of times to loops the benchmarks. More loops cuts down on noise'
   )
+  argp.add_argument(
+    '-c',
+    '--counters',
+    type=bool,
+    default=True,
+    help='Whether or not to run and diff a counters build')
   args = argp.parse_args()
   assert args.name
   if args.loops < 3:
@@ -93,21 +99,22 @@ def _collect_bm_data(bm, cfg, name, reps, idx, loops):
         shortname='%s %s %s %s %d/%d' % (bm, line, cfg, name, idx + 1,
                          loops),
         verbose_success=True,
-        timeout_seconds=60 * 2))
+        timeout_seconds=60 * 60)) # one hour
   return jobs_list
 
 
-def run(name, benchmarks, jobs, loops, reps):
+def run(name, benchmarks, jobs, loops, reps, counters):
   jobs_list = []
   for loop in range(0, loops):
     for bm in benchmarks:
       jobs_list += _collect_bm_data(bm, 'opt', name, reps, loop, loops)
-      jobs_list += _collect_bm_data(bm, 'counters', name, reps, loop,
-                      loops)
+      if counters:
+        jobs_list += _collect_bm_data(bm, 'counters', name, reps, loop,
+                        loops)
   random.shuffle(jobs_list, random.SystemRandom().random)
   jobset.run(jobs_list, maxjobs=jobs)
 
 
 if __name__ == '__main__':
   args = _args()
-  run(args.name, args.benchmarks, args.jobs, args.loops, args.repetitions)
+  run(args.name, args.benchmarks, args.jobs, args.loops, args.repetitions, args.counters)