tools/profiling

tools/distrib/yapf_code.sh

@@ -24,6 +24,7 @@ DIRS=(
     'tools/codegen'
     'tools/distrib'
     'tools/interop_matrix'
+    'tools/profiling'
 )
 EXCLUSIONS=(
     'grpcio/grpc_*.py'

tools/interop_matrix/create_matrix_images.py

@@ -262,8 +262,8 @@ def maybe_apply_patches_on_git_tag(stack_base, lang, release):
     patch_file = os.path.abspath(
         os.path.join(os.path.dirname(__file__), patch_file_relative_path))
     if not os.path.exists(patch_file):
-        jobset.message('FAILED', 'expected patch file |%s| to exist' %
-                       patch_file)
+        jobset.message('FAILED',
+                       'expected patch file |%s| to exist' % patch_file)
         sys.exit(1)
     subprocess.check_output(
         ['git', 'apply', patch_file], cwd=stack_base, stderr=subprocess.STDOUT)

tools/profiling/bloat/bloat_diff.py

@@ -23,12 +23,11 @@ import subprocess
 import sys
 
 sys.path.append(
-  os.path.join(
-    os.path.dirname(sys.argv[0]), '..', '..', 'run_tests', 'python_utils'))
+    os.path.join(
+        os.path.dirname(sys.argv[0]), '..', '..', 'run_tests', 'python_utils'))
 import comment_on_pr
 
-argp = argparse.ArgumentParser(
-    description='Perform diff on microbenchmarks')
+argp = argparse.ArgumentParser(description='Perform diff on microbenchmarks')
 
 argp.add_argument(
     '-d',
@@ -36,64 +35,59 @@ argp.add_argument(
     type=str,
     help='Commit or branch to compare the current one to')
 
-argp.add_argument(
-    '-j',
-    '--jobs',
-    type=int,
-    default=multiprocessing.cpu_count())
+argp.add_argument('-j', '--jobs', type=int, default=multiprocessing.cpu_count())
 
 args = argp.parse_args()
 
 LIBS = [
-  'libgrpc.so',
-  'libgrpc++.so',
+    'libgrpc.so',
+    'libgrpc++.so',
 ]
 
+
 def build(where):
-  subprocess.check_call('make -j%d' % args.jobs,
-                        shell=True, cwd='.')
-  shutil.rmtree('bloat_diff_%s' % where, ignore_errors=True)
-  os.rename('libs', 'bloat_diff_%s' % where)
+    subprocess.check_call('make -j%d' % args.jobs, shell=True, cwd='.')
+    shutil.rmtree('bloat_diff_%s' % where, ignore_errors=True)
+    os.rename('libs', 'bloat_diff_%s' % where)
+
 
 build('new')
 
 if args.diff_base:
     old = 'old'
     where_am_i = subprocess.check_output(
-      ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip()
+        ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip()
     subprocess.check_call(['git', 'checkout', args.diff_base])
     subprocess.check_call(['git', 'submodule', 'update'])
     try:
-      try:
-        build('old')
-      except subprocess.CalledProcessError, e:
-        subprocess.check_call(['make', 'clean'])
-        build('old')
+        try:
+            build('old')
+        except subprocess.CalledProcessError, e:
+            subprocess.check_call(['make', 'clean'])
+            build('old')
     finally:
-      subprocess.check_call(['git', 'checkout', where_am_i])
-      subprocess.check_call(['git', 'submodule', 'update'])
+        subprocess.check_call(['git', 'checkout', where_am_i])
+        subprocess.check_call(['git', 'submodule', 'update'])
 
-subprocess.check_call('make -j%d' % args.jobs,
-                      shell=True, cwd='third_party/bloaty')
+subprocess.check_call(
+    'make -j%d' % args.jobs, shell=True, cwd='third_party/bloaty')
 
 text = ''
 for lib in LIBS:
-  text += '****************************************************************\n\n'
-  text += lib + '\n\n'
-  old_version = glob.glob('bloat_diff_old/opt/%s' % lib)
-  new_version = glob.glob('bloat_diff_new/opt/%s' % lib)
-  assert len(new_version) == 1
-  cmd = 'third_party/bloaty/bloaty -d compileunits,symbols'
-  if old_version:
-    assert len(old_version) == 1
-    text += subprocess.check_output('%s %s -- %s' %
-                                    (cmd, new_version[0], old_version[0]),
-                                    shell=True)
-  else:
-    text += subprocess.check_output('%s %s' %
-                                    (cmd, new_version[0]),
-                                    shell=True)
-  text += '\n\n'
+    text += '****************************************************************\n\n'
+    text += lib + '\n\n'
+    old_version = glob.glob('bloat_diff_old/opt/%s' % lib)
+    new_version = glob.glob('bloat_diff_new/opt/%s' % lib)
+    assert len(new_version) == 1
+    cmd = 'third_party/bloaty/bloaty -d compileunits,symbols'
+    if old_version:
+        assert len(old_version) == 1
+        text += subprocess.check_output(
+            '%s %s -- %s' % (cmd, new_version[0], old_version[0]), shell=True)
+    else:
+        text += subprocess.check_output(
+            '%s %s' % (cmd, new_version[0]), shell=True)
+    text += '\n\n'
 
 print text
 comment_on_pr.comment_on_pr('```\n%s\n```' % text)

tools/profiling/latency_profile/profile_analyzer.py

@@ -23,7 +23,6 @@ import sys
 import tabulate
 import time
 
-
 SELF_TIME = object()
 TIME_FROM_SCOPE_START = object()
 TIME_TO_SCOPE_END = object()
@@ -31,124 +30,129 @@ TIME_FROM_STACK_START = object()
 TIME_TO_STACK_END = object()
 TIME_FROM_LAST_IMPORTANT = object()
 
-
-argp = argparse.ArgumentParser(description='Process output of basic_prof builds')
+argp = argparse.ArgumentParser(
+    description='Process output of basic_prof builds')
 argp.add_argument('--source', default='latency_trace.txt', type=str)
 argp.add_argument('--fmt', choices=tabulate.tabulate_formats, default='simple')
 argp.add_argument('--out', default='-', type=str)
 args = argp.parse_args()
 
+
 class LineItem(object):
 
-  def __init__(self, line, indent):
-    self.tag = line['tag']
-    self.indent = indent
-    self.start_time = line['t']
-    self.end_time = None
-    self.important = line['imp']
-    self.filename = line['file']
-    self.fileline = line['line']
-    self.times = {}
+    def __init__(self, line, indent):
+        self.tag = line['tag']
+        self.indent = indent
+        self.start_time = line['t']
+        self.end_time = None
+        self.important = line['imp']
+        self.filename = line['file']
+        self.fileline = line['line']
+        self.times = {}
 
 
 class ScopeBuilder(object):
 
-  def __init__(self, call_stack_builder, line):
-    self.call_stack_builder = call_stack_builder
-    self.indent = len(call_stack_builder.stk)
-    self.top_line = LineItem(line, self.indent)
-    call_stack_builder.lines.append(self.top_line)
-    self.first_child_pos = len(call_stack_builder.lines)
-
-  def mark(self, line):
-    line_item = LineItem(line, self.indent + 1)
-    line_item.end_time = line_item.start_time
-    self.call_stack_builder.lines.append(line_item)
-
-  def finish(self, line):
-    assert line['tag'] == self.top_line.tag, (
-        'expected %s, got %s; thread=%s; t0=%f t1=%f' %
-        (self.top_line.tag, line['tag'], line['thd'], self.top_line.start_time,
-         line['t']))
-    final_time_stamp = line['t']
-    assert self.top_line.end_time is None
-    self.top_line.end_time = final_time_stamp
-    self.top_line.important = self.top_line.important or line['imp']
-    assert SELF_TIME not in self.top_line.times
-    self.top_line.times[SELF_TIME] = final_time_stamp - self.top_line.start_time
-    for line in self.call_stack_builder.lines[self.first_child_pos:]:
-      if TIME_FROM_SCOPE_START not in line.times:
-        line.times[TIME_FROM_SCOPE_START] = line.start_time - self.top_line.start_time
-        line.times[TIME_TO_SCOPE_END] = final_time_stamp - line.end_time
+    def __init__(self, call_stack_builder, line):
+        self.call_stack_builder = call_stack_builder
+        self.indent = len(call_stack_builder.stk)
+        self.top_line = LineItem(line, self.indent)
+        call_stack_builder.lines.append(self.top_line)
+        self.first_child_pos = len(call_stack_builder.lines)
+
+    def mark(self, line):
+        line_item = LineItem(line, self.indent + 1)
+        line_item.end_time = line_item.start_time
+        self.call_stack_builder.lines.append(line_item)
+
+    def finish(self, line):
+        assert line['tag'] == self.top_line.tag, (
+            'expected %s, got %s; thread=%s; t0=%f t1=%f' %
+            (self.top_line.tag, line['tag'], line['thd'],
+             self.top_line.start_time, line['t']))
+        final_time_stamp = line['t']
+        assert self.top_line.end_time is None
+        self.top_line.end_time = final_time_stamp
+        self.top_line.important = self.top_line.important or line['imp']
+        assert SELF_TIME not in self.top_line.times
+        self.top_line.times[
+            SELF_TIME] = final_time_stamp - self.top_line.start_time
+        for line in self.call_stack_builder.lines[self.first_child_pos:]:
+            if TIME_FROM_SCOPE_START not in line.times:
+                line.times[
+                    TIME_FROM_SCOPE_START] = line.start_time - self.top_line.start_time
+                line.times[TIME_TO_SCOPE_END] = final_time_stamp - line.end_time
 
 
 class CallStackBuilder(object):
 
-  def __init__(self):
-    self.stk = []
-    self.signature = hashlib.md5()
-    self.lines = []
-
-  def finish(self):
-    start_time = self.lines[0].start_time
-    end_time = self.lines[0].end_time
-    self.signature = self.signature.hexdigest()
-    last_important = start_time
-    for line in self.lines:
-      line.times[TIME_FROM_STACK_START] = line.start_time - start_time
-      line.times[TIME_TO_STACK_END] = end_time - line.end_time
-      line.times[TIME_FROM_LAST_IMPORTANT] = line.start_time - last_important
-      if line.important:
-        last_important = line.end_time
-    last_important = end_time
-
-  def add(self, line):
-    line_type = line['type']
-    self.signature.update(line_type)
-    self.signature.update(line['tag'])
-    if line_type == '{':
-      self.stk.append(ScopeBuilder(self, line))
-      return False
-    elif line_type == '}':
-      assert self.stk, (
-          'expected non-empty stack for closing %s; thread=%s; t=%f' %
-          (line['tag'], line['thd'], line['t']))
-      self.stk.pop().finish(line)
-      if not self.stk:
-        self.finish()
-        return True
-      return False
-    elif line_type == '.' or line_type == '!':
-      self.stk[-1].mark(line)
-      return False
-    else:
-      raise Exception('Unknown line type: \'%s\'' % line_type)
+    def __init__(self):
+        self.stk = []
+        self.signature = hashlib.md5()
+        self.lines = []
+
+    def finish(self):
+        start_time = self.lines[0].start_time
+        end_time = self.lines[0].end_time
+        self.signature = self.signature.hexdigest()
+        last_important = start_time
+        for line in self.lines:
+            line.times[TIME_FROM_STACK_START] = line.start_time - start_time
+            line.times[TIME_TO_STACK_END] = end_time - line.end_time
+            line.times[
+                TIME_FROM_LAST_IMPORTANT] = line.start_time - last_important
+            if line.important:
+                last_important = line.end_time
+        last_important = end_time
+
+    def add(self, line):
+        line_type = line['type']
+        self.signature.update(line_type)
+        self.signature.update(line['tag'])
+        if line_type == '{':
+            self.stk.append(ScopeBuilder(self, line))
+            return False
+        elif line_type == '}':
+            assert self.stk, (
+                'expected non-empty stack for closing %s; thread=%s; t=%f' %
+                (line['tag'], line['thd'], line['t']))
+            self.stk.pop().finish(line)
+            if not self.stk:
+                self.finish()
+                return True
+            return False
+        elif line_type == '.' or line_type == '!':
+            self.stk[-1].mark(line)
+            return False
+        else:
+            raise Exception('Unknown line type: \'%s\'' % line_type)
 
 
 class CallStack(object):
 
-  def __init__(self, initial_call_stack_builder):
-    self.count = 1
-    self.signature = initial_call_stack_builder.signature
-    self.lines = initial_call_stack_builder.lines
-    for line in self.lines:
-      for key, val in line.times.items():
-        line.times[key] = [val]
-
-  def add(self, call_stack_builder):
-    assert self.signature == call_stack_builder.signature
-    self.count += 1
-    assert len(self.lines) == len(call_stack_builder.lines)
-    for lsum, line in itertools.izip(self.lines, call_stack_builder.lines):
-      assert lsum.tag == line.tag
-      assert lsum.times.keys() == line.times.keys()
-      for k, lst in lsum.times.iteritems():
-        lst.append(line.times[k])
-
-  def finish(self):
-    for line in self.lines:
-      for lst in line.times.itervalues():
-        lst.sort()
+    def __init__(self, initial_call_stack_builder):
+        self.count = 1
+        self.signature = initial_call_stack_builder.signature
+        self.lines = initial_call_stack_builder.lines
+        for line in self.lines:
+            for key, val in line.times.items():
+                line.times[key] = [val]
+
+    def add(self, call_stack_builder):
+        assert self.signature == call_stack_builder.signature
+        self.count += 1
+        assert len(self.lines) == len(call_stack_builder.lines)
+        for lsum, line in itertools.izip(self.lines, call_stack_builder.lines):
+            assert lsum.tag == line.tag
+            assert lsum.times.keys() == line.times.keys()
+            for k, lst in lsum.times.iteritems():
+                lst.append(line.times[k])
+
+    def finish(self):
+        for line in self.lines:
+            for lst in line.times.itervalues():
+                lst.sort()
+
 
 builder = collections.defaultdict(CallStackBuilder)
 call_stacks = collections.defaultdict(CallStack)
@@ -156,26 +160,28 @@ call_stacks = collections.defaultdict(CallStack)
 lines = 0
 start = time.time()
 with open(args.source) as f:
-  for line in f:
-    lines += 1
-    inf = json.loads(line)
-    thd = inf['thd']
-    cs = builder[thd]
-    if cs.add(inf):
-      if cs.signature in call_stacks:
-        call_stacks[cs.signature].add(cs)
-      else:
-        call_stacks[cs.signature] = CallStack(cs)
-      del builder[thd]
+    for line in f:
+        lines += 1
+        inf = json.loads(line)
+        thd = inf['thd']
+        cs = builder[thd]
+        if cs.add(inf):
+            if cs.signature in call_stacks:
+                call_stacks[cs.signature].add(cs)
+            else:
+                call_stacks[cs.signature] = CallStack(cs)
+            del builder[thd]
 time_taken = time.time() - start
 
-call_stacks = sorted(call_stacks.values(), key=lambda cs: cs.count, reverse=True)
+call_stacks = sorted(
+    call_stacks.values(), key=lambda cs: cs.count, reverse=True)
 total_stacks = 0
 for cs in call_stacks:
-  total_stacks += cs.count
-  cs.finish()
+    total_stacks += cs.count
+    cs.finish()
+
 
-def percentile(N, percent, key=lambda x:x):
+def percentile(N, percent, key=lambda x: x):
     """
     Find the percentile of a list of values.
 
@@ -187,80 +193,83 @@ def percentile(N, percent, key=lambda x:x):
     """
     if not N:
         return None
-    k = (len(N)-1) * percent
+    k = (len(N) - 1) * percent
     f = math.floor(k)
     c = math.ceil(k)
     if f == c:
         return key(N[int(k)])
-    d0 = key(N[int(f)]) * (c-k)
-    d1 = key(N[int(c)]) * (k-f)
-    return d0+d1
+    d0 = key(N[int(f)]) * (c - k)
+    d1 = key(N[int(c)]) * (k - f)
+    return d0 + d1
+
 
 def tidy_tag(tag):
-  if tag[0:10] == 'GRPC_PTAG_':
-    return tag[10:]
-  return tag
+    if tag[0:10] == 'GRPC_PTAG_':
+        return tag[10:]
+    return tag
+
 
 def time_string(values):
-  num_values = len(values)
-  return '%.1f/%.1f/%.1f' % (
-      1e6 * percentile(values, 0.5),
-      1e6 * percentile(values, 0.9),
-      1e6 * percentile(values, 0.99))
+    num_values = len(values)
+    return '%.1f/%.1f/%.1f' % (1e6 * percentile(values, 0.5),
+                               1e6 * percentile(values, 0.9),
+                               1e6 * percentile(values, 0.99))
+
 
 def time_format(idx):
-  def ent(line, idx=idx):
-    if idx in line.times:
-      return time_string(line.times[idx])
-    return ''
-  return ent
 
-BANNER = {
-    'simple': 'Count: %(count)d',
-    'html': '<h1>Count: %(count)d</h1>'
-}
+    def ent(line, idx=idx):
+        if idx in line.times:
+            return time_string(line.times[idx])
+        return ''
+
+    return ent
+
+
+BANNER = {'simple': 'Count: %(count)d', 'html': '<h1>Count: %(count)d</h1>'}
 
 FORMAT = [
-  ('TAG', lambda line: '..'*line.indent + tidy_tag(line.tag)),
-  ('LOC', lambda line: '%s:%d' % (line.filename[line.filename.rfind('/')+1:], line.fileline)),
-  ('IMP', lambda line: '*' if line.important else ''),
-  ('FROM_IMP', time_format(TIME_FROM_LAST_IMPORTANT)),
-  ('FROM_STACK_START', time_format(TIME_FROM_STACK_START)),
-  ('SELF', time_format(SELF_TIME)),
-  ('TO_STACK_END', time_format(TIME_TO_STACK_END)),
-  ('FROM_SCOPE_START', time_format(TIME_FROM_SCOPE_START)),
-  ('SELF', time_format(SELF_TIME)),
-  ('TO_SCOPE_END', time_format(TIME_TO_SCOPE_END)),
+    ('TAG', lambda line: '..' * line.indent + tidy_tag(line.tag)),
+    ('LOC',
+     lambda line: '%s:%d' % (line.filename[line.filename.rfind('/') + 1:], line.fileline)
+    ),
+    ('IMP', lambda line: '*' if line.important else ''),
+    ('FROM_IMP', time_format(TIME_FROM_LAST_IMPORTANT)),
+    ('FROM_STACK_START', time_format(TIME_FROM_STACK_START)),
+    ('SELF', time_format(SELF_TIME)),
+    ('TO_STACK_END', time_format(TIME_TO_STACK_END)),
+    ('FROM_SCOPE_START', time_format(TIME_FROM_SCOPE_START)),
+    ('SELF', time_format(SELF_TIME)),
+    ('TO_SCOPE_END', time_format(TIME_TO_SCOPE_END)),
 ]
 
 out = sys.stdout
 if args.out != '-':
-  out = open(args.out, 'w')
+    out = open(args.out, 'w')
 
 if args.fmt == 'html':
-  print >>out, '<html>'
-  print >>out, '<head>'
-  print >>out, '<title>Profile Report</title>'
-  print >>out, '</head>'
+    print >> out, '<html>'
+    print >> out, '<head>'
+    print >> out, '<title>Profile Report</title>'
+    print >> out, '</head>'
 
 accounted_for = 0
 for cs in call_stacks:
-  if args.fmt in BANNER:
-    print >>out, BANNER[args.fmt] % {
-        'count': cs.count,
-    }
-  header, _ = zip(*FORMAT)
-  table = []
-  for line in cs.lines:
-    fields = []
-    for _, fn in FORMAT:
-      fields.append(fn(line))
-    table.append(fields)
-  print >>out, tabulate.tabulate(table, header, tablefmt=args.fmt)
-  accounted_for += cs.count
-  if accounted_for > .99 * total_stacks:
-    break
+    if args.fmt in BANNER:
+        print >> out, BANNER[args.fmt] % {
+            'count': cs.count,
+        }
+    header, _ = zip(*FORMAT)
+    table = []
+    for line in cs.lines:
+        fields = []
+        for _, fn in FORMAT:
+            fields.append(fn(line))
+        table.append(fields)
+    print >> out, tabulate.tabulate(table, header, tablefmt=args.fmt)
+    accounted_for += cs.count
+    if accounted_for > .99 * total_stacks:
+        break
 
 if args.fmt == 'html':
-  print '</html>'
-
+    print '</html>'

tools/profiling/microbenchmarks/bm2bq.py

@@ -28,37 +28,38 @@ import subprocess
 columns = []
 
 for row in json.loads(
-    subprocess.check_output([
-      'bq','--format=json','show','microbenchmarks.microbenchmarks']))['schema']['fields']:
-  columns.append((row['name'], row['type'].lower()))
+        subprocess.check_output([
+            'bq', '--format=json', 'show', 'microbenchmarks.microbenchmarks'
+        ]))['schema']['fields']:
+    columns.append((row['name'], row['type'].lower()))
 
 SANITIZE = {
-  'integer': int,
-  'float': float,
-  'boolean': bool,
-  'string': str,
-  'timestamp': str,
+    'integer': int,
+    'float': float,
+    'boolean': bool,
+    'string': str,
+    'timestamp': str,
 }
 
 if sys.argv[1] == '--schema':
-  print ',\n'.join('%s:%s' % (k, t.upper()) for k, t in columns)
-  sys.exit(0)
+    print ',\n'.join('%s:%s' % (k, t.upper()) for k, t in columns)
+    sys.exit(0)
 
 with open(sys.argv[1]) as f:
-  js = json.loads(f.read())
+    js = json.loads(f.read())
 
 if len(sys.argv) > 2:
-  with open(sys.argv[2]) as f:
-    js2 = json.loads(f.read())
+    with open(sys.argv[2]) as f:
+        js2 = json.loads(f.read())
 else:
-  js2 = None
+    js2 = None
 
-writer = csv.DictWriter(sys.stdout, [c for c,t in columns])
+writer = csv.DictWriter(sys.stdout, [c for c, t in columns])
 
 for row in bm_json.expand_json(js, js2):
-  sane_row = {}
-  for name, sql_type in columns:
-    if name in row:
-      if row[name] == '': continue
-      sane_row[name] = SANITIZE[sql_type](row[name])
-  writer.writerow(sane_row)
+    sane_row = {}
+    for name, sql_type in columns:
+        if name in row:
+            if row[name] == '': continue
+            sane_row[name] = SANITIZE[sql_type](row[name])
+    writer.writerow(sane_row)

tools/profiling/microbenchmarks/bm_diff/bm_build.py

@@ -13,7 +13,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """ Python utility to build opt and counters benchmarks """
 
 import bm_constants
@@ -26,55 +25,55 @@ import shutil
 
 
 def _args():
-  argp = argparse.ArgumentParser(description='Builds microbenchmarks')
-  argp.add_argument(
-    '-b',
-    '--benchmarks',
-    nargs='+',
-    choices=bm_constants._AVAILABLE_BENCHMARK_TESTS,
-    default=bm_constants._AVAILABLE_BENCHMARK_TESTS,
-    help='Which benchmarks to build')
-  argp.add_argument(
-    '-j',
-    '--jobs',
-    type=int,
-    default=multiprocessing.cpu_count(),
-    help='How many CPUs to dedicate to this task')
-  argp.add_argument(
-    '-n',
-    '--name',
-    type=str,
-    help='Unique name of this build. To be used as a handle to pass to the other bm* scripts'
-  )
-  argp.add_argument('--counters', dest='counters', action='store_true')
-  argp.add_argument('--no-counters', dest='counters', action='store_false')
-  argp.set_defaults(counters=True)
-  args = argp.parse_args()
-  assert args.name
-  return args
+    argp = argparse.ArgumentParser(description='Builds microbenchmarks')
+    argp.add_argument(
+        '-b',
+        '--benchmarks',
+        nargs='+',
+        choices=bm_constants._AVAILABLE_BENCHMARK_TESTS,
+        default=bm_constants._AVAILABLE_BENCHMARK_TESTS,
+        help='Which benchmarks to build')
+    argp.add_argument(
+        '-j',
+        '--jobs',
+        type=int,
+        default=multiprocessing.cpu_count(),
+        help='How many CPUs to dedicate to this task')
+    argp.add_argument(
+        '-n',
+        '--name',
+        type=str,
+        help='Unique name of this build. To be used as a handle to pass to the other bm* scripts'
+    )
+    argp.add_argument('--counters', dest='counters', action='store_true')
+    argp.add_argument('--no-counters', dest='counters', action='store_false')
+    argp.set_defaults(counters=True)
+    args = argp.parse_args()
+    assert args.name
+    return args
 
 
 def _make_cmd(cfg, benchmarks, jobs):
-  return ['make'] + benchmarks + ['CONFIG=%s' % cfg, '-j', '%d' % jobs]
+    return ['make'] + benchmarks + ['CONFIG=%s' % cfg, '-j', '%d' % jobs]
 
 
 def build(name, benchmarks, jobs, counters):
-  shutil.rmtree('bm_diff_%s' % name, ignore_errors=True)
-  subprocess.check_call(['git', 'submodule', 'update'])
-  try:
-    subprocess.check_call(_make_cmd('opt', benchmarks, jobs))
-    if counters:
-      subprocess.check_call(_make_cmd('counters', benchmarks, jobs))
-  except subprocess.CalledProcessError, e:
-    subprocess.check_call(['make', 'clean'])
-    subprocess.check_call(_make_cmd('opt', benchmarks, jobs))
-    if counters:
-      subprocess.check_call(_make_cmd('counters', benchmarks, jobs))
-  os.rename(
-    'bins',
-    'bm_diff_%s' % name,)
+    shutil.rmtree('bm_diff_%s' % name, ignore_errors=True)
+    subprocess.check_call(['git', 'submodule', 'update'])
+    try:
+        subprocess.check_call(_make_cmd('opt', benchmarks, jobs))
+        if counters:
+            subprocess.check_call(_make_cmd('counters', benchmarks, jobs))
+    except subprocess.CalledProcessError, e:
+        subprocess.check_call(['make', 'clean'])
+        subprocess.check_call(_make_cmd('opt', benchmarks, jobs))
+        if counters:
+            subprocess.check_call(_make_cmd('counters', benchmarks, jobs))
+    os.rename(
+        'bins',
+        'bm_diff_%s' % name,)
 
 
 if __name__ == '__main__':
-  args = _args()
-  build(args.name, args.benchmarks, args.jobs, args.counters)
+    args = _args()
+    build(args.name, args.benchmarks, args.jobs, args.counters)

tools/profiling/microbenchmarks/bm_diff/bm_constants.py

@@ -13,19 +13,19 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """ Configurable constants for the bm_*.py family """
 
 _AVAILABLE_BENCHMARK_TESTS = [
-  'bm_fullstack_unary_ping_pong', 'bm_fullstack_streaming_ping_pong',
-  'bm_fullstack_streaming_pump', 'bm_closure', 'bm_cq', 'bm_call_create',
-  'bm_error', 'bm_chttp2_hpack', 'bm_chttp2_transport', 'bm_pollset',
-  'bm_metadata', 'bm_fullstack_trickle'
+    'bm_fullstack_unary_ping_pong', 'bm_fullstack_streaming_ping_pong',
+    'bm_fullstack_streaming_pump', 'bm_closure', 'bm_cq', 'bm_call_create',
+    'bm_error', 'bm_chttp2_hpack', 'bm_chttp2_transport', 'bm_pollset',
+    'bm_metadata', 'bm_fullstack_trickle'
 ]
 
-_INTERESTING = ('cpu_time', 'real_time', 'call_initial_size-median', 'locks_per_iteration',
-        'allocs_per_iteration', 'writes_per_iteration',
-        'atm_cas_per_iteration', 'atm_add_per_iteration',
-        'nows_per_iteration', 'cli_transport_stalls_per_iteration', 
-        'cli_stream_stalls_per_iteration', 'svr_transport_stalls_per_iteration',
-        'svr_stream_stalls_per_iteration', 'http2_pings_sent_per_iteration')
+_INTERESTING = (
+    'cpu_time', 'real_time', 'call_initial_size-median', 'locks_per_iteration',
+    'allocs_per_iteration', 'writes_per_iteration', 'atm_cas_per_iteration',
+    'atm_add_per_iteration', 'nows_per_iteration',
+    'cli_transport_stalls_per_iteration', 'cli_stream_stalls_per_iteration',
+    'svr_transport_stalls_per_iteration', 'svr_stream_stalls_per_iteration',
+    'http2_pings_sent_per_iteration')

tools/profiling/microbenchmarks/bm_diff/bm_diff.py

@@ -34,190 +34,195 @@ verbose = False
 
 
 def _median(ary):
-  assert (len(ary))
-  ary = sorted(ary)
-  n = len(ary)
-  if n % 2 == 0:
-    return (ary[(n - 1) / 2] + ary[(n - 1) / 2 + 1]) / 2.0
-  else:
-    return ary[n / 2]
+    assert (len(ary))
+    ary = sorted(ary)
+    n = len(ary)
+    if n % 2 == 0:
+        return (ary[(n - 1) / 2] + ary[(n - 1) / 2 + 1]) / 2.0
+    else:
+        return ary[n / 2]
 
 
 def _args():
-  argp = argparse.ArgumentParser(
-    description='Perform diff on microbenchmarks')
-  argp.add_argument(
-    '-t',
-    '--track',
-    choices=sorted(bm_constants._INTERESTING),
-    nargs='+',
-    default=sorted(bm_constants._INTERESTING),
-    help='Which metrics to track')
-  argp.add_argument(
-    '-b',
-    '--benchmarks',
-    nargs='+',
-    choices=bm_constants._AVAILABLE_BENCHMARK_TESTS,
-    default=bm_constants._AVAILABLE_BENCHMARK_TESTS,
-    help='Which benchmarks to run')
-  argp.add_argument(
-    '-l',
-    '--loops',
-    type=int,
-    default=20,
-    help='Number of times to loops the benchmarks. Must match what was passed to bm_run.py'
-  )
-  argp.add_argument(
-    '-r',
-    '--regex',
-    type=str,
-    default="",
-    help='Regex to filter benchmarks run')
-  argp.add_argument('--counters', dest='counters', action='store_true')
-  argp.add_argument('--no-counters', dest='counters', action='store_false')
-  argp.set_defaults(counters=True)
-  argp.add_argument('-n', '--new', type=str, help='New benchmark name')
-  argp.add_argument('-o', '--old', type=str, help='Old benchmark name')
-  argp.add_argument(
-    '-v', '--verbose', type=bool, help='Print details of before/after')
-  args = argp.parse_args()
-  global verbose
-  if args.verbose: verbose = True
-  assert args.new
-  assert args.old
-  return args
+    argp = argparse.ArgumentParser(
+        description='Perform diff on microbenchmarks')
+    argp.add_argument(
+        '-t',
+        '--track',
+        choices=sorted(bm_constants._INTERESTING),
+        nargs='+',
+        default=sorted(bm_constants._INTERESTING),
+        help='Which metrics to track')
+    argp.add_argument(
+        '-b',
+        '--benchmarks',
+        nargs='+',
+        choices=bm_constants._AVAILABLE_BENCHMARK_TESTS,
+        default=bm_constants._AVAILABLE_BENCHMARK_TESTS,
+        help='Which benchmarks to run')
+    argp.add_argument(
+        '-l',
+        '--loops',
+        type=int,
+        default=20,
+        help='Number of times to loops the benchmarks. Must match what was passed to bm_run.py'
+    )
+    argp.add_argument(
+        '-r',
+        '--regex',
+        type=str,
+        default="",
+        help='Regex to filter benchmarks run')
+    argp.add_argument('--counters', dest='counters', action='store_true')
+    argp.add_argument('--no-counters', dest='counters', action='store_false')
+    argp.set_defaults(counters=True)
+    argp.add_argument('-n', '--new', type=str, help='New benchmark name')
+    argp.add_argument('-o', '--old', type=str, help='Old benchmark name')
+    argp.add_argument(
+        '-v', '--verbose', type=bool, help='Print details of before/after')
+    args = argp.parse_args()
+    global verbose
+    if args.verbose: verbose = True
+    assert args.new
+    assert args.old
+    return args
 
 
 def _maybe_print(str):
-  if verbose: print str
+    if verbose: print str
 
 
 class Benchmark:
 
-  def __init__(self):
-    self.samples = {
-      True: collections.defaultdict(list),
-      False: collections.defaultdict(list)
-    }
-    self.final = {}
-
-  def add_sample(self, track, data, new):
-    for f in track:
-      if f in data:
-        self.samples[new][f].append(float(data[f]))
-
-  def process(self, track, new_name, old_name):
-    for f in sorted(track):
-      new = self.samples[True][f]
-      old = self.samples[False][f]
-      if not new or not old: continue
-      mdn_diff = abs(_median(new) - _median(old))
-      _maybe_print('%s: %s=%r %s=%r mdn_diff=%r' %
-             (f, new_name, new, old_name, old, mdn_diff))
-      s = bm_speedup.speedup(new, old, 1e-5)
-      if abs(s) > 3:
-        if mdn_diff > 0.5 or 'trickle' in f:
-          self.final[f] = '%+d%%' % s
-    return self.final.keys()
-
-  def skip(self):
-    return not self.final
-
-  def row(self, flds):
-    return [self.final[f] if f in self.final else '' for f in flds]
+    def __init__(self):
+        self.samples = {
+            True: collections.defaultdict(list),
+            False: collections.defaultdict(list)
+        }
+        self.final = {}
+
+    def add_sample(self, track, data, new):
+        for f in track:
+            if f in data:
+                self.samples[new][f].append(float(data[f]))
+
+    def process(self, track, new_name, old_name):
+        for f in sorted(track):
+            new = self.samples[True][f]
+            old = self.samples[False][f]
+            if not new or not old: continue
+            mdn_diff = abs(_median(new) - _median(old))
+            _maybe_print('%s: %s=%r %s=%r mdn_diff=%r' %
+                         (f, new_name, new, old_name, old, mdn_diff))
+            s = bm_speedup.speedup(new, old, 1e-5)
+            if abs(s) > 3:
+                if mdn_diff > 0.5 or 'trickle' in f:
+                    self.final[f] = '%+d%%' % s
+        return self.final.keys()
+
+    def skip(self):
+        return not self.final
+
+    def row(self, flds):
+        return [self.final[f] if f in self.final else '' for f in flds]
 
 
 def _read_json(filename, badjson_files, nonexistant_files):
-  stripped = ".".join(filename.split(".")[:-2])
-  try:
-    with open(filename) as f:
-      r = f.read();
-      return json.loads(r)
-  except IOError, e:
-    if stripped in nonexistant_files:
-      nonexistant_files[stripped] += 1
-    else:
-      nonexistant_files[stripped] = 1
-    return None
-  except ValueError, e:
-    print r
-    if stripped in badjson_files:
-      badjson_files[stripped] += 1
-    else:
-      badjson_files[stripped] = 1
-    return None
+    stripped = ".".join(filename.split(".")[:-2])
+    try:
+        with open(filename) as f:
+            r = f.read()
+            return json.loads(r)
+    except IOError, e:
+        if stripped in nonexistant_files:
+            nonexistant_files[stripped] += 1
+        else:
+            nonexistant_files[stripped] = 1
+        return None
+    except ValueError, e:
+        print r
+        if stripped in badjson_files:
+            badjson_files[stripped] += 1
+        else:
+            badjson_files[stripped] = 1
+        return None
+
 
 def fmt_dict(d):
-  return ''.join(["    " + k + ": " + str(d[k]) + "\n" for k in d])
+    return ''.join(["    " + k + ": " + str(d[k]) + "\n" for k in d])
+
 
 def diff(bms, loops, regex, track, old, new, counters):
-  benchmarks = collections.defaultdict(Benchmark)
-
-  badjson_files = {}
-  nonexistant_files = {}
-  for bm in bms:
-    for loop in range(0, loops):
-      for line in subprocess.check_output(
-        ['bm_diff_%s/opt/%s' % (old, bm),
-         '--benchmark_list_tests', 
-         '--benchmark_filter=%s' % regex]).splitlines():
-        stripped_line = line.strip().replace("/", "_").replace(
-          "<", "_").replace(">", "_").replace(", ", "_")
-        js_new_opt = _read_json('%s.%s.opt.%s.%d.json' %
-                    (bm, stripped_line, new, loop),
-                    badjson_files, nonexistant_files)
-        js_old_opt = _read_json('%s.%s.opt.%s.%d.json' %
-                    (bm, stripped_line, old, loop),
-                    badjson_files, nonexistant_files)
-        if counters:
-          js_new_ctr = _read_json('%s.%s.counters.%s.%d.json' %
-                      (bm, stripped_line, new, loop),
-                      badjson_files, nonexistant_files)
-          js_old_ctr = _read_json('%s.%s.counters.%s.%d.json' %
-                      (bm, stripped_line, old, loop),
-                      badjson_files, nonexistant_files)
+    benchmarks = collections.defaultdict(Benchmark)
+
+    badjson_files = {}
+    nonexistant_files = {}
+    for bm in bms:
+        for loop in range(0, loops):
+            for line in subprocess.check_output([
+                    'bm_diff_%s/opt/%s' % (old, bm), '--benchmark_list_tests',
+                    '--benchmark_filter=%s' % regex
+            ]).splitlines():
+                stripped_line = line.strip().replace("/", "_").replace(
+                    "<", "_").replace(">", "_").replace(", ", "_")
+                js_new_opt = _read_json('%s.%s.opt.%s.%d.json' %
+                                        (bm, stripped_line, new, loop),
+                                        badjson_files, nonexistant_files)
+                js_old_opt = _read_json('%s.%s.opt.%s.%d.json' %
+                                        (bm, stripped_line, old, loop),
+                                        badjson_files, nonexistant_files)
+                if counters:
+                    js_new_ctr = _read_json('%s.%s.counters.%s.%d.json' %
+                                            (bm, stripped_line, new, loop),
+                                            badjson_files, nonexistant_files)
+                    js_old_ctr = _read_json('%s.%s.counters.%s.%d.json' %
+                                            (bm, stripped_line, old, loop),
+                                            badjson_files, nonexistant_files)
+                else:
+                    js_new_ctr = None
+                    js_old_ctr = None
+
+                for row in bm_json.expand_json(js_new_ctr, js_new_opt):
+                    name = row['cpp_name']
+                    if name.endswith('_mean') or name.endswith('_stddev'):
+                        continue
+                    benchmarks[name].add_sample(track, row, True)
+                for row in bm_json.expand_json(js_old_ctr, js_old_opt):
+                    name = row['cpp_name']
+                    if name.endswith('_mean') or name.endswith('_stddev'):
+                        continue
+                    benchmarks[name].add_sample(track, row, False)
+
+    really_interesting = set()
+    for name, bm in benchmarks.items():
+        _maybe_print(name)
+        really_interesting.update(bm.process(track, new, old))
+    fields = [f for f in track if f in really_interesting]
+
+    headers = ['Benchmark'] + fields
+    rows = []
+    for name in sorted(benchmarks.keys()):
+        if benchmarks[name].skip(): continue
+        rows.append([name] + benchmarks[name].row(fields))
+    note = None
+    if len(badjson_files):
+        note = 'Corrupt JSON data (indicates timeout or crash): \n%s' % fmt_dict(
+            badjson_files)
+    if len(nonexistant_files):
+        if note:
+            note += '\n\nMissing files (indicates new benchmark): \n%s' % fmt_dict(
+                nonexistant_files)
         else:
-          js_new_ctr = None
-          js_old_ctr = None
-
-        for row in bm_json.expand_json(js_new_ctr, js_new_opt):
-          name = row['cpp_name']
-          if name.endswith('_mean') or name.endswith('_stddev'):
-            continue
-          benchmarks[name].add_sample(track, row, True)
-        for row in bm_json.expand_json(js_old_ctr, js_old_opt):
-          name = row['cpp_name']
-          if name.endswith('_mean') or name.endswith('_stddev'):
-            continue
-          benchmarks[name].add_sample(track, row, False)
-
-  really_interesting = set()
-  for name, bm in benchmarks.items():
-    _maybe_print(name)
-    really_interesting.update(bm.process(track, new, old))
-  fields = [f for f in track if f in really_interesting]
-
-  headers = ['Benchmark'] + fields
-  rows = []
-  for name in sorted(benchmarks.keys()):
-    if benchmarks[name].skip(): continue
-    rows.append([name] + benchmarks[name].row(fields))
-  note = None
-  if len(badjson_files):
-    note = 'Corrupt JSON data (indicates timeout or crash): \n%s' % fmt_dict(badjson_files)
-  if len(nonexistant_files):
-    if note:
-      note += '\n\nMissing files (indicates new benchmark): \n%s' % fmt_dict(nonexistant_files)
+            note = '\n\nMissing files (indicates new benchmark): \n%s' % fmt_dict(
+                nonexistant_files)
+    if rows:
+        return tabulate.tabulate(rows, headers=headers, floatfmt='+.2f'), note
     else:
-      note = '\n\nMissing files (indicates new benchmark): \n%s' % fmt_dict(nonexistant_files)
-  if rows:
-    return tabulate.tabulate(rows, headers=headers, floatfmt='+.2f'), note
-  else:
-    return None, note
+        return None, note
 
 
 if __name__ == '__main__':
-  args = _args()
-  diff, note = diff(args.benchmarks, args.loops, args.regex, args.track, args.old,
-            args.new, args.counters)
-  print('%s\n%s' % (note, diff if diff else "No performance differences"))
+    args = _args()
+    diff, note = diff(args.benchmarks, args.loops, args.regex, args.track,
+                      args.old, args.new, args.counters)
+    print('%s\n%s' % (note, diff if diff else "No performance differences"))

tools/profiling/microbenchmarks/bm_diff/bm_main.py

@@ -13,7 +13,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """ Runs the entire bm_*.py pipeline, and possible comments on the PR """
 
 import bm_constants
@@ -29,129 +28,132 @@ import multiprocessing
 import subprocess
 
 sys.path.append(
-  os.path.join(
-    os.path.dirname(sys.argv[0]), '..', '..', 'run_tests', 'python_utils'))
+    os.path.join(
+        os.path.dirname(sys.argv[0]), '..', '..', 'run_tests', 'python_utils'))
 import comment_on_pr
 
 sys.path.append(
-  os.path.join(
-    os.path.dirname(sys.argv[0]), '..', '..', '..', 'run_tests',
-    'python_utils'))
+    os.path.join(
+        os.path.dirname(sys.argv[0]), '..', '..', '..', 'run_tests',
+        'python_utils'))
 import jobset
 
 
 def _args():
-  argp = argparse.ArgumentParser(
-    description='Perform diff on microbenchmarks')
-  argp.add_argument(
-    '-t',
-    '--track',
-    choices=sorted(bm_constants._INTERESTING),
-    nargs='+',
-    default=sorted(bm_constants._INTERESTING),
-    help='Which metrics to track')
-  argp.add_argument(
-    '-b',
-    '--benchmarks',
-    nargs='+',
-    choices=bm_constants._AVAILABLE_BENCHMARK_TESTS,
-    default=bm_constants._AVAILABLE_BENCHMARK_TESTS,
-    help='Which benchmarks to run')
-  argp.add_argument(
-    '-d',
-    '--diff_base',
-    type=str,
-    help='Commit or branch to compare the current one to')
-  argp.add_argument(
-    '-o',
-    '--old',
-    default='old',
-    type=str,
-    help='Name of baseline run to compare to. Ususally just called "old"')
-  argp.add_argument(
-    '-r',
-    '--regex',
-    type=str,
-    default="",
-    help='Regex to filter benchmarks run')
-  argp.add_argument(
-    '-l',
-    '--loops',
-    type=int,
-    default=10,
-    help='Number of times to loops the benchmarks. More loops cuts down on noise'
-  )
-  argp.add_argument(
-    '-j',
-    '--jobs',
-    type=int,
-    default=multiprocessing.cpu_count(),
-    help='Number of CPUs to use')
-  argp.add_argument(
-    '--pr_comment_name',
-    type=str,
-    default="microbenchmarks",
-    help='Name that Jenkins will use to commen on the PR')
-  argp.add_argument('--counters', dest='counters', action='store_true')
-  argp.add_argument('--no-counters', dest='counters', action='store_false')
-  argp.set_defaults(counters=True)
-  args = argp.parse_args()
-  assert args.diff_base or args.old, "One of diff_base or old must be set!"
-  if args.loops < 3:
-    print "WARNING: This run will likely be noisy. Increase loops."
-  return args
+    argp = argparse.ArgumentParser(
+        description='Perform diff on microbenchmarks')
+    argp.add_argument(
+        '-t',
+        '--track',
+        choices=sorted(bm_constants._INTERESTING),
+        nargs='+',
+        default=sorted(bm_constants._INTERESTING),
+        help='Which metrics to track')
+    argp.add_argument(
+        '-b',
+        '--benchmarks',
+        nargs='+',
+        choices=bm_constants._AVAILABLE_BENCHMARK_TESTS,
+        default=bm_constants._AVAILABLE_BENCHMARK_TESTS,
+        help='Which benchmarks to run')
+    argp.add_argument(
+        '-d',
+        '--diff_base',
+        type=str,
+        help='Commit or branch to compare the current one to')
+    argp.add_argument(
+        '-o',
+        '--old',
+        default='old',
+        type=str,
+        help='Name of baseline run to compare to. Ususally just called "old"')
+    argp.add_argument(
+        '-r',
+        '--regex',
+        type=str,
+        default="",
+        help='Regex to filter benchmarks run')
+    argp.add_argument(
+        '-l',
+        '--loops',
+        type=int,
+        default=10,
+        help='Number of times to loops the benchmarks. More loops cuts down on noise'
+    )
+    argp.add_argument(
+        '-j',
+        '--jobs',
+        type=int,
+        default=multiprocessing.cpu_count(),
+        help='Number of CPUs to use')
+    argp.add_argument(
+        '--pr_comment_name',
+        type=str,
+        default="microbenchmarks",
+        help='Name that Jenkins will use to commen on the PR')
+    argp.add_argument('--counters', dest='counters', action='store_true')
+    argp.add_argument('--no-counters', dest='counters', action='store_false')
+    argp.set_defaults(counters=True)
+    args = argp.parse_args()
+    assert args.diff_base or args.old, "One of diff_base or old must be set!"
+    if args.loops < 3:
+        print "WARNING: This run will likely be noisy. Increase loops."
+    return args
 
 
 def eintr_be_gone(fn):
-  """Run fn until it doesn't stop because of EINTR"""
+    """Run fn until it doesn't stop because of EINTR"""
 
-  def inner(*args):
-    while True:
-      try:
-        return fn(*args)
-      except IOError, e:
-        if e.errno != errno.EINTR:
-          raise
+    def inner(*args):
+        while True:
+            try:
+                return fn(*args)
+            except IOError, e:
+                if e.errno != errno.EINTR:
+                    raise
 
-  return inner
+    return inner
 
 
 def main(args):
 
-  bm_build.build('new', args.benchmarks, args.jobs, args.counters)
-
-  old = args.old
-  if args.diff_base:
-    old = 'old'
-    where_am_i = subprocess.check_output(
-      ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip()
-    subprocess.check_call(['git', 'checkout', args.diff_base])
-    try:
-      bm_build.build(old, args.benchmarks, args.jobs, args.counters)
-    finally:
-      subprocess.check_call(['git', 'checkout', where_am_i])
-      subprocess.check_call(['git', 'submodule', 'update'])
-
-  jobs_list = []
-  jobs_list += bm_run.create_jobs('new', args.benchmarks, args.loops, args.regex, args.counters)
-  jobs_list += bm_run.create_jobs(old, args.benchmarks, args.loops, args.regex, args.counters)
-
-  # shuffle all jobs to eliminate noise from GCE CPU drift
-  random.shuffle(jobs_list, random.SystemRandom().random)
-  jobset.run(jobs_list, maxjobs=args.jobs)
-
-  diff, note = bm_diff.diff(args.benchmarks, args.loops, args.regex, args.track, old,
-                'new', args.counters)
-  if diff:
-    text = '[%s] Performance differences noted:\n%s' % (args.pr_comment_name, diff)
-  else:
-    text = '[%s] No significant performance differences' % args.pr_comment_name
-  if note:
-    text = note + '\n\n' + text
-  print('%s' % text)
-  comment_on_pr.comment_on_pr('```\n%s\n```' % text)
+    bm_build.build('new', args.benchmarks, args.jobs, args.counters)
+
+    old = args.old
+    if args.diff_base:
+        old = 'old'
+        where_am_i = subprocess.check_output(
+            ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip()
+        subprocess.check_call(['git', 'checkout', args.diff_base])
+        try:
+            bm_build.build(old, args.benchmarks, args.jobs, args.counters)
+        finally:
+            subprocess.check_call(['git', 'checkout', where_am_i])
+            subprocess.check_call(['git', 'submodule', 'update'])
+
+    jobs_list = []
+    jobs_list += bm_run.create_jobs('new', args.benchmarks, args.loops,
+                                    args.regex, args.counters)
+    jobs_list += bm_run.create_jobs(old, args.benchmarks, args.loops,
+                                    args.regex, args.counters)
+
+    # shuffle all jobs to eliminate noise from GCE CPU drift
+    random.shuffle(jobs_list, random.SystemRandom().random)
+    jobset.run(jobs_list, maxjobs=args.jobs)
+
+    diff, note = bm_diff.diff(args.benchmarks, args.loops, args.regex,
+                              args.track, old, 'new', args.counters)
+    if diff:
+        text = '[%s] Performance differences noted:\n%s' % (
+            args.pr_comment_name, diff)
+    else:
+        text = '[%s] No significant performance differences' % args.pr_comment_name
+    if note:
+        text = note + '\n\n' + text
+    print('%s' % text)
+    comment_on_pr.comment_on_pr('```\n%s\n```' % text)
 
 
 if __name__ == '__main__':
-  args = _args()
-  main(args)
+    args = _args()
+    main(args)

tools/profiling/microbenchmarks/bm_diff/bm_run.py

@@ -13,7 +13,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """ Python utility to run opt and counters benchmarks and save json output """
 
 import bm_constants
@@ -27,93 +26,96 @@ import sys
 import os
 
 sys.path.append(
-  os.path.join(
-    os.path.dirname(sys.argv[0]), '..', '..', '..', 'run_tests',
-    'python_utils'))
+    os.path.join(
+        os.path.dirname(sys.argv[0]), '..', '..', '..', 'run_tests',
+        'python_utils'))
 import jobset
 
 
 def _args():
-  argp = argparse.ArgumentParser(description='Runs microbenchmarks')
-  argp.add_argument(
-    '-b',
-    '--benchmarks',
-    nargs='+',
-    choices=bm_constants._AVAILABLE_BENCHMARK_TESTS,
-    default=bm_constants._AVAILABLE_BENCHMARK_TESTS,
-    help='Benchmarks to run')
-  argp.add_argument(
-    '-j',
-    '--jobs',
-    type=int,
-    default=multiprocessing.cpu_count(),
-    help='Number of CPUs to use')
-  argp.add_argument(
-    '-n',
-    '--name',
-    type=str,
-    help='Unique name of the build to run. Needs to match the handle passed to bm_build.py'
-  )
-  argp.add_argument(
-    '-r',
-    '--regex',
-    type=str,
-    default="",
-    help='Regex to filter benchmarks run')
-  argp.add_argument(
-    '-l',
-    '--loops',
-    type=int,
-    default=20,
-    help='Number of times to loops the benchmarks. More loops cuts down on noise'
-  )
-  argp.add_argument('--counters', dest='counters', action='store_true')
-  argp.add_argument('--no-counters', dest='counters', action='store_false')
-  argp.set_defaults(counters=True)
-  args = argp.parse_args()
-  assert args.name
-  if args.loops < 3:
-    print "WARNING: This run will likely be noisy. Increase loops to at least 3."
-  return args
+    argp = argparse.ArgumentParser(description='Runs microbenchmarks')
+    argp.add_argument(
+        '-b',
+        '--benchmarks',
+        nargs='+',
+        choices=bm_constants._AVAILABLE_BENCHMARK_TESTS,
+        default=bm_constants._AVAILABLE_BENCHMARK_TESTS,
+        help='Benchmarks to run')
+    argp.add_argument(
+        '-j',
+        '--jobs',
+        type=int,
+        default=multiprocessing.cpu_count(),
+        help='Number of CPUs to use')
+    argp.add_argument(
+        '-n',
+        '--name',
+        type=str,
+        help='Unique name of the build to run. Needs to match the handle passed to bm_build.py'
+    )
+    argp.add_argument(
+        '-r',
+        '--regex',
+        type=str,
+        default="",
+        help='Regex to filter benchmarks run')
+    argp.add_argument(
+        '-l',
+        '--loops',
+        type=int,
+        default=20,
+        help='Number of times to loops the benchmarks. More loops cuts down on noise'
+    )
+    argp.add_argument('--counters', dest='counters', action='store_true')
+    argp.add_argument('--no-counters', dest='counters', action='store_false')
+    argp.set_defaults(counters=True)
+    args = argp.parse_args()
+    assert args.name
+    if args.loops < 3:
+        print "WARNING: This run will likely be noisy. Increase loops to at least 3."
+    return args
 
 
 def _collect_bm_data(bm, cfg, name, regex, idx, loops):
-  jobs_list = []
-  for line in subprocess.check_output(
-    ['bm_diff_%s/%s/%s' % (name, cfg, bm),
-     '--benchmark_list_tests', '--benchmark_filter=%s' % regex]).splitlines():
-    stripped_line = line.strip().replace("/", "_").replace(
-      "<", "_").replace(">", "_").replace(", ", "_")
-    cmd = [
-      'bm_diff_%s/%s/%s' % (name, cfg, bm), '--benchmark_filter=^%s$' %
-      line, '--benchmark_out=%s.%s.%s.%s.%d.json' %
-      (bm, stripped_line, cfg, name, idx), '--benchmark_out_format=json',
-    ]
-    jobs_list.append(
-      jobset.JobSpec(
-        cmd,
-        shortname='%s %s %s %s %d/%d' % (bm, line, cfg, name, idx + 1,
-                         loops),
-        verbose_success=True,
-        cpu_cost=2,
-        timeout_seconds=60 * 60)) # one hour
-  return jobs_list
+    jobs_list = []
+    for line in subprocess.check_output([
+            'bm_diff_%s/%s/%s' % (name, cfg, bm), '--benchmark_list_tests',
+            '--benchmark_filter=%s' % regex
+    ]).splitlines():
+        stripped_line = line.strip().replace("/", "_").replace(
+            "<", "_").replace(">", "_").replace(", ", "_")
+        cmd = [
+            'bm_diff_%s/%s/%s' % (name, cfg, bm),
+            '--benchmark_filter=^%s$' % line,
+            '--benchmark_out=%s.%s.%s.%s.%d.json' %
+            (bm, stripped_line, cfg, name, idx),
+            '--benchmark_out_format=json',
+        ]
+        jobs_list.append(
+            jobset.JobSpec(
+                cmd,
+                shortname='%s %s %s %s %d/%d' % (bm, line, cfg, name, idx + 1,
+                                                 loops),
+                verbose_success=True,
+                cpu_cost=2,
+                timeout_seconds=60 * 60))  # one hour
+    return jobs_list
 
 
 def create_jobs(name, benchmarks, loops, regex, counters):
-  jobs_list = []
-  for loop in range(0, loops):
-    for bm in benchmarks:
-      jobs_list += _collect_bm_data(bm, 'opt', name, regex, loop, loops)
-      if counters:
-        jobs_list += _collect_bm_data(bm, 'counters', name, regex, loop,
-                        loops)
-  random.shuffle(jobs_list, random.SystemRandom().random)
-  return jobs_list
+    jobs_list = []
+    for loop in range(0, loops):
+        for bm in benchmarks:
+            jobs_list += _collect_bm_data(bm, 'opt', name, regex, loop, loops)
+            if counters:
+                jobs_list += _collect_bm_data(bm, 'counters', name, regex, loop,
+                                              loops)
+    random.shuffle(jobs_list, random.SystemRandom().random)
+    return jobs_list
 
 
 if __name__ == '__main__':
-  args = _args()
-  jobs_list = create_jobs(args.name, args.benchmarks, args.loops, 
-                          args.regex, args.counters)
-  jobset.run(jobs_list, maxjobs=args.jobs)
+    args = _args()
+    jobs_list = create_jobs(args.name, args.benchmarks, args.loops, args.regex,
+                            args.counters)
+    jobset.run(jobs_list, maxjobs=args.jobs)

tools/profiling/microbenchmarks/bm_diff/bm_speedup.py

@@ -19,40 +19,41 @@ import math
 
 _DEFAULT_THRESHOLD = 1e-10
 
+
 def scale(a, mul):
-  return [x * mul for x in a]
+    return [x * mul for x in a]
 
 
 def cmp(a, b):
-  return stats.ttest_ind(a, b)
-
-
-def speedup(new, old, threshold = _DEFAULT_THRESHOLD):
-  if (len(set(new))) == 1 and new == old: return 0
-  s0, p0 = cmp(new, old)
-  if math.isnan(p0): return 0
-  if s0 == 0: return 0
-  if p0 > threshold: return 0
-  if s0 < 0:
-    pct = 1
-    while pct < 100:
-      sp, pp = cmp(new, scale(old, 1 - pct / 100.0))
-      if sp > 0: break
-      if pp > threshold: break
-      pct += 1
-    return -(pct - 1)
-  else:
-    pct = 1
-    while pct < 10000:
-      sp, pp = cmp(new, scale(old, 1 + pct / 100.0))
-      if sp < 0: break
-      if pp > threshold: break
-      pct += 1
-    return pct - 1
+    return stats.ttest_ind(a, b)
+
+
+def speedup(new, old, threshold=_DEFAULT_THRESHOLD):
+    if (len(set(new))) == 1 and new == old: return 0
+    s0, p0 = cmp(new, old)
+    if math.isnan(p0): return 0
+    if s0 == 0: return 0
+    if p0 > threshold: return 0
+    if s0 < 0:
+        pct = 1
+        while pct < 100:
+            sp, pp = cmp(new, scale(old, 1 - pct / 100.0))
+            if sp > 0: break
+            if pp > threshold: break
+            pct += 1
+        return -(pct - 1)
+    else:
+        pct = 1
+        while pct < 10000:
+            sp, pp = cmp(new, scale(old, 1 + pct / 100.0))
+            if sp < 0: break
+            if pp > threshold: break
+            pct += 1
+        return pct - 1
 
 
 if __name__ == "__main__":
-  new = [0.0, 0.0, 0.0, 0.0] 
-  old = [2.96608e-06, 3.35076e-06, 3.45384e-06, 3.34407e-06]
-  print speedup(new, old, 1e-5)
-  print speedup(old, new, 1e-5)
+    new = [0.0, 0.0, 0.0, 0.0]
+    old = [2.96608e-06, 3.35076e-06, 3.45384e-06, 3.34407e-06]
+    print speedup(new, old, 1e-5)
+    print speedup(old, new, 1e-5)

tools/profiling/microbenchmarks/bm_json.py

@@ -15,187 +15,196 @@
 import os
 
 _BM_SPECS = {
-  'BM_UnaryPingPong': {
-    'tpl': ['fixture', 'client_mutator', 'server_mutator'],
-    'dyn': ['request_size', 'response_size'],
-  },
-  'BM_PumpStreamClientToServer': {
-    'tpl': ['fixture'],
-    'dyn': ['request_size'],
-  },
-  'BM_PumpStreamServerToClient': {
-    'tpl': ['fixture'],
-    'dyn': ['request_size'],
-  },
-  'BM_StreamingPingPong': {
-    'tpl': ['fixture', 'client_mutator', 'server_mutator'],
-    'dyn': ['request_size', 'request_count'],
-  },
-  'BM_StreamingPingPongMsgs': {
-    'tpl': ['fixture', 'client_mutator', 'server_mutator'],
-    'dyn': ['request_size'],
-  },
-  'BM_PumpStreamServerToClient_Trickle': {
-    'tpl': [],
-    'dyn': ['request_size', 'bandwidth_kilobits'],
-  },
-  'BM_PumpUnbalancedUnary_Trickle': {
-    'tpl': [],
-    'dyn': ['cli_req_size', 'svr_req_size', 'bandwidth_kilobits'],
-  },
-  'BM_ErrorStringOnNewError': {
-    'tpl': ['fixture'],
-    'dyn': [],
-  },
-  'BM_ErrorStringRepeatedly': {
-    'tpl': ['fixture'],
-    'dyn': [],
-  },
-  'BM_ErrorGetStatus': {
-    'tpl': ['fixture'],
-    'dyn': [],
-  },
-  'BM_ErrorGetStatusCode': {
-    'tpl': ['fixture'],
-    'dyn': [],
-  },
-  'BM_ErrorHttpError': {
-    'tpl': ['fixture'],
-    'dyn': [],
-  },
-  'BM_HasClearGrpcStatus': {
-    'tpl': ['fixture'],
-    'dyn': [],
-  },
-  'BM_IsolatedFilter': {
-    'tpl': ['fixture', 'client_mutator'],
-    'dyn': [],
-  },
-  'BM_HpackEncoderEncodeHeader': {
-    'tpl': ['fixture'],
-    'dyn': ['end_of_stream', 'request_size'],
-  },
-  'BM_HpackParserParseHeader': {
-    'tpl': ['fixture', 'on_header'],
-    'dyn': [],
-  },
-  'BM_CallCreateDestroy': {
-    'tpl': ['fixture'],
-    'dyn': [],
-  },
-  'BM_Zalloc': {
-    'tpl': [],
-    'dyn': ['request_size'],
-  },
-  'BM_PollEmptyPollset_SpeedOfLight': {
-    'tpl': [],
-    'dyn': ['request_size', 'request_count'],
-  },
-  'BM_StreamCreateSendInitialMetadataDestroy': {
-    'tpl': ['fixture'],
-    'dyn': [],
-  },
-  'BM_TransportStreamSend': {
-    'tpl': [],
-    'dyn': ['request_size'],
-  },
-  'BM_TransportStreamRecv': {
-    'tpl': [],
-    'dyn': ['request_size'],
-  },
-  'BM_StreamingPingPongWithCoalescingApi': {
-    'tpl': ['fixture', 'client_mutator', 'server_mutator'],
-    'dyn': ['request_size', 'request_count', 'end_of_stream'],
-  },
-  'BM_Base16SomeStuff': {
-    'tpl': [],
-    'dyn': ['request_size'],
-  }
+    'BM_UnaryPingPong': {
+        'tpl': ['fixture', 'client_mutator', 'server_mutator'],
+        'dyn': ['request_size', 'response_size'],
+    },
+    'BM_PumpStreamClientToServer': {
+        'tpl': ['fixture'],
+        'dyn': ['request_size'],
+    },
+    'BM_PumpStreamServerToClient': {
+        'tpl': ['fixture'],
+        'dyn': ['request_size'],
+    },
+    'BM_StreamingPingPong': {
+        'tpl': ['fixture', 'client_mutator', 'server_mutator'],
+        'dyn': ['request_size', 'request_count'],
+    },
+    'BM_StreamingPingPongMsgs': {
+        'tpl': ['fixture', 'client_mutator', 'server_mutator'],
+        'dyn': ['request_size'],
+    },
+    'BM_PumpStreamServerToClient_Trickle': {
+        'tpl': [],
+        'dyn': ['request_size', 'bandwidth_kilobits'],
+    },
+    'BM_PumpUnbalancedUnary_Trickle': {
+        'tpl': [],
+        'dyn': ['cli_req_size', 'svr_req_size', 'bandwidth_kilobits'],
+    },
+    'BM_ErrorStringOnNewError': {
+        'tpl': ['fixture'],
+        'dyn': [],
+    },
+    'BM_ErrorStringRepeatedly': {
+        'tpl': ['fixture'],
+        'dyn': [],
+    },
+    'BM_ErrorGetStatus': {
+        'tpl': ['fixture'],
+        'dyn': [],
+    },
+    'BM_ErrorGetStatusCode': {
+        'tpl': ['fixture'],
+        'dyn': [],
+    },
+    'BM_ErrorHttpError': {
+        'tpl': ['fixture'],
+        'dyn': [],
+    },
+    'BM_HasClearGrpcStatus': {
+        'tpl': ['fixture'],
+        'dyn': [],
+    },
+    'BM_IsolatedFilter': {
+        'tpl': ['fixture', 'client_mutator'],
+        'dyn': [],
+    },
+    'BM_HpackEncoderEncodeHeader': {
+        'tpl': ['fixture'],
+        'dyn': ['end_of_stream', 'request_size'],
+    },
+    'BM_HpackParserParseHeader': {
+        'tpl': ['fixture', 'on_header'],
+        'dyn': [],
+    },
+    'BM_CallCreateDestroy': {
+        'tpl': ['fixture'],
+        'dyn': [],
+    },
+    'BM_Zalloc': {
+        'tpl': [],
+        'dyn': ['request_size'],
+    },
+    'BM_PollEmptyPollset_SpeedOfLight': {
+        'tpl': [],
+        'dyn': ['request_size', 'request_count'],
+    },
+    'BM_StreamCreateSendInitialMetadataDestroy': {
+        'tpl': ['fixture'],
+        'dyn': [],
+    },
+    'BM_TransportStreamSend': {
+        'tpl': [],
+        'dyn': ['request_size'],
+    },
+    'BM_TransportStreamRecv': {
+        'tpl': [],
+        'dyn': ['request_size'],
+    },
+    'BM_StreamingPingPongWithCoalescingApi': {
+        'tpl': ['fixture', 'client_mutator', 'server_mutator'],
+        'dyn': ['request_size', 'request_count', 'end_of_stream'],
+    },
+    'BM_Base16SomeStuff': {
+        'tpl': [],
+        'dyn': ['request_size'],
+    }
 }
 
+
 def numericalize(s):
-  if not s: return ''
-  if s[-1] == 'k':
-    return float(s[:-1]) * 1024
-  if s[-1] == 'M':
-    return float(s[:-1]) * 1024 * 1024
-  if 0 <= (ord(s[-1]) - ord('0')) <= 9:
-    return float(s)
-  assert 'not a number: %s' % s
+    if not s: return ''
+    if s[-1] == 'k':
+        return float(s[:-1]) * 1024
+    if s[-1] == 'M':
+        return float(s[:-1]) * 1024 * 1024
+    if 0 <= (ord(s[-1]) - ord('0')) <= 9:
+        return float(s)
+    assert 'not a number: %s' % s
+
 
 def parse_name(name):
-  cpp_name = name
-  if '<' not in name and '/' not in name and name not in _BM_SPECS:
-    return {'name': name, 'cpp_name': name}
-  rest = name
-  out = {}
-  tpl_args = []
-  dyn_args = []
-  if '<' in rest:
-    tpl_bit = rest[rest.find('<') + 1 : rest.rfind('>')]
-    arg = ''
-    nesting = 0
-    for c in tpl_bit:
-      if c == '<':
-        nesting += 1
-        arg += c
-      elif c == '>':
-        nesting -= 1
-        arg += c
-      elif c == ',':
-        if nesting == 0:
-          tpl_args.append(arg.strip())
-          arg = ''
-        else:
-          arg += c
-      else:
-        arg += c
-    tpl_args.append(arg.strip())
-    rest = rest[:rest.find('<')] + rest[rest.rfind('>') + 1:]
-  if '/' in rest:
-    s = rest.split('/')
-    rest = s[0]
-    dyn_args = s[1:]
-  name = rest
-  print (name)
-  print (dyn_args, _BM_SPECS[name]['dyn'])
-  print (tpl_args, _BM_SPECS[name]['tpl'])
-  assert name in _BM_SPECS, '_BM_SPECS needs to be expanded for %s' % name
-  assert len(dyn_args) == len(_BM_SPECS[name]['dyn'])
-  assert len(tpl_args) == len(_BM_SPECS[name]['tpl'])
-  out['name'] = name
-  out['cpp_name'] = cpp_name
-  out.update(dict((k, numericalize(v)) for k, v in zip(_BM_SPECS[name]['dyn'], dyn_args)))
-  out.update(dict(zip(_BM_SPECS[name]['tpl'], tpl_args)))
-  return out
+    cpp_name = name
+    if '<' not in name and '/' not in name and name not in _BM_SPECS:
+        return {'name': name, 'cpp_name': name}
+    rest = name
+    out = {}
+    tpl_args = []
+    dyn_args = []
+    if '<' in rest:
+        tpl_bit = rest[rest.find('<') + 1:rest.rfind('>')]
+        arg = ''
+        nesting = 0
+        for c in tpl_bit:
+            if c == '<':
+                nesting += 1
+                arg += c
+            elif c == '>':
+                nesting -= 1
+                arg += c
+            elif c == ',':
+                if nesting == 0:
+                    tpl_args.append(arg.strip())
+                    arg = ''
+                else:
+                    arg += c
+            else:
+                arg += c
+        tpl_args.append(arg.strip())
+        rest = rest[:rest.find('<')] + rest[rest.rfind('>') + 1:]
+    if '/' in rest:
+        s = rest.split('/')
+        rest = s[0]
+        dyn_args = s[1:]
+    name = rest
+    print(name)
+    print(dyn_args, _BM_SPECS[name]['dyn'])
+    print(tpl_args, _BM_SPECS[name]['tpl'])
+    assert name in _BM_SPECS, '_BM_SPECS needs to be expanded for %s' % name
+    assert len(dyn_args) == len(_BM_SPECS[name]['dyn'])
+    assert len(tpl_args) == len(_BM_SPECS[name]['tpl'])
+    out['name'] = name
+    out['cpp_name'] = cpp_name
+    out.update(
+        dict((k, numericalize(v))
+             for k, v in zip(_BM_SPECS[name]['dyn'], dyn_args)))
+    out.update(dict(zip(_BM_SPECS[name]['tpl'], tpl_args)))
+    return out
 
-def expand_json(js, js2 = None):
-  if not js and not js2: raise StopIteration()
-  if not js: js = js2
-  for bm in js['benchmarks']:
-    if bm['name'].endswith('_stddev') or bm['name'].endswith('_mean'): continue
-    context = js['context']
-    if 'label' in bm:
-      labels_list = [s.split(':') for s in bm['label'].strip().split(' ') if len(s) and s[0] != '#']
-      for el in labels_list:
-        el[0] = el[0].replace('/iter', '_per_iteration')
-      labels = dict(labels_list)
-    else:
-      labels = {}
-    row = {
-      'jenkins_build': os.environ.get('BUILD_NUMBER', ''),
-      'jenkins_job': os.environ.get('JOB_NAME', ''),
-    }
-    row.update(context)
-    row.update(bm)
-    row.update(parse_name(row['name']))
-    row.update(labels)
-    if js2:
-      for bm2 in js2['benchmarks']:
-        if bm['name'] == bm2['name'] and 'already_used' not in bm2:
-          row['cpu_time'] = bm2['cpu_time']
-          row['real_time'] = bm2['real_time']
-          row['iterations'] = bm2['iterations']
-          bm2['already_used'] = True
-          break
-    yield row
+
+def expand_json(js, js2=None):
+    if not js and not js2: raise StopIteration()
+    if not js: js = js2
+    for bm in js['benchmarks']:
+        if bm['name'].endswith('_stddev') or bm['name'].endswith('_mean'):
+            continue
+        context = js['context']
+        if 'label' in bm:
+            labels_list = [
+                s.split(':') for s in bm['label'].strip().split(' ')
+                if len(s) and s[0] != '#'
+            ]
+            for el in labels_list:
+                el[0] = el[0].replace('/iter', '_per_iteration')
+            labels = dict(labels_list)
+        else:
+            labels = {}
+        row = {
+            'jenkins_build': os.environ.get('BUILD_NUMBER', ''),
+            'jenkins_job': os.environ.get('JOB_NAME', ''),
+        }
+        row.update(context)
+        row.update(bm)
+        row.update(parse_name(row['name']))
+        row.update(labels)
+        if js2:
+            for bm2 in js2['benchmarks']:
+                if bm['name'] == bm2['name'] and 'already_used' not in bm2:
+                    row['cpu_time'] = bm2['cpu_time']
+                    row['real_time'] = bm2['real_time']
+                    row['iterations'] = bm2['iterations']
+                    bm2['already_used'] = True
+                    break
+        yield row

tools/profiling/qps/qps_diff.py

@@ -26,144 +26,146 @@ import sys
 import tabulate
 
 sys.path.append(
-  os.path.join(
-    os.path.dirname(sys.argv[0]), '..', 'microbenchmarks', 'bm_diff'))
+    os.path.join(
+        os.path.dirname(sys.argv[0]), '..', 'microbenchmarks', 'bm_diff'))
 import bm_speedup
 
 sys.path.append(
-  os.path.join(
-    os.path.dirname(sys.argv[0]), '..', '..', 'run_tests', 'python_utils'))
+    os.path.join(
+        os.path.dirname(sys.argv[0]), '..', '..', 'run_tests', 'python_utils'))
 import comment_on_pr
 
 
 def _args():
-  argp = argparse.ArgumentParser(
-    description='Perform diff on QPS Driver')
-  argp.add_argument(
-    '-d',
-    '--diff_base',
-    type=str,
-    help='Commit or branch to compare the current one to')
-  argp.add_argument(
-    '-l',
-    '--loops',
-    type=int,
-    default=4,
-    help='Number of loops for each benchmark. More loops cuts down on noise'
-  )
-  argp.add_argument(
-    '-j',
-    '--jobs',
-    type=int,
-    default=multiprocessing.cpu_count(),
-    help='Number of CPUs to use')
-  args = argp.parse_args()
-  assert args.diff_base, "diff_base must be set"
-  return args
+    argp = argparse.ArgumentParser(description='Perform diff on QPS Driver')
+    argp.add_argument(
+        '-d',
+        '--diff_base',
+        type=str,
+        help='Commit or branch to compare the current one to')
+    argp.add_argument(
+        '-l',
+        '--loops',
+        type=int,
+        default=4,
+        help='Number of loops for each benchmark. More loops cuts down on noise')
+    argp.add_argument(
+        '-j',
+        '--jobs',
+        type=int,
+        default=multiprocessing.cpu_count(),
+        help='Number of CPUs to use')
+    args = argp.parse_args()
+    assert args.diff_base, "diff_base must be set"
+    return args
 
 
 def _make_cmd(jobs):
-  return ['make', '-j', '%d' % jobs, 'qps_json_driver', 'qps_worker']
+    return ['make', '-j', '%d' % jobs, 'qps_json_driver', 'qps_worker']
 
 
 def build(name, jobs):
-  shutil.rmtree('qps_diff_%s' % name, ignore_errors=True)
-  subprocess.check_call(['git', 'submodule', 'update'])
-  try:
-    subprocess.check_call(_make_cmd(jobs))
-  except subprocess.CalledProcessError, e:
-    subprocess.check_call(['make', 'clean'])
-    subprocess.check_call(_make_cmd(jobs))
-  os.rename('bins', 'qps_diff_%s' % name)
+    shutil.rmtree('qps_diff_%s' % name, ignore_errors=True)
+    subprocess.check_call(['git', 'submodule', 'update'])
+    try:
+        subprocess.check_call(_make_cmd(jobs))
+    except subprocess.CalledProcessError, e:
+        subprocess.check_call(['make', 'clean'])
+        subprocess.check_call(_make_cmd(jobs))
+    os.rename('bins', 'qps_diff_%s' % name)
 
 
 def _run_cmd(name, scenario, fname):
-  return ['qps_diff_%s/opt/qps_json_driver' % name, '--scenarios_json', scenario, '--json_file_out', fname]
+    return [
+        'qps_diff_%s/opt/qps_json_driver' % name, '--scenarios_json', scenario,
+        '--json_file_out', fname
+    ]
 
 
 def run(name, scenarios, loops):
-  for sn in scenarios:
-    for i in range(0, loops):
-      fname = "%s.%s.%d.json" % (sn, name, i)
-      subprocess.check_call(_run_cmd(name, scenarios[sn], fname))
+    for sn in scenarios:
+        for i in range(0, loops):
+            fname = "%s.%s.%d.json" % (sn, name, i)
+            subprocess.check_call(_run_cmd(name, scenarios[sn], fname))
 
 
 def _load_qps(fname):
-  try:
-    with open(fname) as f:
-      return json.loads(f.read())['qps']
-  except IOError, e:
-    print("IOError occurred reading file: %s" % fname)
-    return None
-  except ValueError, e:
-    print("ValueError occurred reading file: %s" % fname)
-    return None
+    try:
+        with open(fname) as f:
+            return json.loads(f.read())['qps']
+    except IOError, e:
+        print("IOError occurred reading file: %s" % fname)
+        return None
+    except ValueError, e:
+        print("ValueError occurred reading file: %s" % fname)
+        return None
 
 
 def _median(ary):
-  assert (len(ary))
-  ary = sorted(ary)
-  n = len(ary)
-  if n % 2 == 0:
-    return (ary[(n - 1) / 2] + ary[(n - 1) / 2 + 1]) / 2.0
-  else:
-    return ary[n / 2]
+    assert (len(ary))
+    ary = sorted(ary)
+    n = len(ary)
+    if n % 2 == 0:
+        return (ary[(n - 1) / 2] + ary[(n - 1) / 2 + 1]) / 2.0
+    else:
+        return ary[n / 2]
 
 
 def diff(scenarios, loops, old, new):
-  old_data = {}
-  new_data = {}
-
-  # collect data
-  for sn in scenarios:
-    old_data[sn] = []
-    new_data[sn] = []
-    for i in range(loops):
-      old_data[sn].append(_load_qps("%s.%s.%d.json" % (sn, old, i)))
-      new_data[sn].append(_load_qps("%s.%s.%d.json" % (sn, new, i)))
-
-  # crunch data
-  headers = ['Benchmark', 'qps']
-  rows = []
-  for sn in scenarios:
-    mdn_diff = abs(_median(new_data[sn]) - _median(old_data[sn]))
-    print('%s: %s=%r %s=%r mdn_diff=%r' % (sn, new, new_data[sn], old, old_data[sn], mdn_diff))
-    s = bm_speedup.speedup(new_data[sn], old_data[sn], 10e-5)
-    if abs(s) > 3 and mdn_diff > 0.5:
-      rows.append([sn, '%+d%%' % s])
-
-  if rows:
-    return tabulate.tabulate(rows, headers=headers, floatfmt='+.2f')
-  else:
-    return None
+    old_data = {}
+    new_data = {}
+
+    # collect data
+    for sn in scenarios:
+        old_data[sn] = []
+        new_data[sn] = []
+        for i in range(loops):
+            old_data[sn].append(_load_qps("%s.%s.%d.json" % (sn, old, i)))
+            new_data[sn].append(_load_qps("%s.%s.%d.json" % (sn, new, i)))
+
+    # crunch data
+    headers = ['Benchmark', 'qps']
+    rows = []
+    for sn in scenarios:
+        mdn_diff = abs(_median(new_data[sn]) - _median(old_data[sn]))
+        print('%s: %s=%r %s=%r mdn_diff=%r' %
+              (sn, new, new_data[sn], old, old_data[sn], mdn_diff))
+        s = bm_speedup.speedup(new_data[sn], old_data[sn], 10e-5)
+        if abs(s) > 3 and mdn_diff > 0.5:
+            rows.append([sn, '%+d%%' % s])
+
+    if rows:
+        return tabulate.tabulate(rows, headers=headers, floatfmt='+.2f')
+    else:
+        return None
 
 
 def main(args):
-  build('new', args.jobs)
+    build('new', args.jobs)
 
-  if args.diff_base:
-    where_am_i = subprocess.check_output(
-      ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip()
-    subprocess.check_call(['git', 'checkout', args.diff_base])
-    try:
-      build('old', args.jobs)
-    finally:
-      subprocess.check_call(['git', 'checkout', where_am_i])
-      subprocess.check_call(['git', 'submodule', 'update'])
+    if args.diff_base:
+        where_am_i = subprocess.check_output(
+            ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip()
+        subprocess.check_call(['git', 'checkout', args.diff_base])
+        try:
+            build('old', args.jobs)
+        finally:
+            subprocess.check_call(['git', 'checkout', where_am_i])
+            subprocess.check_call(['git', 'submodule', 'update'])
 
-  run('new', qps_scenarios._SCENARIOS, args.loops)
-  run('old', qps_scenarios._SCENARIOS, args.loops)
+    run('new', qps_scenarios._SCENARIOS, args.loops)
+    run('old', qps_scenarios._SCENARIOS, args.loops)
 
-  diff_output = diff(qps_scenarios._SCENARIOS, args.loops, 'old', 'new')
+    diff_output = diff(qps_scenarios._SCENARIOS, args.loops, 'old', 'new')
 
-  if diff_output:
-    text = '[qps] Performance differences noted:\n%s' % diff_output
-  else:
-    text = '[qps] No significant performance differences'
-  print('%s' % text)
-  comment_on_pr.comment_on_pr('```\n%s\n```' % text)
+    if diff_output:
+        text = '[qps] Performance differences noted:\n%s' % diff_output
+    else:
+        text = '[qps] No significant performance differences'
+    print('%s' % text)
+    comment_on_pr.comment_on_pr('```\n%s\n```' % text)
 
 
 if __name__ == '__main__':
-  args = _args()
-  main(args)
+    args = _args()
+    main(args)

tools/profiling/qps/qps_scenarios.py

@@ -14,6 +14,8 @@
 """ QPS Scenarios to run """
 
 _SCENARIOS = {
-  'large-message-throughput': '{"scenarios":[{"name":"large-message-throughput", "spawn_local_worker_count": -2, "warmup_seconds": 30, "benchmark_seconds": 270, "num_servers": 1, "server_config": {"async_server_threads": 1, "security_params": null, "server_type": "ASYNC_SERVER"}, "num_clients": 1, "client_config": {"client_type": "ASYNC_CLIENT", "security_params": null, "payload_config": {"simple_params": {"resp_size": 1048576, "req_size": 1048576}}, "client_channels": 1, "async_client_threads": 1, "outstanding_rpcs_per_channel": 1, "rpc_type": "UNARY", "load_params": {"closed_loop": {}}, "histogram_params": {"max_possible": 60000000000.0, "resolution": 0.01}}}]}',
-  'multi-channel-64-KiB': '{"scenarios":[{"name":"multi-channel-64-KiB", "spawn_local_worker_count": -3, "warmup_seconds": 30, "benchmark_seconds": 270, "num_servers": 1, "server_config": {"async_server_threads": 31, "security_params": null, "server_type": "ASYNC_SERVER"}, "num_clients": 2, "client_config": {"client_type": "ASYNC_CLIENT", "security_params": null, "payload_config": {"simple_params": {"resp_size": 65536, "req_size": 65536}}, "client_channels": 32, "async_client_threads": 31, "outstanding_rpcs_per_channel": 100, "rpc_type": "UNARY", "load_params": {"closed_loop": {}}, "histogram_params": {"max_possible": 60000000000.0, "resolution": 0.01}}}]}'
+    'large-message-throughput':
+    '{"scenarios":[{"name":"large-message-throughput", "spawn_local_worker_count": -2, "warmup_seconds": 30, "benchmark_seconds": 270, "num_servers": 1, "server_config": {"async_server_threads": 1, "security_params": null, "server_type": "ASYNC_SERVER"}, "num_clients": 1, "client_config": {"client_type": "ASYNC_CLIENT", "security_params": null, "payload_config": {"simple_params": {"resp_size": 1048576, "req_size": 1048576}}, "client_channels": 1, "async_client_threads": 1, "outstanding_rpcs_per_channel": 1, "rpc_type": "UNARY", "load_params": {"closed_loop": {}}, "histogram_params": {"max_possible": 60000000000.0, "resolution": 0.01}}}]}',
+    'multi-channel-64-KiB':
+    '{"scenarios":[{"name":"multi-channel-64-KiB", "spawn_local_worker_count": -3, "warmup_seconds": 30, "benchmark_seconds": 270, "num_servers": 1, "server_config": {"async_server_threads": 31, "security_params": null, "server_type": "ASYNC_SERVER"}, "num_clients": 2, "client_config": {"client_type": "ASYNC_CLIENT", "security_params": null, "payload_config": {"simple_params": {"resp_size": 65536, "req_size": 65536}}, "client_channels": 32, "async_client_threads": 31, "outstanding_rpcs_per_channel": 100, "rpc_type": "UNARY", "load_params": {"closed_loop": {}}, "histogram_params": {"max_possible": 60000000000.0, "resolution": 0.01}}}]}'
 }