generate html report for interop tests

tools/run_tests/jobset.py

@@ -48,7 +48,7 @@ _DEFAULT_MAX_JOBS = 16 * multiprocessing.cpu_count()
 # setup a signal handler so that signal.pause registers 'something'
 # when a child finishes
 # not using futures and threading to avoid a dependency on subprocess32
-if platform.system() == "Windows":
+if platform.system() == 'Windows':
   pass
 else:
   have_alarm = False
@@ -118,8 +118,8 @@ def message(tag, msg, explanatory_text=None, do_newline=False):
   except:
     pass
 
-message.old_tag = ""
-message.old_msg = ""
+message.old_tag = ''
+message.old_msg = ''
 
 def which(filename):
   if '/' in filename:
@@ -177,6 +177,15 @@ class JobSpec(object):
     return self.identity() == other.identity()
 
 
+class JobResult(object):
+  def __init__(self):
+    self.state = 'UNKNOWN'
+    self.returncode = -1
+    self.elapsed_time = 0
+    self.retries = 0
+    self.message = ''
+    
+
 class Job(object):
   """Manages one job."""
 
@@ -192,8 +201,12 @@ class Job(object):
     self._timeout_retries = 0
     self._suppress_failure_message = False
     message('START', spec.shortname, do_newline=self._travis)
+    self.result = JobResult()
     self.start()
 
+  def GetSpec(self):
+    return self._spec
+
   def start(self):
     self._tempfile = tempfile.TemporaryFile()
     env = dict(os.environ)
@@ -219,6 +232,8 @@ class Job(object):
       # implemented efficiently. This is an experiment to workaround the issue by making sure
       # results.xml file is small enough.
       filtered_stdout = filtered_stdout[-128:]
+      self.result.message = filtered_stdout
+      self.result.elapsed_time = elapsed
       if self._xml_test is not None:
         self._xml_test.set('time', str(elapsed))
         ET.SubElement(self._xml_test, 'system-out').text = filtered_stdout
@@ -228,6 +243,7 @@ class Job(object):
             self._spec.shortname, self._process.returncode, self._process.pid),
             stdout, do_newline=True)
           self._retries += 1
+          self.result.retries = self._timeout_retries + self._retries
           self.start()
         else:
           self._state = _FAILURE
@@ -235,6 +251,8 @@ class Job(object):
             message('FAILED', '%s [ret=%d, pid=%d]' % (
                 self._spec.shortname, self._process.returncode, self._process.pid),
                 stdout, do_newline=True)
+          self.result.state = 'FAILED'
+          self.result.returncode = self._process.returncode
           if self._xml_test is not None:
             ET.SubElement(self._xml_test, 'failure', message='Failure')
       else:
@@ -242,20 +260,24 @@ class Job(object):
         message('PASSED', '%s [time=%.1fsec; retries=%d;%d]' % (
                     self._spec.shortname, elapsed, self._retries, self._timeout_retries),
             do_newline=self._newline_on_success or self._travis)
+        self.result.state = 'PASSED'
         if self._bin_hash:
           update_cache.finished(self._spec.identity(), self._bin_hash)
     elif self._state == _RUNNING and time.time() - self._start > self._spec.timeout_seconds:
       self._tempfile.seek(0)
       stdout = self._tempfile.read()
       filtered_stdout = _filter_stdout(stdout)
+      self.result.message = filtered_stdout
       if self._timeout_retries < self._spec.timeout_retries:
         message('TIMEOUT_FLAKE', self._spec.shortname, stdout, do_newline=True)
         self._timeout_retries += 1
+        self.result.retries = self._timeout_retries + self._retries
         self._process.terminate()
         self.start()
       else:
         message('TIMEOUT', self._spec.shortname, stdout, do_newline=True)
         self.kill()
+        self.result.state = 'TIMEOUT'
         if self._xml_test is not None:
           ET.SubElement(self._xml_test, 'system-out').text = filtered_stdout
           ET.SubElement(self._xml_test, 'error', message='Timeout')
@@ -290,6 +312,10 @@ class Jobset(object):
     self._hashes = {}
     self._xml_report = xml_report
     self._add_env = add_env
+    self.resultset = {}
+    
+  def get_num_failures(self):
+    return self._failures  
 
   def start(self, spec):
     """Start a job. Return True on success, False on failure."""
@@ -312,12 +338,14 @@ class Jobset(object):
       bin_hash = None
       should_run = True
     if should_run:
-      self._running.add(Job(spec,
-                            bin_hash,
-                            self._newline_on_success,
-                            self._travis,
-                            self._add_env,
-                            self._xml_report))
+      job = Job(spec,
+                bin_hash,
+                self._newline_on_success,
+                self._travis,
+                self._add_env,
+                self._xml_report)
+      self._running.add(job)
+      self.resultset[job.GetSpec().shortname] = None
     return True
 
   def reap(self):
@@ -337,6 +365,7 @@ class Jobset(object):
         break
       for job in dead:
         self._completed += 1
+        self.resultset[job.GetSpec().shortname] = job.result
         self._running.remove(job)
       if dead: return
       if (not self._travis):
@@ -398,4 +427,5 @@ def run(cmdlines,
   for cmdline in cmdlines:
     if not js.start(cmdline):
       break
-  return js.finish()
+  js.finish()  
+  return js.get_num_failures(), js.resultset

tools/run_tests/run_interop_tests.py

@@ -391,7 +391,7 @@ def cloud_to_prod_jobspec(language, test_case, docker_image=None, auth=False):
           cmdline=cmdline,
           cwd=cwd,
           environ=environ,
-          shortname="%s:%s:%s" % (suite_name, language, test_case),
+          shortname='%s:%s:%s' % (suite_name, language, test_case),
           timeout_seconds=2*60,
           flake_retries=5 if args.allow_flakes else 0,
           timeout_retries=2 if args.allow_flakes else 0,
@@ -423,7 +423,7 @@ def cloud_to_cloud_jobspec(language, test_case, server_name, server_host,
           cmdline=cmdline,
           cwd=cwd,
           environ=environ,
-          shortname="cloud_to_cloud:%s:%s_server:%s" % (language, server_name,
+          shortname='cloud_to_cloud:%s:%s_server:%s' % (language, server_name,
                                                  test_case),
           timeout_seconds=2*60,
           flake_retries=5 if args.allow_flakes else 0,
@@ -448,7 +448,7 @@ def server_jobspec(language, docker_image):
   server_job = jobset.JobSpec(
           cmdline=docker_cmdline,
           environ=environ,
-          shortname="interop_server_%s" % language,
+          shortname='interop_server_%s' % language,
           timeout_seconds=30*60)
   server_job.container_name = container_name
   return server_job
@@ -467,16 +467,132 @@ def build_interop_image_jobspec(language, tag=None):
   # TODO(stanleycheung): find a more elegant way to do this
   if language.safename == 'php' and os.path.exists('/var/local/.composer/auth.json'):
     env['BUILD_INTEROP_DOCKER_EXTRA_ARGS'] = \
-      "-v /var/local/.composer/auth.json:/root/.composer/auth.json:ro"
+      '-v /var/local/.composer/auth.json:/root/.composer/auth.json:ro'
   build_job = jobset.JobSpec(
           cmdline=['tools/jenkins/build_interop_image.sh'],
           environ=env,
-          shortname="build_docker_%s" % (language),
+          shortname='build_docker_%s' % (language),
           timeout_seconds=30*60)
   build_job.tag = tag
   return build_job
 
 
+# TODO(adelez): Use mako template.
+def fill_one_test_result(shortname, resultset, html_str):
+  if shortname in resultset:
+    result = resultset[shortname]
+    if result.state == 'PASSED':
+      html_str = '%s<td bgcolor=\"green\">PASS</td>\n' % html_str
+    else:
+      tooltip = ''
+      if result.returncode > 0 or result.message:
+        if result.returncode > 0:
+          tooltip = 'returncode: %d ' % result.returncode
+        if result.message:
+          tooltip = '%smessage: %s' % (tooltip, result.message)     
+      if result.state == 'FAILED':
+        html_str = '%s<td bgcolor=\"red\">' % html_str
+        if tooltip:  
+          html_str = ('%s<a href=\"#\" data-toggle=\"tooltip\" '
+                      'data-placement=\"auto\" title=\"%s\">FAIL</a></td>\n' % 
+                      (html_str, tooltip))
+        else:
+          html_str = '%sFAIL</td>\n' % html_str
+      elif result.state == 'TIMEOUT':
+        html_str = '%s<td bgcolor=\"yellow\">' % html_str
+        if tooltip:
+          html_str = ('%s<a href=\"#\" data-toggle=\"tooltip\" '
+                      'data-placement=\"auto\" title=\"%s\">TIMEOUT</a></td>\n' 
+                      % (html_str, tooltip))
+        else:
+          html_str = '%sTIMEOUT</td>\n' % html_str
+  else:
+    html_str = '%s<td bgcolor=\"magenta\">Not implemented</td>\n' % html_str
+  
+  return html_str
+
+
+def render_html_report(test_cases, client_langs, server_langs, resultset,
+                       num_failures):
+  """Generate html report."""
+  sorted_test_cases = sorted(test_cases)
+  sorted_client_langs = sorted(client_langs)
+  print sorted_client_langs
+  sorted_server_langs = sorted(server_langs)
+  html_str = ('<!DOCTYPE html>\n'
+              '<html lang=\"en\">\n'
+              '<head><title>Interop Test Result</title></head>\n'
+              '<body>\n')
+  if num_failures > 1:
+    html_str = (
+        '%s<p><h2><font color=\"red\">%d tests failed!</font></h2></p>\n' % 
+        (html_str, num_failures))
+  elif num_failures:
+    html_str = (
+        '%s<p><h2><font color=\"red\">%d test failed!</font></h2></p>\n' % 
+        (html_str, num_failures))
+  else:
+    html_str = (
+        '%s<p><h2><font color=\"green\">All tests passed!</font></h2></p>\n' % 
+        html_str)
+  if args.cloud_to_prod_auth or args.cloud_to_prod:
+    # Each column header is the client language.
+    html_str = ('%s<h2>Cloud to Prod</h2>\n' 
+                '<table style=\"width:100%%\" border=\"1\">\n'
+                '<tr bgcolor=\"#00BFFF\">\n'
+                '<th/>\n') % html_str
+    for client_lang in sorted_client_langs:
+      html_str = '%s<th>%s\n' % (html_str, client_lang)
+    html_str = '%s</tr>\n' % html_str
+    for test_case in sorted_test_cases:
+      html_str = '%s<tr><td><b>%s</b></td>\n' % (html_str, test_case)
+      for client_lang in sorted_client_langs:
+        if args.cloud_to_prod:
+          shortname = 'cloud_to_prod:%s:%s' % (client_lang, test_case)
+        else:
+          shortname = 'cloud_to_prod_auth:%s:%s' % (client_lang, test_case)
+        html_str = fill_one_test_result(shortname, resultset, html_str)       
+      html_str = '%s</tr>\n' % html_str 
+    html_str = '%s</table>\n' % html_str
+  if servers:
+    for test_case in sorted_test_cases:
+      # Each column header is the client language.
+      html_str = ('%s<h2>%s</h2>\n' 
+                  '<table style=\"width:100%%\" border=\"1\">\n'
+                  '<tr bgcolor=\"#00BFFF\">\n'
+                  '<th/>\n') % (html_str, test_case)
+      for client_lang in sorted_client_langs:
+        html_str = '%s<th>%s\n' % (html_str, client_lang)
+      html_str = '%s</tr>\n' % html_str
+      # Each row head is the server language.
+      for server_lang in sorted_server_langs:
+        html_str = '%s<tr><td><b>%s</b></td>\n' % (html_str, server_lang)
+        # Fill up the cells with test result.
+        for client_lang in sorted_client_langs:
+          shortname = 'cloud_to_cloud:%s:%s_server:%s' % (
+              client_lang, server_lang, test_case)
+          html_str = fill_one_test_result(shortname, resultset, html_str)
+        html_str = '%s</tr>\n' % html_str
+      html_str = '%s</table>\n' % html_str
+
+  html_str = ('%s\n'
+              '<script>\n'
+              '$(document).ready(function(){'
+              '$(\'[data-toggle=\"tooltip\"]\').tooltip();\n'   
+              '});\n'
+              '</script>\n'
+              '</body>\n'
+              '</html>') % html_str  
+  
+  # Write to reports/index.html as set up in Jenkins plugin.
+  html_report_dir = 'reports'
+  if not os.path.exists(html_report_dir):
+    os.mkdir(html_report_dir)
+  html_file_path = os.path.join(html_report_dir, 'index.html')
+  with open(html_file_path, 'w') as f:
+    f.write(html_str)
+
+
 argp = argparse.ArgumentParser(description='Run interop tests.')
 argp.add_argument('-l', '--language',
                   choices=['all'] + sorted(_LANGUAGES),
@@ -503,7 +619,7 @@ argp.add_argument('-s', '--server',
                   default=[])
 argp.add_argument('--override_server',
                   action='append',
-                  type=lambda kv: kv.split("="),
+                  type=lambda kv: kv.split('='),
                   help='Use servername=HOST:PORT to explicitly specify a server. E.g. csharp=localhost:50000',
                   default=[])
 argp.add_argument('-t', '--travis',
@@ -521,7 +637,7 @@ argp.add_argument('--allow_flakes',
                   default=False,
                   action='store_const',
                   const=True,
-                  help="Allow flaky tests to show as passing (re-runs failed tests up to five times)")
+                  help='Allow flaky tests to show as passing (re-runs failed tests up to five times)')
 args = argp.parse_args()
 
 servers = set(s for s in itertools.chain.from_iterable(_SERVERS
@@ -538,7 +654,7 @@ if args.use_docker:
     time.sleep(5)
 
 if not args.use_docker and servers:
-  print "Running interop servers is only supported with --use_docker option enabled."
+  print 'Running interop servers is only supported with --use_docker option enabled.'
   sys.exit(1)
 
 languages = set(_LANGUAGES[l]
@@ -560,10 +676,14 @@ if args.use_docker:
 
   if build_jobs:
     jobset.message('START', 'Building interop docker images.', do_newline=True)
-    if jobset.run(build_jobs, newline_on_success=True, maxjobs=args.jobs):
-      jobset.message('SUCCESS', 'All docker images built successfully.', do_newline=True)
+    num_failures, _ = jobset.run(
+        build_jobs, newline_on_success=True, maxjobs=args.jobs)
+    if num_failures == 0:
+      jobset.message('SUCCESS', 'All docker images built successfully.', 
+                     do_newline=True)
     else:
-      jobset.message('FAILED', 'Failed to build interop docker images.', do_newline=True)
+      jobset.message('FAILED', 'Failed to build interop docker images.', 
+                     do_newline=True)
       for image in docker_images.itervalues():
         dockerjob.remove_image(image, skip_nonexistent=True)
       exit(1);
@@ -614,7 +734,7 @@ try:
         jobs.append(test_job)
 
   if not jobs:
-    print "No jobs to run."
+    print 'No jobs to run.'
     for image in docker_images.itervalues():
       dockerjob.remove_image(image, skip_nonexistent=True)
     sys.exit(1)
@@ -622,13 +742,19 @@ try:
   root = ET.Element('testsuites')
   testsuite = ET.SubElement(root, 'testsuite', id='1', package='grpc', name='tests')
 
-  if jobset.run(jobs, newline_on_success=True, maxjobs=args.jobs, xml_report=testsuite):
-    jobset.message('SUCCESS', 'All tests passed', do_newline=True)
-  else:
+  num_failures, resultset = jobset.run(jobs, newline_on_success=True, 
+                                       maxjobs=args.jobs, xml_report=testsuite)
+  if num_failures:
     jobset.message('FAILED', 'Some tests failed', do_newline=True)
+  else:
+    jobset.message('SUCCESS', 'All tests passed', do_newline=True)
 
   tree = ET.ElementTree(root)
   tree.write('report.xml', encoding='UTF-8')
+  
+  # Generate HTML report.
+  render_html_report(_TEST_CASES, set([str(l) for l in languages]), servers, 
+                     resultset, num_failures)
 
 finally:
   # Check if servers are still running.

tools/run_tests/run_tests.py

@@ -161,7 +161,7 @@ class CLanguage(object):
       if os.path.isfile(binary):
         out.append(config.job_spec([binary], [binary]))
       else:
-        print "\nWARNING: binary not found, skipping", binary
+        print '\nWARNING: binary not found, skipping', binary
     return sorted(out)
 
   def make_targets(self):
@@ -516,7 +516,7 @@ def runs_per_test_type(arg_str):
         if n <= 0: raise ValueError
         return n
     except:
-        msg = "'{}' isn't a positive integer or 'inf'".format(arg_str)
+        msg = '\'{}\' is not a positive integer or \'inf\''.format(arg_str)
         raise argparse.ArgumentTypeError(msg)
 
 # parse command line
@@ -555,14 +555,14 @@ argp.add_argument('--use_docker',
                   default=False,
                   action='store_const',
                   const=True,
-                  help="Run all the tests under docker. That provides " +
-                  "additional isolation and prevents the need to install " +
-                  "language specific prerequisites. Only available on Linux.")
+                  help='Run all the tests under docker. That provides ' +
+                  'additional isolation and prevents the need to install ' +
+                  'language specific prerequisites. Only available on Linux.')
 argp.add_argument('--allow_flakes',
                   default=False,
                   action='store_const',
                   const=True,
-                  help="Allow flaky tests to show as passing (re-runs failed tests up to five times)")
+                  help='Allow flaky tests to show as passing (re-runs failed tests up to five times)')
 argp.add_argument('-a', '--antagonists', default=0, type=int)
 argp.add_argument('-x', '--xml_report', default=None, type=str,
         help='Generates a JUnit-compatible XML report')
@@ -578,7 +578,7 @@ if args.use_docker:
     time.sleep(5)
 
   child_argv = [ arg for arg in sys.argv if not arg == '--use_docker' ]
-  run_tests_cmd = 'tools/run_tests/run_tests.py %s' % " ".join(child_argv[1:])
+  run_tests_cmd = 'tools/run_tests/run_tests.py %s' % ' '.join(child_argv[1:])
 
   # TODO(jtattermusch): revisit if we need special handling for arch here
   # set arch command prefix in case we are working with different arch.
@@ -625,9 +625,9 @@ if platform.system() == 'Windows':
     # better do parallel compilation
     # empirically /m:2 gives the best performance/price and should prevent
     # overloading the windows workers.
-    extra_args.extend(["/m:2"])
+    extra_args.extend(['/m:2'])
     # disable PDB generation: it's broken, and we don't need it during CI
-    extra_args.extend(["/p:Jenkins=true"])
+    extra_args.extend(['/p:Jenkins=true'])
     return [
       jobset.JobSpec(['vsprojects\\build.bat',
                       'vsprojects\\%s.sln' % target,
@@ -802,8 +802,10 @@ def _build_and_run(
     check_cancelled, newline_on_success, travis, cache, xml_report=None):
   """Do one pass of building & running tests."""
   # build latest sequentially
-  if not jobset.run(build_steps, maxjobs=1, stop_on_failure=True,
-                    newline_on_success=newline_on_success, travis=travis):
+  num_failures, _ = jobset.run(
+      build_steps, maxjobs=1, stop_on_failure=True,
+      newline_on_success=newline_on_success, travis=travis)
+  if num_failures:
     return 1
 
   # start antagonists