Sync internal benchmark changes

benchmarks/util/run_and_upload.py → benchmarks/util/result_parser.py

@@ -1,47 +1,34 @@
-from __future__ import print_function
-from __future__ import absolute_import
-import argparse
-import os
-import re
-import copy
-import uuid
-import calendar
-import time
-from . import big_query_utils
-import datetime
-import json
 # This import depends on the automake rule protoc_middleman, please make sure
 # protoc_middleman has been built before run this file.
-import os.path, sys
+import json
+import re
+import os.path
+# BEGIN OPENSOURCE
+import sys
 sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir))
+# END OPENSOURCE
 import tmp.benchmarks_pb2 as benchmarks_pb2
-from click.types import STRING
-
-_PROJECT_ID = 'grpc-testing'
-_DATASET = 'protobuf_benchmark_result'
-_TABLE = 'opensource_result_v1'
-_NOW = "%d%02d%02d" % (datetime.datetime.now().year,
-                       datetime.datetime.now().month,
-                       datetime.datetime.now().day)
 
-file_size_map = {}
+__file_size_map = {}
 
-def get_data_size(file_name):
-  if file_name in file_size_map:
-    return file_size_map[file_name]
+def __get_data_size(filename):
+  if filename[0] != '/':
+    filename = os.path.dirname(os.path.abspath(__file__)) + "/../" + filename
+  if filename in __file_size_map:
+    return __file_size_map[filename]
   benchmark_dataset = benchmarks_pb2.BenchmarkDataset()
   benchmark_dataset.ParseFromString(
-      open(os.path.dirname(os.path.abspath(__file__)) + "/../" + file_name).read())
+      open(filename).read())
   size = 0
   count = 0
   for payload in benchmark_dataset.payload:
     size += len(payload)
     count += 1
-  file_size_map[file_name] = (size, 1.0 * size / count)
+  __file_size_map[filename] = (size, 1.0 * size / count)
   return size, 1.0 * size / count
 
 
-def extract_file_name(file_name):
+def __extract_file_name(file_name):
   name_list = re.split("[/\.]", file_name)
   short_file_name = ""
   for name in name_list:
@@ -50,15 +37,12 @@ def extract_file_name(file_name):
   return short_file_name
 
 
-cpp_result = []
-python_result = []
-java_result = []
-go_result = []
+__results = []
 
 
 # CPP results example:
-# [ 
-#   "benchmarks": [ 
+# [
+#   "benchmarks": [
 #     {
 #       "bytes_per_second": int,
 #       "cpu_time": int,
@@ -66,12 +50,11 @@ go_result = []
 #       "time_unit: string,
 #       ...
 #     },
-#     ... 
+#     ...
 #   ],
-#   ... 
+#   ...
 # ]
-def parse_cpp_result(filename):
-  global cpp_result
+def __parse_cpp_result(filename):
   if filename == "":
     return
   if filename[0] != '/':
@@ -82,31 +65,32 @@ def parse_cpp_result(filename):
       data_filename = "".join(
           re.split("(_parse_|_serialize)", benchmark["name"])[0])
       behavior = benchmark["name"][len(data_filename) + 1:]
-      cpp_result.append({
+      if data_filename[:2] == "BM":
+        data_filename = data_filename[3:]
+      __results.append({
         "language": "cpp",
-        "dataFileName": data_filename,
+        "dataFilename": data_filename,
         "behavior": behavior,
         "throughput": benchmark["bytes_per_second"] / 2.0 ** 20
       })
 
 
 # Python results example:
-# [ 
-#   [ 
+# [
+#   [
 #     {
 #       "filename": string,
 #       "benchmarks": {
-#         behavior: results, 
+#         behavior: results,
 #         ...
 #       },
 #       "message_name": STRING
 #     },
-#     ... 
+#     ...
 #   ], #pure-python
-#   ... 
+#   ...
 # ]
-def parse_python_result(filename):
-  global python_result
+def __parse_python_result(filename):
   if filename == "":
     return
   if filename[0] != '/':
@@ -115,11 +99,11 @@ def parse_python_result(filename):
     results_list = json.loads(f.read())
     for results in results_list:
       for result in results:
-        _, avg_size = get_data_size(result["filename"])
+        _, avg_size = __get_data_size(result["filename"])
         for behavior in result["benchmarks"]:
-          python_result.append({
+          __results.append({
             "language": "python",
-            "dataFileName": extract_file_name(result["filename"]),
+            "dataFilename": __extract_file_name(result["filename"]),
             "behavior": behavior,
             "throughput": avg_size /
                           result["benchmarks"][behavior] * 1e9 / 2 ** 20
@@ -127,7 +111,7 @@ def parse_python_result(filename):
 
 
 # Java results example:
-# [ 
+# [
 #   {
 #     "id": string,
 #     "instrumentSpec": {...},
@@ -153,12 +137,11 @@ def parse_python_result(filename):
 #       },
 #       ...
 #     }
-#     
-#   }, 
-#   ... 
+#
+#   },
+#   ...
 # ]
-def parse_java_result(filename):
-  global average_bytes_per_message, java_result
+def __parse_java_result(filename):
   if filename == "":
     return
   if filename[0] != '/':
@@ -172,13 +155,13 @@ def parse_java_result(filename):
         total_weight += measurement["weight"]
         total_value += measurement["value"]["magnitude"]
       avg_time = total_value * 1.0 / total_weight
-      total_size, _ = get_data_size(
+      total_size, _ = __get_data_size(
           result["scenario"]["benchmarkSpec"]["parameters"]["dataFile"])
-      java_result.append({
+      __results.append({
         "language": "java",
         "throughput": total_size / avg_time * 1e9 / 2 ** 20,
         "behavior": result["scenario"]["benchmarkSpec"]["methodName"],
-        "dataFileName": extract_file_name(
+        "dataFilename": __extract_file_name(
             result["scenario"]["benchmarkSpec"]["parameters"]["dataFile"])
       })
 
@@ -194,8 +177,7 @@ def parse_java_result(filename):
 # Benchmark/.././datasets/google_message2/dataset.google_message2.pb/Merge-12                    300     4108632 ns/op
 # PASS
 # ok    _/usr/local/google/home/yilunchong/mygit/protobuf/benchmarks  124.173s
-def parse_go_result(filename):
-  global go_result
+def __parse_go_result(filename):
   if filename == "":
     return
   if filename[0] != '/':
@@ -208,85 +190,29 @@ def parse_go_result(filename):
       first_slash_index = result_list[0].find('/')
       last_slash_index = result_list[0].rfind('/')
       full_filename = result_list[0][first_slash_index+4:last_slash_index] # delete ../ prefix
-      total_bytes, _ = get_data_size(full_filename)
+      total_bytes, _ = __get_data_size(full_filename)
       behavior_with_suffix = result_list[0][last_slash_index+1:]
       last_dash = behavior_with_suffix.rfind("-")
       if last_dash == -1:
         behavior = behavior_with_suffix
       else:
         behavior = behavior_with_suffix[:last_dash]
-      go_result.append({
-        "dataFilename": extract_file_name(full_filename),
+      __results.append({
+        "dataFilename": __extract_file_name(full_filename),
         "throughput": total_bytes / float(result_list[2]) * 1e9 / 2 ** 20,
         "behavior": behavior,
         "language": "go"
       })
 
-
-def get_metadata():
-  build_number = os.getenv('BUILD_NUMBER')
-  build_url = os.getenv('BUILD_URL')
-  job_name = os.getenv('JOB_NAME')
-  git_commit = os.getenv('GIT_COMMIT')
-  # actual commit is the actual head of PR that is getting tested
-  git_actual_commit = os.getenv('ghprbActualCommit')
-
-  utc_timestamp = str(calendar.timegm(time.gmtime()))
-  metadata = {'created': utc_timestamp}
-
-  if build_number:
-    metadata['buildNumber'] = build_number
-  if build_url:
-    metadata['buildUrl'] = build_url
-  if job_name:
-    metadata['jobName'] = job_name
-  if git_commit:
-    metadata['gitCommit'] = git_commit
-  if git_actual_commit:
-    metadata['gitActualCommit'] = git_actual_commit
-
-  return metadata
-
-
-def upload_result(result_list, metadata):
-  for result in result_list:
-    new_result = copy.deepcopy(result)
-    new_result['metadata'] = metadata
-    bq = big_query_utils.create_big_query()
-    row = big_query_utils.make_row(str(uuid.uuid4()), new_result)
-    if not big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET,
-                                       _TABLE + "$" + _NOW,
-                                       [row]):
-      print('Error when uploading result', new_result)
-
-
-if __name__ == "__main__":
-  parser = argparse.ArgumentParser()
-  parser.add_argument("-cpp", "--cpp_input_file",
-                      help="The CPP benchmark result file's name",
-                      default="")
-  parser.add_argument("-java", "--java_input_file",
-                      help="The Java benchmark result file's name",
-                      default="")
-  parser.add_argument("-python", "--python_input_file",
-                      help="The Python benchmark result file's name",
-                      default="")
-  parser.add_argument("-go", "--go_input_file",
-                      help="The golang benchmark result file's name",
-                      default="")
-  args = parser.parse_args()
-
-  parse_cpp_result(args.cpp_input_file)
-  parse_python_result(args.python_input_file)
-  parse_java_result(args.java_input_file)
-  parse_go_result(args.go_input_file)
-
-  metadata = get_metadata()
-  print("uploading cpp results...")
-  upload_result(cpp_result, metadata)
-  print("uploading java results...")
-  upload_result(java_result, metadata)
-  print("uploading python results...")
-  upload_result(python_result, metadata)
-  print("uploading go results...")
-  upload_result(go_result, metadata)
+def get_result_from_file(cpp_file="", java_file="", python_file="", go_file=""):
+  results = {}
+  if cpp_file != "":
+    __parse_cpp_result(cpp_file)
+  if java_file != "":
+    __parse_java_result(java_file)
+  if python_file != "":
+    __parse_python_result(python_file)
+  if go_file != "":
+    __parse_go_result(go_file)
+
+  return __results

benchmarks/util/result_uploader.py

@@ -0,0 +1,94 @@
+from __future__ import print_function
+from __future__ import absolute_import
+import argparse
+import os
+import re
+import copy
+import uuid
+import calendar
+import time
+import datetime
+
+from util import big_query_utils
+from util import result_parser
+
+_PROJECT_ID = 'grpc-testing'
+_DATASET = 'protobuf_benchmark_result'
+_TABLE = 'opensource_result_v2'
+_NOW = "%d%02d%02d" % (datetime.datetime.now().year,
+                       datetime.datetime.now().month,
+                       datetime.datetime.now().day)
+
+_INITIAL_TIME = calendar.timegm(time.gmtime())
+
+def get_metadata():
+  build_number = os.getenv('BUILD_NUMBER')
+  build_url = os.getenv('BUILD_URL')
+  job_name = os.getenv('JOB_NAME')
+  git_commit = os.getenv('GIT_COMMIT')
+  # actual commit is the actual head of PR that is getting tested
+  git_actual_commit = os.getenv('ghprbActualCommit')
+
+  utc_timestamp = str(calendar.timegm(time.gmtime()))
+  metadata = {'created': utc_timestamp}
+
+  if build_number:
+    metadata['buildNumber'] = build_number
+  if build_url:
+    metadata['buildUrl'] = build_url
+  if job_name:
+    metadata['jobName'] = job_name
+  if git_commit:
+    metadata['gitCommit'] = git_commit
+  if git_actual_commit:
+    metadata['gitActualCommit'] = git_actual_commit
+
+  return metadata
+
+
+def upload_result(result_list, metadata):
+  for result in result_list:
+    new_result = {}
+    new_result["metric"] = "throughput"
+    new_result["value"] = result["throughput"]
+    new_result["unit"] = "MB/s"
+    new_result["test"] = "protobuf_benchmark"
+    new_result["product_name"] = "protobuf"
+    labels_string = ""
+    for key in result:
+      labels_string += ",|%s:%s|" % (key, result[key])
+    new_result["labels"] = labels_string[1:]
+    new_result["timestamp"] = _INITIAL_TIME
+
+    bq = big_query_utils.create_big_query()
+    row = big_query_utils.make_row(str(uuid.uuid4()), new_result)
+    if not big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET,
+                                       _TABLE + "$" + _NOW,
+                                       [row]):
+      print('Error when uploading result', new_result)
+
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser()
+  parser.add_argument("-cpp", "--cpp_input_file",
+                      help="The CPP benchmark result file's name",
+                      default="")
+  parser.add_argument("-java", "--java_input_file",
+                      help="The Java benchmark result file's name",
+                      default="")
+  parser.add_argument("-python", "--python_input_file",
+                      help="The Python benchmark result file's name",
+                      default="")
+  parser.add_argument("-go", "--go_input_file",
+                      help="The golang benchmark result file's name",
+                      default="")
+  args = parser.parse_args()
+
+  metadata = get_metadata()
+  print("uploading results...")
+  upload_result(result_parser.get_result_from_file(
+      cpp_file=args.cpp_input_file,
+      java_file=args.java_input_file,
+      python_file=args.python_input_file,
+      go_file=args.go_input_file
+  ), metadata)

kokoro/linux/benchmark/build.sh

@@ -86,7 +86,7 @@ echo "benchmarking java..."
 
 # upload result to bq
 make python_add_init
-env LD_LIBRARY_PATH="$oldpwd/src/.libs" python util/run_and_upload.py -cpp="../tmp/cpp_result.json" -java="../tmp/java_result.json" \
+env LD_LIBRARY_PATH="$oldpwd/src/.libs" python -m util.result_uploader -cpp="../tmp/cpp_result.json" -java="../tmp/java_result.json" \
     -python="../tmp/python_result.json" -go="../tmp/go_result.txt"
 
 cd $oldpwd