Merge pull request #4951 from BSBandme/add_js_benchmark

Add JS and Protobuf.js benchmark, fix js's reader.skipGroup

benchmarks/Makefile.am

@@ -546,6 +546,58 @@ php_c: php-c-benchmark proto3_middleman_php
 
 ############ PHP RULES END #################
 
+############ protobuf.js RULE BEGIN #############
+
+pbjs_preparation:
+	mkdir -p tmp/protobuf.js
+	cd tmp/protobuf.js && git clone https://github.com/dcodeIO/protobuf.js.git && \
+			cd protobuf.js && npm install && npm run build
+	cd tmp/protobuf.js && npm install benchmark
+	cp protobuf.js/* tmp/protobuf.js
+	cp js/benchmark_suite.js tmp/protobuf.js
+	touch pbjs_preparation
+
+pbjs_middleman: pbjs_preparation
+	export OLDDIR=$$(pwd) && cd tmp/protobuf.js && node generate_pbjs_files.js --target static-module --include_path=$$OLDDIR -o generated_bundle_code.js $(benchmarks_protoc_inputs) $(benchmarks_protoc_inputs_benchmark_wrapper) $(benchmarks_protoc_inputs_proto2)
+	touch pbjs_middleman
+
+pbjs-benchmark: pbjs_middleman
+	@echo '#! /bin/bash' > pbjs-benchmark
+	@echo 'cd tmp/protobuf.js' >> pbjs-benchmark
+	@echo 'sed -i "s/protobufjs/.\/protobuf.js/g" generated_bundle_code.js' >> pbjs-benchmark
+	@echo 'env NODE_PATH=".:./node_modules:$$NODE_PATH" node protobufjs_benchmark.js $$@' >> pbjs-benchmark
+	@chmod +x pbjs-benchmark
+
+pbjs: pbjs-benchmark
+	./pbjs-benchmark $(all_data)
+
+############ protobuf.js RULE END #############
+
+############ JS RULE BEGIN #############
+
+js_preparation:
+	mkdir -p tmp/js
+	oldpwd=$$(pwd) && cd $(top_srcdir)/js && npm install && npm test
+	cd tmp/js && npm install benchmark
+	cp js/* tmp/js
+	touch js_preparation
+
+js_middleman: js_preparation
+	oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --js_out=import_style=commonjs,binary:$$oldpwd/tmp/js $(benchmarks_protoc_inputs) $(benchmarks_protoc_inputs_benchmark_wrapper) $(benchmarks_protoc_inputs_proto2))
+	touch js_middleman
+
+js-benchmark: js_middleman
+	@echo '#! /bin/bash' > js-benchmark
+	@echo 'export TOP_JS_SRCDIR=$$(cd $(top_srcdir)/js && pwd)' >> js-benchmark
+	@echo 'cd tmp/js' >> js-benchmark
+	@echo 'env NODE_PATH="$$TOP_JS_SRCDIR:.:./node_modules:$$NODE_PATH" node --max-old-space-size=4096 js_benchmark.js $$@' >> js-benchmark
+	@chmod +x js-benchmark
+
+js: js-benchmark
+	./js-benchmark $(all_data)
+
+############ JS RULE END #############
+
 MAINTAINERCLEANFILES =                                                     \
 	Makefile.in
 
@@ -593,8 +645,13 @@ CLEANFILES =                                                               \
 	generate_proto3_data                                                     \
 	php-benchmark                                                            \
 	php-c-benchmark                                                          \
-	proto3_middleman_php
-	
+	proto3_middleman_php                                                     \
+	pbjs_preparation                                                         \
+	pbjs_middleman                                                           \
+	pbjs-benchmark                                                           \
+	js_preparation                                                           \
+	js_middleman                                                             \
+	js-benchmark
 
 clean-local:
 	-rm -rf tmp/*

benchmarks/README.md

@@ -62,6 +62,9 @@ The second command adds the `bin` directory to your `PATH` so that `protoc` can
 PHP benchmark's requirement is the same as PHP protobuf's requirements. The benchmark will automaticly 
 include PHP protobuf's src and build the c extension if required.
 
+### Node.js
+Node.js benchmark need [node](https://nodejs.org/en/)(higher than V6) and [npm](https://www.npmjs.com/) package manager installed. This benchmark is using the [benchmark](https://www.npmjs.com/package/benchmark) framework to test, which needn't to manually install. And another prerequisite is [protobuf js](https://github.com/google/protobuf/tree/master/js), which needn't to manually install either
+
 ### Big data
 
 There's some optional big testing data which is not included in the directory
@@ -136,6 +139,11 @@ $ make php
 $ make php_c
 ```
 
+### Node.js
+```
+$ make js
+```
+
 To run a specific dataset or run with specific options:
 
 ### Java:
@@ -195,6 +203,11 @@ $ make php-c-benchmark
 $ ./php-c-benchmark $(specific generated dataset file name)
 ```
 
+### Node.js
+```
+$ make js-benchmark
+$ ./js-benchmark $(specific generated dataset file name)
+```
 
 ## Benchmark datasets
 

benchmarks/js/benchmark_suite.js

@@ -0,0 +1,33 @@
+var benchmark = require("benchmark");
+
+function newBenchmark(messageName, filename, language) {
+  var benches = [];
+  return {
+    suite: new benchmark.Suite(messageName + filename + language )
+      .on("add", function(event) {
+          benches.push(event.target);
+      })
+      .on("start", function() {
+          process.stdout.write(
+            "benchmarking message " + messageName 
+            + " of dataset file " + filename 
+            + "'s performance ..." + "\n\n");
+      })
+      .on("cycle", function(event) {
+          process.stdout.write(String(event.target) + "\n");
+      })
+      .on("complete", function() {
+          var getHz = function(bench) {
+            return 1 / (bench.stats.mean + bench.stats.moe);
+          }
+          benches.forEach(function(val, index) {
+            benches[index] = getHz(val); 
+          });
+      }),
+     benches: benches
+  }
+}
+
+module.exports = {
+        newBenchmark: newBenchmark
+}

benchmarks/js/js_benchmark.js

@@ -0,0 +1,70 @@
+require('./datasets/google_message1/proto2/benchmark_message1_proto2_pb.js');
+require('./datasets/google_message1/proto3/benchmark_message1_proto3_pb.js');
+require('./datasets/google_message2/benchmark_message2_pb.js');
+require('./datasets/google_message3/benchmark_message3_pb.js');
+require('./datasets/google_message4/benchmark_message4_pb.js');
+require('./benchmarks_pb.js');
+
+var fs = require('fs');
+var benchmarkSuite = require("./benchmark_suite.js");
+
+
+function getNewPrototype(name) {
+  var message = eval("proto." + name);
+  if (typeof(message) == "undefined") {
+    throw "type " + name + " is undefined";
+  }
+  return message;
+}
+
+var results = [];
+
+console.log("#####################################################");
+console.log("Js Benchmark: ");
+process.argv.forEach(function(filename, index) {
+  if (index < 2) {
+    return;
+  }
+  var benchmarkDataset =
+      proto.benchmarks.BenchmarkDataset.deserializeBinary(fs.readFileSync(filename));
+  var messageList = [];
+  var totalBytes = 0;
+  benchmarkDataset.getPayloadList().forEach(function(onePayload) {
+    var message = getNewPrototype(benchmarkDataset.getMessageName());
+    messageList.push(message.deserializeBinary(onePayload));
+    totalBytes += onePayload.length;
+  });
+  
+  var senarios = benchmarkSuite.newBenchmark(
+      benchmarkDataset.getMessageName(), filename, "js");
+  senarios.suite
+  .add("js deserialize", function() {
+    benchmarkDataset.getPayloadList().forEach(function(onePayload) {
+      var protoType = getNewPrototype(benchmarkDataset.getMessageName());
+      protoType.deserializeBinary(onePayload);
+    });    
+  })
+  .add("js serialize", function() {
+    var protoType = getNewPrototype(benchmarkDataset.getMessageName());
+    messageList.forEach(function(message) {
+      message.serializeBinary();
+    });
+  }) 
+  .run({"Async": false});
+
+  results.push({
+    filename: filename,
+    benchmarks: {
+      protobufjs_decoding: senarios.benches[0] * totalBytes,
+      protobufjs_encoding: senarios.benches[1] * totalBytes
+    }
+  })
+
+  console.log("Throughput for deserialize: " 
+    + senarios.benches[0] * totalBytes / 1024 / 1024 + "MB/s" );
+  console.log("Throughput for serialize: " 
+    + senarios.benches[1] * totalBytes / 1024 / 1024 + "MB/s" );
+  console.log("");
+});
+console.log("#####################################################");
+

benchmarks/protobuf.js/generate_pbjs_files.js

@@ -0,0 +1,25 @@
+var pbjs = require("./protobuf.js/cli").pbjs
+
+var argv = [];
+var protoFiles = [];
+var prefix = "";
+process.argv.forEach(function(val, index) {
+  var arg = val;
+  if (arg.length > 6 && arg.substring(arg.length - 6) == ".proto") {
+    protoFiles.push(arg);
+  } else if (arg.length > 15 && arg.substring(0, 15) ==  "--include_path=") {
+    prefix = arg.substring(15);
+  } else if (index >= 2) {
+    argv.push(arg);
+  }
+});
+protoFiles.forEach(function(val) {
+  argv.push(prefix + "/" + val);
+});
+
+pbjs.main(argv, function(err, output){
+  if (err) {
+    console.log(err);
+  }
+});
+

benchmarks/protobuf.js/protobufjs_benchmark.js

@@ -0,0 +1,66 @@
+var root = require("./generated_bundle_code.js");
+var fs = require('fs');
+var benchmark = require("./node_modules/benchmark");
+var benchmarkSuite = require("./benchmark_suite.js");
+
+
+function getNewPrototype(name) {
+  var message = eval("root." + name);
+  if (typeof(message) == "undefined") {
+    throw "type " + name + " is undefined";
+  }
+  return message;
+}
+
+
+var results = [];
+
+console.log("#####################################################");
+console.log("ProtobufJs Benchmark: ");
+process.argv.forEach(function(filename, index) {
+  if (index < 2) {
+    return;
+  }
+  var benchmarkDataset =
+      root.benchmarks.BenchmarkDataset.decode(fs.readFileSync(filename));
+  var messageList = [];
+  var totalBytes = 0;
+  benchmarkDataset.payload.forEach(function(onePayload) {
+    var message = getNewPrototype(benchmarkDataset.messageName);
+    messageList.push(message.decode(onePayload));
+    totalBytes += onePayload.length;
+  });
+  
+  var senarios = benchmarkSuite.newBenchmark(
+    benchmarkDataset.messageName, filename, "protobufjs");
+  senarios.suite
+  .add("protobuf.js static decoding", function() {
+    benchmarkDataset.payload.forEach(function(onePayload) {
+      var protoType = getNewPrototype(benchmarkDataset.messageName);
+      protoType.decode(onePayload);
+    });    
+  })
+  .add("protobuf.js static encoding", function() {
+    var protoType = getNewPrototype(benchmarkDataset.messageName);
+    messageList.forEach(function(message) {
+      protoType.encode(message).finish();
+    });
+  }) 
+  .run({"Async": false});
+
+  results.push({
+    filename: filename,
+    benchmarks: {
+      protobufjs_decoding: senarios.benches[0] * totalBytes,
+      protobufjs_encoding: senarios.benches[1] * totalBytes
+    }
+  })
+
+  console.log("Throughput for decoding: " 
+    + senarios.benches[0] * totalBytes / 1024 / 1024 + "MB/s" );
+  console.log("Throughput for encoding: " 
+    + senarios.benches[1] * totalBytes / 1024 / 1024 + "MB/s" );
+  console.log("");
+});
+console.log("#####################################################");
+