Add caliper supported to java benchmark

.gitmodules

@@ -1,3 +1,4 @@
 [submodule "third_party/benchmark"]
 	path = third_party/benchmark
 	url = https://github.com/google/benchmark.git
+

benchmarks/Makefile.am

@@ -20,7 +20,7 @@ benchmarks_protoc_inputs_proto2 =                                          \
 	datasets/google_message4/benchmark_message4_3.proto
 
 make_tmp_dir:
-	mkdir -p 'tmp'
+	mkdir -p 'tmp/java/src/main/java'
 	touch make_tmp_dir
 
 if USE_EXTERNAL_PROTOC
@@ -39,11 +39,11 @@ else
 # relative to srcdir, which may not be the same as the current directory when
 # building out-of-tree.
 protoc_middleman: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs) $(well_known_type_protoc_inputs)
-	oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp $(benchmarks_protoc_inputs) )
+	oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp/java/src/main/java $(benchmarks_protoc_inputs) )
 	touch protoc_middleman
 
 protoc_middleman2:  make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs_proto2) $(well_known_type_protoc_inputs)
-	oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2) )
+	oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp/java/src/main/java $(benchmarks_protoc_inputs_proto2) )
 	touch protoc_middleman2
 	
 endif
@@ -124,18 +124,19 @@ cpp: protoc_middleman protoc_middleman2 cpp-benchmark
 
 ############# JAVA RULES ##############
 
-javac_middleman: ProtoBench.java protoc_middleman protoc_middleman2
-	jar=`ls $(top_srcdir)/java/util/target/*.jar` &&                         \
-	jar1=`ls $(top_srcdir)/java/core/target/*.jar` &&                        \
-	javac -d tmp -classpath ../java/target/classes:$$jar:$$jar1 ProtoBench.java $(benchmark_java_protoc_outputs_proto2) $(benchmarks_java_protoc_outputs) `find tmp -type f -name "*.java"`
+java_benchmark_testing_files = 
+	java/src/main/java/com/google/protocolbuffers/ProtoBench.java            \
+	java/src/main/java/com/google/protocolbuffers/ProtoBenchCaliper.java
+
+javac_middleman: $(java_benchmark_testing_files) protoc_middleman protoc_middleman2
+	cp -r java tmp && cd tmp/java &&  mvn clean compile assembly:single
+	cd ../..
 	@touch javac_middleman
 
 java-benchmark: javac_middleman 
 	@echo "Writing shortcut script java-benchmark..."
 	@echo '#! /bin/sh' > java-benchmark
-	@jar=`ls $(top_srcdir)/java/util/target/*.jar` &&                        \
-	jar1=`ls $(top_srcdir)/java/core/target/*.jar` &&                        \
-	echo java -classpath tmp:../java/target/classes:$$jar:$$jar1 com.google.protocolbuffers.ProtoBench '$$@' >> java-benchmark
+	@echo 'java -cp '"tmp/java/target/*.jar"' com.google.protocolbuffers.ProtoBench $$@' >> java-benchmark
 	@chmod +x java-benchmark
 
 java: protoc_middleman protoc_middleman2 java-benchmark

benchmarks/ProtoBench.java

@@ -1,319 +0,0 @@
-// Protocol Buffers - Google's data interchange format
-// Copyright 2009 Google Inc.  All rights reserved.
-// https://developers.google.com/protocol-buffers/
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-//     * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-//     * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-//     * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-package com.google.protocolbuffers;
-
-import com.google.protobuf.ByteString;
-import com.google.protobuf.CodedInputStream;
-import com.google.protobuf.CodedOutputStream;
-import com.google.protobuf.ExtensionRegistry;
-import com.google.protobuf.Message;
-import com.google.protobuf.benchmarks.Benchmarks.BenchmarkDataset;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.util.ArrayList;
-import java.util.List;
-
-public class ProtoBench {
-
-  private static final long MIN_SAMPLE_TIME_MS = 1 * 1000;
-  private static final long TARGET_TIME_MS = 5 * 1000;
-
-  private ProtoBench() {
-    // Prevent instantiation
-  }
-  
-  public static void main(String[] args) {
-    if (args.length < 1) {
-      System.err.println("Usage: ./java-benchmark <input data>");
-      System.err.println("input data is in the format of \"benchmarks.proto\"");
-      System.exit(1);
-    }
-    boolean success = true;
-    for (int i = 0; i < args.length; i++) {
-      success &= runTest(args[i]);
-    }
-    System.exit(success ? 0 : 1);
-  }
-  
-  public static ExtensionRegistry getExtensionsRegistry(BenchmarkDataset benchmarkDataset) {
-    ExtensionRegistry extensions = ExtensionRegistry.newInstance();
-    if (benchmarkDataset.getMessageName().equals("benchmarks.google_message3.GoogleMessage3")) {
-      benchmarks.google_message3.BenchmarkMessage38.registerAllExtensions(extensions);
-      benchmarks.google_message3.BenchmarkMessage37.registerAllExtensions(extensions);
-      benchmarks.google_message3.BenchmarkMessage36.registerAllExtensions(extensions);
-      benchmarks.google_message3.BenchmarkMessage35.registerAllExtensions(extensions);
-      benchmarks.google_message3.BenchmarkMessage34.registerAllExtensions(extensions);
-      benchmarks.google_message3.BenchmarkMessage33.registerAllExtensions(extensions);
-      benchmarks.google_message3.BenchmarkMessage32.registerAllExtensions(extensions);
-      benchmarks.google_message3.BenchmarkMessage31.registerAllExtensions(extensions);
-      benchmarks.google_message3.BenchmarkMessage3.registerAllExtensions(extensions);
-    } else if (benchmarkDataset.getMessageName().equals(
-        "benchmarks.google_message4.GoogleMessage4")) {
-      benchmarks.google_message4.BenchmarkMessage43.registerAllExtensions(extensions);
-      benchmarks.google_message4.BenchmarkMessage42.registerAllExtensions(extensions);
-      benchmarks.google_message4.BenchmarkMessage41.registerAllExtensions(extensions);
-      benchmarks.google_message4.BenchmarkMessage4.registerAllExtensions(extensions);      
-    }
-    
-    return extensions;
-  }
-
-  /**
-   * Return an message instance for one specific dataset, and register the extensions for that 
-   * message. 
-   */
-  public static Message registerBenchmarks(BenchmarkDataset benchmarkDataset) {
-    if (benchmarkDataset.getMessageName().equals("benchmarks.proto3.GoogleMessage1")) {
-      return com.google.protobuf.benchmarks.BenchmarkMessage1Proto3.GoogleMessage1
-          .getDefaultInstance();
-    } else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage1")) {
-      return com.google.protobuf.benchmarks.BenchmarkMessage1Proto2.GoogleMessage1
-          .getDefaultInstance();
-    } else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage2")) {
-      return com.google.protobuf.benchmarks.BenchmarkMessage2.GoogleMessage2.getDefaultInstance();
-    } else if (benchmarkDataset.getMessageName().
-        equals("benchmarks.google_message3.GoogleMessage3")) {
-      return benchmarks.google_message3.BenchmarkMessage3.GoogleMessage3.getDefaultInstance();
-    } else if (benchmarkDataset.getMessageName().
-        equals("benchmarks.google_message4.GoogleMessage4")) {
-      return benchmarks.google_message4.BenchmarkMessage4.GoogleMessage4.getDefaultInstance();
-    } else {
-      return null;
-    }
-  }
-
-  /**
-   * Runs a single test. Error messages are displayed to stderr, and the return value indicates
-   * general success/failure.
-   */
-  public static boolean runTest(String file) {
-    final Message defaultMessage;
-    BenchmarkDataset benchmarkDataset;
-    ExtensionRegistry extensions;
-    final byte[] inputData;
-
-    try {
-      inputData = readAllBytes(file);
-      benchmarkDataset = BenchmarkDataset.parseFrom(inputData);
-    } catch (IOException e) {
-      System.err.println("Unable to get input data");
-      return false;
-    }
-    defaultMessage = registerBenchmarks(benchmarkDataset);
-    extensions = getExtensionsRegistry(benchmarkDataset);
-    if (defaultMessage == null) {
-      System.err.println("Unable to get default message " + benchmarkDataset.getMessageName());
-      return false;
-    }
-    System.out.println("Benchmarking " + benchmarkDataset.getMessageName() + " with file " + file);
-
-    try {
-      List<byte[]> inputDataList = new ArrayList<byte[]>();
-      List<ByteArrayInputStream> inputStreamList = new ArrayList<ByteArrayInputStream>();
-      List<ByteString> inputStringList = new ArrayList<ByteString>();
-      List<Message> sampleMessageList = new ArrayList<Message>();
-
-      for (int i = 0; i < benchmarkDataset.getPayloadCount(); i++) {
-        byte[] singleInputData = benchmarkDataset.getPayload(i).toByteArray();
-        inputDataList.add(benchmarkDataset.getPayload(i).toByteArray());
-        inputStreamList.add(new ByteArrayInputStream(benchmarkDataset.getPayload(i).toByteArray()));
-        inputStringList.add(benchmarkDataset.getPayload(i));
-        sampleMessageList.add(
-            defaultMessage.newBuilderForType().mergeFrom(singleInputData, extensions).build());
-      }
-
-      FileOutputStream devNullTemp = null;
-      CodedOutputStream reuseDevNullTemp = null;
-      try {
-        devNullTemp = new FileOutputStream("/dev/null");
-        reuseDevNullTemp = CodedOutputStream.newInstance(devNullTemp);
-      } catch (FileNotFoundException e) {
-        // ignore: this is probably Windows, where /dev/null does not exist
-      }
-      final FileOutputStream devNull = devNullTemp;
-      final CodedOutputStream reuseDevNull = reuseDevNullTemp;
-      benchmark(
-          "Serialize to byte string",
-          inputData.length,
-          new Action() {
-            public void execute() {
-              for (int i = 0; i < sampleMessageList.size(); i++) {
-                sampleMessageList.get(i).toByteString();
-              }
-            }
-          });
-      benchmark(
-          "Serialize to byte array",
-          inputData.length,
-          new Action() {
-            public void execute() {
-              for (int i = 0; i < sampleMessageList.size(); i++) {
-                sampleMessageList.get(i).toByteString();
-              }
-            }
-          });
-      benchmark(
-          "Serialize to memory stream",
-          inputData.length,
-          new Action() {
-            public void execute() throws IOException {
-              ByteArrayOutputStream output = new ByteArrayOutputStream();
-              for (int i = 0; i < sampleMessageList.size(); i++) {
-                sampleMessageList.get(i).writeTo(output);
-              }
-            }
-          });
-      if (devNull != null) {
-        benchmark(
-            "Serialize to /dev/null with FileOutputStream",
-            inputData.length,
-            new Action() {
-              public void execute() throws IOException {
-                for (int i = 0; i < sampleMessageList.size(); i++) {
-                  sampleMessageList.get(i).writeTo(devNull);
-                }
-              }
-            });
-        benchmark(
-            "Serialize to /dev/null reusing FileOutputStream",
-            inputData.length,
-            new Action() {
-              public void execute() throws IOException {
-                for (int i = 0; i < sampleMessageList.size(); i++) {
-                  sampleMessageList.get(i).writeTo(reuseDevNull);
-                  reuseDevNull.flush(); // force the write to the OutputStream
-                }
-              }
-            });
-      }
-      benchmark(
-          "Deserialize from byte string",
-          inputData.length,
-          new Action() {
-            public void execute() throws IOException {
-              for (int i = 0; i < inputStringList.size(); i++) {
-                defaultMessage
-                    .newBuilderForType()
-                    .mergeFrom(inputStringList.get(i), extensions)
-                    .build();
-              }
-            }
-          });
-      benchmark(
-          "Deserialize from byte array",
-          inputData.length,
-          new Action() {
-            public void execute() throws IOException {
-              for (int i = 0; i < inputDataList.size(); i++) {
-                defaultMessage
-                    .newBuilderForType()
-                    .mergeFrom(CodedInputStream.newInstance(inputDataList.get(i)), extensions)
-                    .build();
-              }
-            }
-          });
-      benchmark(
-          "Deserialize from memory stream",
-          inputData.length,
-          new Action() {
-            public void execute() throws IOException {
-              for (int i = 0; i < inputStreamList.size(); i++) {
-                defaultMessage
-                    .newBuilderForType()
-                    .mergeFrom(CodedInputStream.newInstance(inputStreamList.get(i)), extensions)
-                    .build();
-                inputStreamList.get(i).reset();
-              }
-            }
-          });
-      System.out.println();
-      return true;
-    } catch (Exception e) {
-      System.err.println("Error: " + e.getMessage());
-      System.err.println("Detailed exception information:");
-      e.printStackTrace(System.err);
-      return false;
-    }
-  }
-  
-  private static void benchmark(String name, long dataSize, Action action) throws IOException {
-    // Make sure it's JITted "reasonably" hard before running the first progress test
-    for (int i=0; i < 100; i++) {
-      action.execute();
-    }
-    
-    // Run it progressively more times until we've got a reasonable sample
-    int iterations = 1;
-    long elapsed = timeAction(action, iterations);
-    while (elapsed < MIN_SAMPLE_TIME_MS) {
-      iterations *= 2;
-      elapsed = timeAction(action, iterations);
-    }
-    
-    // Upscale the sample to the target time. Do this in floating point arithmetic
-    // to avoid overflow issues.
-    iterations = (int) ((TARGET_TIME_MS / (double) elapsed) * iterations);
-    elapsed = timeAction(action, iterations);
-    System.out.println(name + ": " + iterations + " iterations in "
-         + (elapsed/1000f) + "s; " 
-         + (iterations * dataSize) / (elapsed * 1024 * 1024 / 1000f) 
-         + "MB/s");
-  }
-  
-  private static long timeAction(Action action, int iterations) throws IOException {
-    System.gc();    
-    long start = System.currentTimeMillis();
-    for (int i = 0; i < iterations; i++) {
-      action.execute();
-    }
-    long end = System.currentTimeMillis();
-    return end - start;
-  }
-  
-  private static byte[] readAllBytes(String filename) throws IOException {
-    RandomAccessFile file = new RandomAccessFile(new File(filename), "r");
-    byte[] content = new byte[(int) file.length()];
-    file.readFully(content);
-    return content;
-  }
-
-  /**
-   * Interface used to capture a single action to benchmark.
-   */
-  interface Action {
-    void execute() throws IOException;
-  }
-}

benchmarks/README.md

@@ -7,6 +7,18 @@ protobuf language runtime.
 
 The schema for the datasets is described in `benchmarks.proto`.
 
+The benchmark is based on some submodules. To initialize the submodues:
+
+For java:
+```
+$ ./initialize_submodule.sh java
+```
+
+For java:
+```
+$ ./initialize_submodule.sh cpp
+```
+
 To run all the benchmark dataset:
 
 For java:
@@ -26,17 +38,24 @@ To run a specific dataset:
 For java:
 
 ```
-$ make java
+$ make java-benchmark
 $ ./java-benchmark $(specific generated dataset file name)
 ```
 
 For cpp:
 
 ```
-$ make cpp
+$ make cpp-benchmark
 $ ./cpp-benchmark $(specific generated dataset file name)
 ```
 
+There's some big testing data not included in the directory initially, you need to 
+run the following command to download the testing data:
+
+```
+$ ./download_data.sh 
+```
+
 Each data set is in the format of benchmarks.proto:
 1. name is the benchmark dataset's name.
 2. message_name is the benchmark's message type full name (including package and message name)

benchmarks/initialize_submodule.sh

@@ -0,0 +1,7 @@
+#! /bin/sh
+
+oldpwd=`pwd`
+cd "../third_party"
+git submodule update --init -r
+cd benchmark && cmake -DCMAKE_BUILD_TYPE=Release && make && cd ..
+cd "$oldpwd"

benchmarks/java/pom.xml

@@ -0,0 +1,148 @@
+
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Copyright 2011 Google Inc.
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+   <modelVersion>4.0.0</modelVersion>
+<!--  <parent>
+	  <groupId>com.google.protobuf</groupId>
+	  <artifactId>protobuf-parent</artifactId>
+	  <version>3.5.0</version>
+  </parent> 
+
+  <packaging>pom</packaging> -->
+  <artifactId>protobuf-java-benchmark</artifactId>
+  <groupId>com.google.protocolbuffers</groupId>
+  <version>1.0.0</version>
+  <name>Protocol Buffers [Benchmark]</name>
+  <description>
+	  The benchmark tools for Protobuf Java.
+  </description>
+
+  <dependencies>
+    <dependency>
+	    <groupId>com.google.protobuf</groupId>
+	    <artifactId>protobuf-java</artifactId>
+	    <version>3.5.0</version>
+	</dependency>
+	<dependency>
+		<groupId>com.google.caliper</groupId>
+	    <artifactId>caliper</artifactId>
+		<version>1.0-beta-2</version>
+	</dependency>
+  </dependencies>
+
+  <build>
+    <pluginManagement>
+	  <plugins>
+		<plugin>
+			<groupId>org.apache.maven.plugins</groupId>
+			<artifactId>maven-assembly-plugin</artifactId>
+			<version>2.4.1</version>
+			<configuration>
+				<!-- get all project dependencies -->
+				<descriptorRefs>
+					<descriptorRef>jar-with-dependencies</descriptorRef>
+				</descriptorRefs>
+				<!-- MainClass in mainfest make a executable jar -->
+				<archive>
+				  <manifest>
+					<mainClass>com.mkyong.core.utils.App</mainClass>
+				  </manifest>
+				</archive>
+
+			</configuration>
+			<executions>
+			  <execution>
+				<id>make-assembly</id>
+                                    <!-- bind to the packaging phase -->
+				<phase>package</phase>
+				<goals>
+					<goal>single</goal>
+				</goals>
+			  </execution>
+			</executions>
+		</plugin>
+	    <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-compiler-plugin</artifactId>
+          <version>3.5.1</version>
+          <configuration>
+            <source>1.8</source>
+            <target>1.8</target>
+          </configuration>
+        </plugin>
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-jar-plugin</artifactId>
+          <version>2.5</version>
+          <configuration>
+            <archive>
+              <manifest>
+                <addClasspath>true</addClasspath>
+                <mainClass>com.google.protocolbuffers.ProtoBench</mainClass>
+              </manifest>
+            </archive>
+          </configuration>
+        </plugin>
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-source-plugin</artifactId>
+          <version>2.4</version>
+          <executions>
+            <execution>
+              <id>attach-sources</id>
+              <goals>
+                <goal>jar-no-fork</goal>
+              </goals>
+            </execution>
+          </executions>
+        </plugin>
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-shade-plugin</artifactId>
+          <version>2.3</version>
+          <executions>
+            <execution>
+              <phase>package</phase>
+              <goals>
+                <goal>shade</goal>
+              </goals>
+              <configuration>
+                <shadedArtifactAttached>true</shadedArtifactAttached>
+                <shadedClassifierName>all</shadedClassifierName>
+                <transformers>
+                  <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                    <manifestEntries>
+                      <Premain-Class>com.google.monitoring.runtime.instrumentation.AllocationInstrumenter</Premain-Class>
+                      <Can-Redefine-Classes>true</Can-Redefine-Classes>
+                      <Can-Retransform-Classes>true</Can-Retransform-Classes>
+                    </manifestEntries>
+                  </transformer>
+                </transformers>
+              </configuration>
+            </execution>
+          </executions>
+        </plugin>
+      </plugins>
+    </pluginManagement>
+  </build>
+
+</project>
+

benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBench.java

@@ -0,0 +1,155 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2009 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+package com.google.protocolbuffers;
+
+import com.google.caliper.BeforeExperiment;
+import com.google.caliper.Benchmark;
+import com.google.caliper.Param;
+import com.google.caliper.runner.CaliperMain;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.CodedInputStream;
+import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
+import com.google.protobuf.Message;
+import com.google.protobuf.benchmarks.Benchmarks.BenchmarkDataset;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.RandomAccessFile;
+import java.util.ArrayList;
+import java.util.List;
+
+
+public class ProtoBench {
+
+  private ProtoBench() {
+    // Prevent instantiation
+  }
+    
+  public static void main(String[] args) {
+    if (args.length < 1) {
+      System.err.println("Usage: ./java-benchmark <input data>");
+      System.err.println("input data is in the format of \"benchmarks.proto\"");
+      System.exit(1);
+    }
+    boolean success = true;
+    for (int i = 0; i < args.length; i++) {
+      success &= runTest(args[i]);
+    }
+    System.exit(success ? 0 : 1);
+  }
+ 
+  /**
+   * Runs a single test with specific test data. Error messages are displayed to stderr, 
+   * and the return value indicates general success/failure.
+   */
+  public static boolean runTest(String file) {
+    byte[] inputData;
+    BenchmarkDataset benchmarkDataset;
+    try {
+      inputData = readAllBytes(file);
+      benchmarkDataset = BenchmarkDataset.parseFrom(inputData);
+    } catch (IOException e) {
+      System.err.println("Unable to get input data");
+      return false;
+    }
+    
+    List<String> argsList = getCaliperOption(benchmarkDataset);
+    if (argsList == null) {
+      System.err.println("Unable to get default message " + benchmarkDataset.getMessageName());
+      return false;
+    }
+    argsList.add("-DdataFile=" + file);
+    argsList.add("com.google.protocolbuffers.ProtoBenchCaliper");
+    
+    try {
+      String args[] = new String[argsList.size()];
+      argsList.toArray(args);
+      CaliperMain.exitlessMain(args, 
+          new PrintWriter(System.out, true), new PrintWriter(System.err, true));
+      return true;
+    } catch (Exception e) {
+      System.err.println("Error: " + e.getMessage());
+      System.err.println("Detailed exception information:");
+      e.printStackTrace(System.err);
+      return false;
+    }
+  }
+  
+  
+  private static List<String> getCaliperOption(final BenchmarkDataset benchmarkDataset) {
+    List<String> temp = new ArrayList<String>();
+    if (benchmarkDataset.getMessageName().equals("benchmarks.proto3.GoogleMessage1")) {
+      temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE1_PROTO3");
+    } else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage1")) {
+      temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE1_PROTO2");
+    } else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage2")) {
+      temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE2");
+    } else if (benchmarkDataset.getMessageName().
+        equals("benchmarks.google_message3.GoogleMessage3")) {
+      temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE3");
+      for (String opt : ProtoBenchCaliper
+                          .BenchmarkMessageType.GOOGLE_MESSAGE3.getSpecificCaliperOption()) {
+        temp.add(opt);
+      }
+    } else if (benchmarkDataset.getMessageName().
+        equals("benchmarks.google_message4.GoogleMessage4")) {
+      temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE4");
+      for (String opt : ProtoBenchCaliper
+                          .BenchmarkMessageType.GOOGLE_MESSAGE4.getSpecificCaliperOption()) {
+        temp.add(opt);
+      }
+    } else {
+      return null;
+    }
+    
+    temp.add("-i");
+    temp.add("runtime");
+    temp.add("-b");
+    String benchmarkNames = "serializeToByteString,serializeToByteArray,serializeToMemoryStream"
+       + ",deserializeFromByteString,deserializeFromByteArray,deserializeFromMemoryStream";
+    temp.add(benchmarkNames);
+    
+    return temp;
+  }
+  
+  public static byte[] readAllBytes(String filename) throws IOException {
+    RandomAccessFile file = new RandomAccessFile(new File(filename), "r");
+    byte[] content = new byte[(int) file.length()];
+    file.readFully(content);
+    return content;
+  }
+}

benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBenchCaliper.java

@@ -0,0 +1,210 @@
+
+package com.google.protocolbuffers;
+
+import com.google.caliper.BeforeExperiment;
+import com.google.caliper.Benchmark;
+import com.google.caliper.Param;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
+import com.google.protobuf.Message;
+import com.google.protobuf.benchmarks.Benchmarks.BenchmarkDataset;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class ProtoBenchCaliper {
+  public enum BenchmarkMessageType {
+    GOOGLE_MESSAGE1_PROTO3 {
+      @Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); }
+      @Override
+      Message getDefaultInstance() {
+        return com.google.protobuf.benchmarks.BenchmarkMessage1Proto3.GoogleMessage1
+            .getDefaultInstance();
+      }
+      @Override String[] getSpecificCaliperOption() { return null; }
+    },
+    GOOGLE_MESSAGE1_PROTO2 {
+      @Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); }
+      @Override
+      Message getDefaultInstance() {
+        return com.google.protobuf.benchmarks.BenchmarkMessage1Proto2.GoogleMessage1
+            .getDefaultInstance();
+      }
+      @Override String[] getSpecificCaliperOption() { return null; }
+    },
+    GOOGLE_MESSAGE2 {
+      @Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); }
+      @Override
+      Message getDefaultInstance() {
+        return com.google.protobuf.benchmarks.BenchmarkMessage2.GoogleMessage2.getDefaultInstance();
+      }
+      @Override String[] getSpecificCaliperOption() { return null; }
+    },
+    GOOGLE_MESSAGE3 {
+      @Override
+      ExtensionRegistry getExtensionRegistry() {
+        ExtensionRegistry extensions = ExtensionRegistry.newInstance();
+        benchmarks.google_message3.BenchmarkMessage38.registerAllExtensions(extensions);
+        benchmarks.google_message3.BenchmarkMessage37.registerAllExtensions(extensions);
+        benchmarks.google_message3.BenchmarkMessage36.registerAllExtensions(extensions);
+        benchmarks.google_message3.BenchmarkMessage35.registerAllExtensions(extensions);
+        benchmarks.google_message3.BenchmarkMessage34.registerAllExtensions(extensions);
+        benchmarks.google_message3.BenchmarkMessage33.registerAllExtensions(extensions);
+        benchmarks.google_message3.BenchmarkMessage32.registerAllExtensions(extensions);
+        benchmarks.google_message3.BenchmarkMessage31.registerAllExtensions(extensions);
+        benchmarks.google_message3.BenchmarkMessage3.registerAllExtensions(extensions);
+        return extensions;
+      }
+      @Override
+      Message getDefaultInstance() {
+        return benchmarks.google_message3.BenchmarkMessage3.GoogleMessage3.getDefaultInstance();
+      }
+      @Override 
+      String[] getSpecificCaliperOption() { 
+        String[] opt = new String[1];
+        opt[0] = "-Cinstrument.runtime.options.timingInterval=3000ms";
+        return opt;
+      }
+    },
+    GOOGLE_MESSAGE4 {
+      @Override
+      ExtensionRegistry getExtensionRegistry() {
+        ExtensionRegistry extensions = ExtensionRegistry.newInstance();
+        benchmarks.google_message4.BenchmarkMessage43.registerAllExtensions(extensions);
+        benchmarks.google_message4.BenchmarkMessage42.registerAllExtensions(extensions);
+        benchmarks.google_message4.BenchmarkMessage41.registerAllExtensions(extensions);
+        benchmarks.google_message4.BenchmarkMessage4.registerAllExtensions(extensions);
+        return extensions;
+      }
+      @Override
+      Message getDefaultInstance() {
+        return benchmarks.google_message4.BenchmarkMessage4.GoogleMessage4.getDefaultInstance();
+      }
+      @Override 
+      String[] getSpecificCaliperOption() { 
+        String[] opt = new String[1];
+        opt[0] = "-Cinstrument.runtime.options.timingInterval=3000ms";
+        return opt;
+      }
+    };
+    
+    abstract ExtensionRegistry getExtensionRegistry();
+    abstract Message getDefaultInstance();
+    abstract String[] getSpecificCaliperOption();
+  }
+  
+  @Param 
+  private BenchmarkMessageType benchmarkMessageType;
+  @Param
+  private String dataFile;
+  
+  private byte[] inputData;
+  private BenchmarkDataset benchmarkDataset;
+  private Message defaultMessage;
+  private ExtensionRegistry extensions;
+  private List<byte[]> inputDataList;
+  private List<ByteArrayInputStream> inputStreamList;
+  private List<ByteString> inputStringList;
+  private List<Message> sampleMessageList;
+  private int counter;
+  private FileOutputStream devNull;
+  private CodedOutputStream reuseDevNull;
+  
+  @BeforeExperiment
+  void setUp() throws IOException {
+    inputData = ProtoBench.readAllBytes(dataFile);
+    benchmarkDataset = BenchmarkDataset.parseFrom(inputData);
+    defaultMessage = benchmarkMessageType.getDefaultInstance();
+    extensions = benchmarkMessageType.getExtensionRegistry();
+    inputDataList = new ArrayList<byte[]>();
+    inputStreamList = new ArrayList<ByteArrayInputStream>();
+    inputStringList = new ArrayList<ByteString>();
+    sampleMessageList = new ArrayList<Message>();
+    
+    for (int i = 0; i < benchmarkDataset.getPayloadCount(); i++) {
+      byte[] singleInputData = benchmarkDataset.getPayload(i).toByteArray();
+      inputDataList.add(benchmarkDataset.getPayload(i).toByteArray());
+      inputStreamList.add(new ByteArrayInputStream(
+          benchmarkDataset.getPayload(i).toByteArray()));
+      inputStringList.add(benchmarkDataset.getPayload(i));
+      sampleMessageList.add(
+          defaultMessage.newBuilderForType().mergeFrom(singleInputData, extensions).build());
+    }
+    devNull = null;
+    reuseDevNull = null;
+    
+    try {
+      devNull = new FileOutputStream("/dev/null");
+      reuseDevNull = CodedOutputStream.newInstance(devNull);
+    } catch (FileNotFoundException e) {
+      // ignore: this is probably Windows, where /dev/null does not exist
+    }
+    
+    counter = 0;
+  }
+  
+  
+  @Benchmark
+  void serializeToByteString(int reps) throws IOException {
+    for (int i = 0; i < reps; i++) {
+      sampleMessageList.get(counter % sampleMessageList.size()).toByteString();
+      counter++;
+    }
+  }
+  
+  @Benchmark
+  void serializeToByteArray(int reps) throws IOException {
+    for (int i = 0; i < reps; i++) {
+      sampleMessageList.get(counter % sampleMessageList.size()).toByteArray();
+      counter++;
+    }
+  }
+  
+  @Benchmark
+  void serializeToMemoryStream(int reps) throws IOException {
+    for (int i = 0; i < reps; i++) {
+      ByteArrayOutputStream output = new ByteArrayOutputStream();
+      sampleMessageList.get(counter % sampleMessageList.size()).writeTo(output);
+      counter++;
+    }
+  }
+  
+  @Benchmark
+  void deserializeFromByteString(int reps) throws IOException {
+    for (int i = 0; i < reps; i++) {
+      defaultMessage
+        .newBuilderForType()
+        .mergeFrom(inputStringList.get(counter % inputStringList.size()), extensions)
+        .build();
+      counter++;
+    }
+  }
+  
+  @Benchmark
+  void deserializeFromByteArray(int reps) throws IOException {
+    for (int i = 0; i < reps; i++) {
+      defaultMessage
+        .newBuilderForType()
+        .mergeFrom(inputDataList.get(counter % inputDataList.size()), extensions)
+        .build();
+      counter++;
+    }
+  }
+  
+  @Benchmark
+  void deserializeFromMemoryStream(int reps) throws IOException {
+    for (int i = 0; i < reps; i++) {
+      defaultMessage
+        .newBuilderForType()
+        .mergeFrom(inputStreamList.get(counter % inputStreamList.size()), extensions)
+        .build();
+      inputStreamList.get(counter % inputStreamList.size()).reset();
+      counter++;
+    }
+  }
+}

tests.sh

@@ -44,10 +44,7 @@ build_cpp() {
   # appears to be missing it: https://github.com/travis-ci/travis-ci/issues/6996
   if [[ $(type cmake 2>/dev/null) ]]; then
     # Verify benchmarking code can build successfully.
-    git submodule init
-    git submodule update
-    cd third_party/benchmark && cmake -DCMAKE_BUILD_TYPE=Release && make && cd ../..
-    cd benchmarks && make cpp-benchmark && cd ..
+	cd benchmarks && ./initialize_submodule.sh cpp && make cpp-benchmark && cd ..
   else
     echo ""
     echo "WARNING: Skipping validation of the bench marking code, cmake isn't installed."