Fix bugs

benchmarks/Makefile.am

@@ -136,7 +136,7 @@ javac_middleman: $(java_benchmark_testing_files) protoc_middleman protoc_middlem
 java-benchmark: javac_middleman 
 	@echo "Writing shortcut script java-benchmark..."
 	@echo '#! /bin/sh' > java-benchmark
-	@echo 'java -cp '"tmp/java/target/*.jar"' com.google.protocolbuffers.ProtoBench $$@' >> java-benchmark
+	@echo 'java -cp '"tmp/java/target/*.jar"' com.google.protobuf.ProtoBench $$@' >> java-benchmark
 	@chmod +x java-benchmark
 
 java: protoc_middleman protoc_middleman2 java-benchmark

benchmarks/README.md

@@ -5,20 +5,31 @@ This directory contains benchmarking schemas and data sets that you
 can use to test a variety of performance scenarios against your
 protobuf language runtime.
 
-The schema for the datasets is described in `benchmarks.proto`.
+## Benchmark tools and build instructions
 
-The benchmark is based on some submodules. To initialize the submodues:
+First, you need to follow the instruction in the root directory's README to 
+build your language's protobuf, then:
 
-For java:
-```
-$ ./initialize_submodule.sh java
-```
+### CPP
+We are using [google/benchmark](https://github.com/google/benchmark) as the 
+benchmark tool for testing cpp. This is included as submodule under third_party
+directory. To init and build this tools, you need to do this under root dirctory:
 
-For java:
 ```
-$ ./initialize_submodule.sh cpp
+$ cd third_party
+$ git submodule update --init -r
+$ cd benchmark && cmake -DCMAKE_BUILD_TYPE=Release && make && cd ../..
 ```
 
+### JAVA
+We're using maven to build the java benchmarks, which is the same as to build 
+the Java protobuf. There're no other tools need to install. We're using 
+[google/caliper](https://github.com/google/caliper) as benchmark tool, which 
+can be automaticly included by maven.
+  
+
+## Run instructions
+
 To run all the benchmark dataset:
 
 For java:
@@ -49,6 +60,8 @@ $ make cpp-benchmark
 $ ./cpp-benchmark $(specific generated dataset file name)
 ```
 
+## Benchmark datasets
+
 There's some big testing data not included in the directory initially, you need to 
 run the following command to download the testing data:
 
@@ -57,10 +70,13 @@ $ ./download_data.sh
 ```
 
 Each data set is in the format of benchmarks.proto:
+
 1. name is the benchmark dataset's name.
 2. message_name is the benchmark's message type full name (including package and message name)
 3. payload is the list of raw data.
 
+The schema for the datasets is described in `benchmarks.proto`.
+
 Benchmark likely want to run several benchmarks against each data set (parse,
 serialize, possibly JSON, possibly using different APIs, etc).
 

benchmarks/initialize_submodule.sh

@@ -1,7 +0,0 @@
-#! /bin/sh
-
-oldpwd=`pwd`
-cd "../third_party"
-git submodule update --init -r
-cd benchmark && cmake -DCMAKE_BUILD_TYPE=Release && make && cd ..
-cd "$oldpwd"

benchmarks/java/pom.xml

@@ -1,85 +1,59 @@
-
 <?xml version="1.0" encoding="UTF-8"?>
-<!--
-  ~ Copyright 2011 Google Inc.
-  ~
-  ~ Licensed under the Apache License, Version 2.0 (the "License");
-  ~ you may not use this file except in compliance with the License.
-  ~ You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-   <modelVersion>4.0.0</modelVersion>
-<!--  <parent>
-	  <groupId>com.google.protobuf</groupId>
-	  <artifactId>protobuf-parent</artifactId>
-	  <version>3.5.0</version>
-  </parent> 
-
-  <packaging>pom</packaging> -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" 
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" 
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  
+  <modelVersion>4.0.0</modelVersion>
   <artifactId>protobuf-java-benchmark</artifactId>
-  <groupId>com.google.protocolbuffers</groupId>
+  <groupId>com.google.protobuf</groupId>
   <version>1.0.0</version>
   <name>Protocol Buffers [Benchmark]</name>
-  <description>
-	  The benchmark tools for Protobuf Java.
-  </description>
-
+  <description>The benchmark tools for Protobuf Java.</description>
+  
   <dependencies>
     <dependency>
-	    <groupId>com.google.protobuf</groupId>
-	    <artifactId>protobuf-java</artifactId>
-	    <version>3.5.0</version>
-	</dependency>
-	<dependency>
-		<groupId>com.google.caliper</groupId>
-	    <artifactId>caliper</artifactId>
-		<version>1.0-beta-2</version>
-	</dependency>
+      <groupId>com.google.protobuf</groupId>
+      <artifactId>protobuf-java</artifactId>
+      <version>3.5.0</version>
+    </dependency>
+    <dependency>
+      <groupId>com.google.caliper</groupId>
+      <artifactId>caliper</artifactId>
+      <version>1.0-beta-2</version>
+    </dependency>
   </dependencies>
-
+  
   <build>
     <pluginManagement>
-	  <plugins>
-		<plugin>
-			<groupId>org.apache.maven.plugins</groupId>
-			<artifactId>maven-assembly-plugin</artifactId>
-			<version>2.4.1</version>
-			<configuration>
-				<!-- get all project dependencies -->
-				<descriptorRefs>
-					<descriptorRef>jar-with-dependencies</descriptorRef>
-				</descriptorRefs>
-				<!-- MainClass in mainfest make a executable jar -->
-				<archive>
-				  <manifest>
-					<mainClass>com.mkyong.core.utils.App</mainClass>
-				  </manifest>
-				</archive>
-
-			</configuration>
-			<executions>
-			  <execution>
-				<id>make-assembly</id>
-                                    <!-- bind to the packaging phase -->
-				<phase>package</phase>
-				<goals>
-					<goal>single</goal>
-				</goals>
-			  </execution>
-			</executions>
-		</plugin>
-	    <plugin>
+      <plugins>
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-assembly-plugin</artifactId>
+          <version>2.4.1</version>
+          <configuration>
+            <!-- get all project dependencies -->
+            <descriptorRefs>
+              <descriptorRef>jar-with-dependencies</descriptorRef>
+            </descriptorRefs>
+            <!-- MainClass in mainfest make a executable jar -->
+            <archive>
+              <manifest>
+                <mainClass>com.mkyong.core.utils.App</mainClass>
+              </manifest>
+            </archive>
+          </configuration>
+          <executions>
+            <execution>
+              <id>make-assembly</id>
+              <!-- bind to the packaging phase -->
+              <phase>package</phase>
+              <goals>
+                <goal>single</goal>
+              </goals>
+            </execution>
+          </executions>
+        </plugin>
+        <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-compiler-plugin</artifactId>
           <version>3.5.1</version>
@@ -114,35 +88,7 @@
             </execution>
           </executions>
         </plugin>
-        <plugin>
-          <groupId>org.apache.maven.plugins</groupId>
-          <artifactId>maven-shade-plugin</artifactId>
-          <version>2.3</version>
-          <executions>
-            <execution>
-              <phase>package</phase>
-              <goals>
-                <goal>shade</goal>
-              </goals>
-              <configuration>
-                <shadedArtifactAttached>true</shadedArtifactAttached>
-                <shadedClassifierName>all</shadedClassifierName>
-                <transformers>
-                  <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
-                    <manifestEntries>
-                      <Premain-Class>com.google.monitoring.runtime.instrumentation.AllocationInstrumenter</Premain-Class>
-                      <Can-Redefine-Classes>true</Can-Redefine-Classes>
-                      <Can-Retransform-Classes>true</Can-Retransform-Classes>
-                    </manifestEntries>
-                  </transformer>
-                </transformers>
-              </configuration>
-            </execution>
-          </executions>
-        </plugin>
       </plugins>
     </pluginManagement>
   </build>
-
-</project>
-
+</project>

benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBench.java → benchmarks/java/src/main/java/com/google/protobuf/ProtoBench.java

@@ -29,7 +29,7 @@
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
-package com.google.protocolbuffers;
+package com.google.protobuf;
 
 import com.google.caliper.BeforeExperiment;
 import com.google.caliper.Benchmark;
@@ -93,7 +93,7 @@ public class ProtoBench {
       return false;
     }
     argsList.add("-DdataFile=" + file);
-    argsList.add("com.google.protocolbuffers.ProtoBenchCaliper");
+    argsList.add("com.google.protobuf.ProtoBenchCaliper");
     
     try {
       String args[] = new String[argsList.size()];
@@ -121,17 +121,9 @@ public class ProtoBench {
     } else if (benchmarkDataset.getMessageName().
         equals("benchmarks.google_message3.GoogleMessage3")) {
       temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE3");
-      for (String opt : ProtoBenchCaliper
-                          .BenchmarkMessageType.GOOGLE_MESSAGE3.getSpecificCaliperOption()) {
-        temp.add(opt);
-      }
     } else if (benchmarkDataset.getMessageName().
         equals("benchmarks.google_message4.GoogleMessage4")) {
       temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE4");
-      for (String opt : ProtoBenchCaliper
-                          .BenchmarkMessageType.GOOGLE_MESSAGE4.getSpecificCaliperOption()) {
-        temp.add(opt);
-      }
     } else {
       return null;
     }
@@ -142,6 +134,7 @@ public class ProtoBench {
     String benchmarkNames = "serializeToByteString,serializeToByteArray,serializeToMemoryStream"
        + ",deserializeFromByteString,deserializeFromByteArray,deserializeFromMemoryStream";
     temp.add(benchmarkNames);
+    temp.add("-Cinstrument.runtime.options.timingInterval=3000ms");
     
     return temp;
   }

benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBenchCaliper.java → benchmarks/java/src/main/java/com/google/protobuf/ProtoBenchCaliper.java

@@ -1,5 +1,5 @@
 
-package com.google.protocolbuffers;
+package com.google.protobuf;
 
 import com.google.caliper.BeforeExperiment;
 import com.google.caliper.Benchmark;
@@ -26,7 +26,6 @@ public class ProtoBenchCaliper {
         return com.google.protobuf.benchmarks.BenchmarkMessage1Proto3.GoogleMessage1
             .getDefaultInstance();
       }
-      @Override String[] getSpecificCaliperOption() { return null; }
     },
     GOOGLE_MESSAGE1_PROTO2 {
       @Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); }
@@ -35,7 +34,6 @@ public class ProtoBenchCaliper {
         return com.google.protobuf.benchmarks.BenchmarkMessage1Proto2.GoogleMessage1
             .getDefaultInstance();
       }
-      @Override String[] getSpecificCaliperOption() { return null; }
     },
     GOOGLE_MESSAGE2 {
       @Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); }
@@ -43,7 +41,6 @@ public class ProtoBenchCaliper {
       Message getDefaultInstance() {
         return com.google.protobuf.benchmarks.BenchmarkMessage2.GoogleMessage2.getDefaultInstance();
       }
-      @Override String[] getSpecificCaliperOption() { return null; }
     },
     GOOGLE_MESSAGE3 {
       @Override
@@ -64,12 +61,6 @@ public class ProtoBenchCaliper {
       Message getDefaultInstance() {
         return benchmarks.google_message3.BenchmarkMessage3.GoogleMessage3.getDefaultInstance();
       }
-      @Override 
-      String[] getSpecificCaliperOption() { 
-        String[] opt = new String[1];
-        opt[0] = "-Cinstrument.runtime.options.timingInterval=3000ms";
-        return opt;
-      }
     },
     GOOGLE_MESSAGE4 {
       @Override
@@ -85,17 +76,10 @@ public class ProtoBenchCaliper {
       Message getDefaultInstance() {
         return benchmarks.google_message4.BenchmarkMessage4.GoogleMessage4.getDefaultInstance();
       }
-      @Override 
-      String[] getSpecificCaliperOption() { 
-        String[] opt = new String[1];
-        opt[0] = "-Cinstrument.runtime.options.timingInterval=3000ms";
-        return opt;
-      }
     };
     
     abstract ExtensionRegistry getExtensionRegistry();
     abstract Message getDefaultInstance();
-    abstract String[] getSpecificCaliperOption();
   }
   
   @Param 
@@ -112,8 +96,6 @@ public class ProtoBenchCaliper {
   private List<ByteString> inputStringList;
   private List<Message> sampleMessageList;
   private int counter;
-  private FileOutputStream devNull;
-  private CodedOutputStream reuseDevNull;
   
   @BeforeExperiment
   void setUp() throws IOException {
@@ -135,15 +117,6 @@ public class ProtoBenchCaliper {
       sampleMessageList.add(
           defaultMessage.newBuilderForType().mergeFrom(singleInputData, extensions).build());
     }
-    devNull = null;
-    reuseDevNull = null;
-    
-    try {
-      devNull = new FileOutputStream("/dev/null");
-      reuseDevNull = CodedOutputStream.newInstance(devNull);
-    } catch (FileNotFoundException e) {
-      // ignore: this is probably Windows, where /dev/null does not exist
-    }
     
     counter = 0;
   }

tests.sh

@@ -44,7 +44,10 @@ build_cpp() {
   # appears to be missing it: https://github.com/travis-ci/travis-ci/issues/6996
   if [[ $(type cmake 2>/dev/null) ]]; then
     # Verify benchmarking code can build successfully.
-	cd benchmarks && ./initialize_submodule.sh cpp && make cpp-benchmark && cd ..
+    git submodule init
+    git submodule update
+    cd third_party/benchmark && cmake -DCMAKE_BUILD_TYPE=Release && make && cd ../..
+    cd benchmarks && make cpp-benchmark && cd ..
   else
     echo ""
     echo "WARNING: Skipping validation of the bench marking code, cmake isn't installed."