Merge pull request #4132 from BSBandme/JavaCaliperCounter

Java caliper counter

benchmarks/java/src/main/java/com/google/protobuf/ProtoBench.java

@@ -43,14 +43,18 @@ import com.google.protobuf.Message;
 import com.google.protobuf.benchmarks.Benchmarks.BenchmarkDataset;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
+import java.io.EOFException;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.io.RandomAccessFile;
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Scanner;
 
 
 public class ProtoBench {
@@ -71,6 +75,7 @@ public class ProtoBench {
     }
     System.exit(success ? 0 : 1);
   }
+  
  
   /**
    * Runs a single test with specific test data. Error messages are displayed to stderr, 
@@ -86,7 +91,6 @@ public class ProtoBench {
       System.err.println("Unable to get input data");
       return false;
     }
-    
     List<String> argsList = getCaliperOption(benchmarkDataset);
     if (argsList == null) {
       System.err.println("Unable to get default message " + benchmarkDataset.getMessageName());
@@ -98,15 +102,40 @@ public class ProtoBench {
     try {
       String args[] = new String[argsList.size()];
       argsList.toArray(args);
-      CaliperMain.exitlessMain(args, 
+      CaliperMain.exitlessMain(args,
           new PrintWriter(System.out, true), new PrintWriter(System.err, true));
-      return true;
     } catch (Exception e) {
       System.err.println("Error: " + e.getMessage());
       System.err.println("Detailed exception information:");
       e.printStackTrace(System.err);
       return false;
     }
+    try {
+      double mininumScale = 0;
+      // If the file not exist, this will throw IOException, which won't print the warning 
+      // information below.
+      Scanner scanner = new Scanner(new String(readAllBytes("JavaBenchmarkWarning.txt")));
+      while (scanner.hasNext()) {
+        mininumScale = Math.max(mininumScale, scanner.nextDouble());
+      }
+      scanner.close();
+      
+      System.out.println(
+         "WARNING: This benchmark's whole iterations are not enough, consider to config caliper to "
+        + "run for more time to make the result more convincing. You may change the configure "
+        + "code in com.google.protobuf.ProtoBench.getCaliperOption() of benchmark " 
+        + benchmarkDataset.getMessageName()
+        + " to run for more time. e.g. Change the value of "
+        + "instrument.runtime.options.timingInterval or value of "
+        + "instrument.runtime.options.measurements to be at least "
+        + Math.round(mininumScale * 10 + 1) / 10.0 
+        + " times of before, then build and run the benchmark again\n");
+      Files.deleteIfExists(Paths.get("JavaBenchmarkWarning.txt"));
+    } catch (IOException e) {
+      // The IOException here should be file not found, which means there's no warning generated by
+      // The benchmark, so this IOException should be discarded.
+    }
+    return true;
   }
   
   
@@ -121,9 +150,13 @@ public class ProtoBench {
     } else if (benchmarkDataset.getMessageName().
         equals("benchmarks.google_message3.GoogleMessage3")) {
       temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE3");
+      temp.add("-Cinstrument.runtime.options.timingInterval=3000ms");
+      temp.add("-Cinstrument.runtime.options.measurements=20");
     } else if (benchmarkDataset.getMessageName().
         equals("benchmarks.google_message4.GoogleMessage4")) {
       temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE4");
+      temp.add("-Cinstrument.runtime.options.timingInterval=1500ms");
+      temp.add("-Cinstrument.runtime.options.measurements=20");
     } else {
       return null;
     }
@@ -134,7 +167,6 @@ public class ProtoBench {
     String benchmarkNames = "serializeToByteString,serializeToByteArray,serializeToMemoryStream"
        + ",deserializeFromByteString,deserializeFromByteArray,deserializeFromMemoryStream";
     temp.add(benchmarkNames);
-    temp.add("-Cinstrument.runtime.options.timingInterval=3000ms");
     
     return temp;
   }

benchmarks/java/src/main/java/com/google/protobuf/ProtoBenchCaliper.java

@@ -2,6 +2,7 @@
 package com.google.protobuf;
 
 import com.google.caliper.BeforeExperiment;
+import com.google.caliper.AfterExperiment;
 import com.google.caliper.Benchmark;
 import com.google.caliper.Param;
 import com.google.protobuf.ByteString;
@@ -11,8 +12,10 @@ import com.google.protobuf.Message;
 import com.google.protobuf.benchmarks.Benchmarks.BenchmarkDataset;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
+import java.io.BufferedWriter;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
+import java.io.FileWriter;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
@@ -95,7 +98,7 @@ public class ProtoBenchCaliper {
   private List<ByteArrayInputStream> inputStreamList;
   private List<ByteString> inputStringList;
   private List<Message> sampleMessageList;
-  private int counter;
+  private long counter;
   
   @BeforeExperiment
   void setUp() throws IOException {
@@ -125,7 +128,7 @@ public class ProtoBenchCaliper {
   @Benchmark
   void serializeToByteString(int reps) throws IOException {
     for (int i = 0; i < reps; i++) {
-      sampleMessageList.get(counter % sampleMessageList.size()).toByteString();
+      sampleMessageList.get((int) (counter % sampleMessageList.size())).toByteString();
       counter++;
     }
   }
@@ -133,7 +136,7 @@ public class ProtoBenchCaliper {
   @Benchmark
   void serializeToByteArray(int reps) throws IOException {
     for (int i = 0; i < reps; i++) {
-      sampleMessageList.get(counter % sampleMessageList.size()).toByteArray();
+      sampleMessageList.get((int) (counter % sampleMessageList.size())).toByteArray();
       counter++;
     }
   }
@@ -142,7 +145,7 @@ public class ProtoBenchCaliper {
   void serializeToMemoryStream(int reps) throws IOException {
     for (int i = 0; i < reps; i++) {
       ByteArrayOutputStream output = new ByteArrayOutputStream();
-      sampleMessageList.get(counter % sampleMessageList.size()).writeTo(output);
+      sampleMessageList.get((int) (counter % sampleMessageList.size())).writeTo(output);
       counter++;
     }
   }
@@ -152,7 +155,7 @@ public class ProtoBenchCaliper {
     for (int i = 0; i < reps; i++) {
       defaultMessage
         .newBuilderForType()
-        .mergeFrom(inputStringList.get(counter % inputStringList.size()), extensions)
+        .mergeFrom(inputStringList.get((int) (counter % inputStringList.size())), extensions)
         .build();
       counter++;
     }
@@ -163,7 +166,7 @@ public class ProtoBenchCaliper {
     for (int i = 0; i < reps; i++) {
       defaultMessage
         .newBuilderForType()
-        .mergeFrom(inputDataList.get(counter % inputDataList.size()), extensions)
+        .mergeFrom(inputDataList.get((int) (counter % inputDataList.size())), extensions)
         .build();
       counter++;
     }
@@ -174,10 +177,27 @@ public class ProtoBenchCaliper {
     for (int i = 0; i < reps; i++) {
       defaultMessage
         .newBuilderForType()
-        .mergeFrom(inputStreamList.get(counter % inputStreamList.size()), extensions)
+        .mergeFrom(inputStreamList.get((int) (counter % inputStreamList.size())), extensions)
         .build();
-      inputStreamList.get(counter % inputStreamList.size()).reset();
+      inputStreamList.get((int) (counter % inputStreamList.size())).reset();
       counter++;
     }
   }
+  
+  @AfterExperiment
+  void checkCounter() throws IOException {
+    if (counter == 1) {
+      // Dry run
+      return;
+    }
+    if (benchmarkDataset.getPayloadCount() != 1 
+        && counter < benchmarkDataset.getPayloadCount() * 10L) {
+      BufferedWriter writer = new BufferedWriter(new FileWriter("JavaBenchmarkWarning.txt", true));
+      // If the total number of non-warmup reps is smaller than 100 times of the total number of 
+      // datasets, then output the scale that need to multiply to the configuration (either extend
+      // the running time for one timingInterval or run for more measurements). 
+      writer.append(1.0 * benchmarkDataset.getPayloadCount() * 10L / counter + " ");
+      writer.close();
+    }
+  }
 }