Merge branch 'sync-piper' into sync-stage

conformance/binary_json_conformance_suite.h

@@ -53,35 +53,34 @@ class BinaryAndJsonConformanceSuite : public ConformanceTestSuite {
   void RunJsonTestsForStruct();
   void RunJsonTestsForValue();
   void RunJsonTestsForAny();
-  void RunValidJsonTest(const string& test_name,
-                        ConformanceLevel level,
-                        const string& input_json,
-                        const string& equivalent_text_format);
+  void RunValidJsonTest(const std::string& test_name, ConformanceLevel level,
+                        const std::string& input_json,
+                        const std::string& equivalent_text_format);
   void RunValidJsonTestWithProtobufInput(
-      const string& test_name,
-      ConformanceLevel level,
+      const std::string& test_name, ConformanceLevel level,
       const protobuf_test_messages::proto3::TestAllTypesProto3& input,
-      const string& equivalent_text_format);
-  void RunValidJsonIgnoreUnknownTest(
-      const string& test_name, ConformanceLevel level, const string& input_json,
-      const string& equivalent_text_format);
-  void RunValidProtobufTest(const string& test_name, ConformanceLevel level,
-                            const string& input_protobuf,
-                            const string& equivalent_text_format,
+      const std::string& equivalent_text_format);
+  void RunValidJsonIgnoreUnknownTest(const std::string& test_name,
+                                     ConformanceLevel level,
+                                     const std::string& input_json,
+                                     const std::string& equivalent_text_format);
+  void RunValidProtobufTest(const std::string& test_name,
+                            ConformanceLevel level,
+                            const std::string& input_protobuf,
+                            const std::string& equivalent_text_format,
                             bool is_proto3);
-  void RunValidBinaryProtobufTest(const string& test_name,
+  void RunValidBinaryProtobufTest(const std::string& test_name,
                                   ConformanceLevel level,
-                                  const string& input_protobuf,
+                                  const std::string& input_protobuf,
                                   bool is_proto3);
-  void RunValidBinaryProtobufTest(const string& test_name,
+  void RunValidBinaryProtobufTest(const std::string& test_name,
                                   ConformanceLevel level,
-                                  const string& input_protobuf,
-                                  const string& expected_protobuf,
+                                  const std::string& input_protobuf,
+                                  const std::string& expected_protobuf,
                                   bool is_proto3);
   void RunValidProtobufTestWithMessage(
-      const string& test_name, ConformanceLevel level,
-      const Message *input,
-      const string& equivalent_text_format,
+      const std::string& test_name, ConformanceLevel level,
+      const Message* input, const std::string& equivalent_text_format,
       bool is_proto3);
 
   bool ParseJsonResponse(
@@ -93,20 +92,20 @@ class BinaryAndJsonConformanceSuite : public ConformanceTestSuite {
       Message* test_message) override;
 
   typedef std::function<bool(const Json::Value&)> Validator;
-  void RunValidJsonTestWithValidator(const string& test_name,
+  void RunValidJsonTestWithValidator(const std::string& test_name,
                                      ConformanceLevel level,
-                                     const string& input_json,
+                                     const std::string& input_json,
                                      const Validator& validator);
-  void ExpectParseFailureForJson(const string& test_name,
+  void ExpectParseFailureForJson(const std::string& test_name,
                                  ConformanceLevel level,
-                                 const string& input_json);
-  void ExpectSerializeFailureForJson(const string& test_name,
+                                 const std::string& input_json);
+  void ExpectSerializeFailureForJson(const std::string& test_name,
                                      ConformanceLevel level,
-                                     const string& text_format);
-  void ExpectParseFailureForProtoWithProtoVersion (const string& proto,
-                                                   const string& test_name,
-                                                   ConformanceLevel level,
-                                                   bool is_proto3);
+                                     const std::string& text_format);
+  void ExpectParseFailureForProtoWithProtoVersion(const std::string& proto,
+                                                  const std::string& test_name,
+                                                  ConformanceLevel level,
+                                                  bool is_proto3);
   void ExpectParseFailureForProto(const std::string& proto,
                                   const std::string& test_name,
                                   ConformanceLevel level);

conformance/conformance_test.h

@@ -88,7 +88,7 @@ class ForkPipeRunner : public ConformanceTestRunner {
                  const std::vector<ConformanceTestSuite*>& suites);
 
   ForkPipeRunner(const std::string& executable,
-                 const std::vector<string>& executable_args)
+                 const std::vector<std::string>& executable_args)
       : child_pid_(-1),
         executable_(executable),
         executable_args_(executable_args) {}
@@ -113,7 +113,7 @@ class ForkPipeRunner : public ConformanceTestRunner {
   int read_fd_;
   pid_t child_pid_;
   std::string executable_;
-  const std::vector<string> executable_args_;
+  const std::vector<std::string> executable_args_;
   std::string current_test_name_;
 };
 
@@ -168,9 +168,7 @@ class ConformanceTestSuite {
 
   // Gets the flag name to the failure list file.
   // By default, this would return --failure_list
-  string GetFailureListFlagName() {
-    return failure_list_flag_name_;
-  }
+  std::string GetFailureListFlagName() { return failure_list_flag_name_; }
 
   void SetFailureListFlagName(const std::string& failure_list_flag_name) {
     failure_list_flag_name_ = failure_list_flag_name;
@@ -207,18 +205,18 @@ class ConformanceTestSuite {
 
   class ConformanceRequestSetting {
    public:
-    ConformanceRequestSetting(
-        ConformanceLevel level,
-        conformance::WireFormat input_format,
-        conformance::WireFormat output_format,
-        conformance::TestCategory test_category,
-        const Message& prototype_message,
-        const string& test_name, const string& input);
+    ConformanceRequestSetting(ConformanceLevel level,
+                              conformance::WireFormat input_format,
+                              conformance::WireFormat output_format,
+                              conformance::TestCategory test_category,
+                              const Message& prototype_message,
+                              const std::string& test_name,
+                              const std::string& input);
     virtual ~ConformanceRequestSetting() {}
 
     std::unique_ptr<Message> NewTestMessage() const;
 
-    string GetTestName() const;
+    std::string GetTestName() const;
 
     const conformance::ConformanceRequest& GetRequest() const {
       return request_;
@@ -228,7 +226,7 @@ class ConformanceTestSuite {
       return level_;
     }
 
-    string ConformanceLevelToString(ConformanceLevel level) const;
+    std::string ConformanceLevelToString(ConformanceLevel level) const;
 
     void SetPrintUnknownFields(bool print_unknown_fields) {
       request_.set_print_unknown_fields(true);
@@ -239,8 +237,9 @@ class ConformanceTestSuite {
     }
 
    protected:
-    virtual string InputFormatString(conformance::WireFormat format) const;
-    virtual string OutputFormatString(conformance::WireFormat format) const;
+    virtual std::string InputFormatString(conformance::WireFormat format) const;
+    virtual std::string OutputFormatString(
+        conformance::WireFormat format) const;
     conformance::ConformanceRequest request_;
 
    private:
@@ -249,12 +248,12 @@ class ConformanceTestSuite {
     ::conformance::WireFormat output_format_;
     const Message& prototype_message_;
     std::unique_ptr<Message> prototype_message_for_compare_;
-    string test_name_;
+    std::string test_name_;
   };
 
-  bool CheckSetEmpty(const std::set<string>& set_to_check,
+  bool CheckSetEmpty(const std::set<std::string>& set_to_check,
                      const std::string& write_to_file, const std::string& msg);
-  string WireFormatToString(conformance::WireFormat wire_format);
+  std::string WireFormatToString(conformance::WireFormat wire_format);
 
   // Parse payload in the response to the given message. Returns true on
   // success.
@@ -264,24 +263,23 @@ class ConformanceTestSuite {
       Message* test_message) = 0;
 
   void VerifyResponse(const ConformanceRequestSetting& setting,
-                      const string& equivalent_wire_format,
+                      const std::string& equivalent_wire_format,
                       const conformance::ConformanceResponse& response,
                       bool need_report_success, bool require_same_wire_format);
 
   void ReportSuccess(const std::string& test_name);
-  void ReportFailure(const string& test_name,
-                     ConformanceLevel level,
+  void ReportFailure(const std::string& test_name, ConformanceLevel level,
                      const conformance::ConformanceRequest& request,
                      const conformance::ConformanceResponse& response,
                      const char* fmt, ...);
-  void ReportSkip(const string& test_name,
+  void ReportSkip(const std::string& test_name,
                   const conformance::ConformanceRequest& request,
                   const conformance::ConformanceResponse& response);
 
   void RunValidInputTest(const ConformanceRequestSetting& setting,
-                         const string& equivalent_text_format);
+                         const std::string& equivalent_text_format);
   void RunValidBinaryInputTest(const ConformanceRequestSetting& setting,
-                               const string& equivalent_wire_format,
+                               const std::string& equivalent_wire_format,
                                bool require_same_wire_format = false);
 
   void RunTest(const std::string& test_name,

conformance/text_format_conformance_suite.h

@@ -42,19 +42,19 @@ class TextFormatConformanceTestSuite : public ConformanceTestSuite {
 
  private:
   void RunSuiteImpl();
-  void RunValidTextFormatTest(const string& test_name, ConformanceLevel level,
-                              const string& input);
-  void RunValidTextFormatTestProto2(const string& test_name,
+  void RunValidTextFormatTest(const std::string& test_name,
+                              ConformanceLevel level, const std::string& input);
+  void RunValidTextFormatTestProto2(const std::string& test_name,
                                     ConformanceLevel level,
-                                    const string& input);
-  void RunValidTextFormatTestWithMessage(const string& test_name,
+                                    const std::string& input);
+  void RunValidTextFormatTestWithMessage(const std::string& test_name,
                                          ConformanceLevel level,
-                                         const string& input_text,
+                                         const std::string& input_text,
                                          const Message& prototype);
-  void RunValidUnknownTextFormatTest(const string& test_name,
+  void RunValidUnknownTextFormatTest(const std::string& test_name,
                                      const Message& message);
-  void ExpectParseFailure(const string& test_name, ConformanceLevel level,
-                          const string& input);
+  void ExpectParseFailure(const std::string& test_name, ConformanceLevel level,
+                          const std::string& input);
   bool ParseTextFormatResponse(const conformance::ConformanceResponse& response,
                                const ConformanceRequestSetting& setting,
                                Message* test_message);

java/core/src/main/java/com/google/protobuf/Descriptors.java

@@ -702,6 +702,11 @@ public final class Descriptors {
       return Collections.unmodifiableList(Arrays.asList(oneofs));
     }
 
+    /** Get a list of this message type's real oneofs. */
+    public List<OneofDescriptor> getRealOneofs() {
+      return Collections.unmodifiableList(Arrays.asList(oneofs).subList(0, realOneofCount));
+    }
+
     /** Get a list of this message type's extensions. */
     public List<FieldDescriptor> getExtensions() {
       return Collections.unmodifiableList(Arrays.asList(extensions));
@@ -821,6 +826,7 @@ public final class Descriptors {
     private final FieldDescriptor[] fields;
     private final FieldDescriptor[] extensions;
     private final OneofDescriptor[] oneofs;
+    private final int realOneofCount;
 
     // Used to create a placeholder when the type cannot be found.
     Descriptor(final String fullname) throws DescriptorValidationException {
@@ -846,6 +852,7 @@ public final class Descriptors {
       this.fields = new FieldDescriptor[0];
       this.extensions = new FieldDescriptor[0];
       this.oneofs = new OneofDescriptor[0];
+      this.realOneofCount = 0;
 
       // Create a placeholder FileDescriptor to hold this message.
       this.file = new FileDescriptor(packageName, this);
@@ -899,6 +906,18 @@ public final class Descriptors {
         }
       }
 
+      int syntheticOneofCount = 0;
+      for (OneofDescriptor oneof : this.oneofs) {
+        if (oneof.isSynthetic()) {
+          syntheticOneofCount++;
+        } else {
+          if (syntheticOneofCount > 0) {
+            throw new DescriptorValidationException(this, "Synthetic oneofs must come last.");
+          }
+        }
+      }
+      this.realOneofCount = this.oneofs.length - syntheticOneofCount;
+
       file.pool.addSymbol(this);
     }
 
@@ -1125,6 +1144,11 @@ public final class Descriptors {
       return containingOneof;
     }
 
+    /** Get the field's containing oneof, only if non-synthetic. */
+    public OneofDescriptor getRealContainingOneof() {
+      return containingOneof != null && !containingOneof.isSynthetic() ? containingOneof : null;
+    }
+
     /**
      * Returns true if this field was syntactically written with "optional" in the .proto file.
      * Excludes singular proto3 fields that do not have a label.
@@ -1135,22 +1159,23 @@ public final class Descriptors {
     }
 
     /**
-     * Returns true if this is a non-oneof field that tracks presence.
+     * Returns true if this field tracks presence, ie. does the field distinguish between "unset"
+     * and "present with default value."
      *
-     * <p>This includes all "required" and "optional" fields in the .proto file, but excludes oneof
-     * fields and singular proto3 fields without "optional".
+     * <p>This includes required, optional, and oneof fields. It excludes maps, repeated fields, and
+     * singular proto3 fields without "optional".
      *
-     * <p>In implementations that use hasbits, this method will probably indicate whether this field
-     * uses a hasbit.
+     * <p>For fields where hasPresence() == true, the return value of msg.hasField() is semantically
+     * meaningful.
      */
-    boolean isSingularWithPresence() {
+    boolean hasPresence() {
       if (isRepeated()) {
         return false;
       }
-      if (getContainingOneof() != null && !getContainingOneof().isSynthetic()) {
-        return false;
-      }
-      return getType() == Type.MESSAGE || isProto3Optional || file.getSyntax() == Syntax.PROTO2;
+      return getType() == Type.MESSAGE
+          || getType() == Type.GROUP
+          || getContainingOneof() != null
+          || file.getSyntax() == Syntax.PROTO2;
     }
 
     /**

python/google/protobuf/internal/text_format_test.py

@@ -57,6 +57,7 @@ from google.protobuf import unittest_proto3_arena_pb2
 from google.protobuf import descriptor_pb2
 from google.protobuf.internal import any_test_pb2 as test_extend_any
 from google.protobuf.internal import message_set_extensions_pb2
+from google.protobuf.internal import test_proto3_optional_pb2
 from google.protobuf.internal import test_util
 from google.protobuf import descriptor_pool
 from google.protobuf import text_format
@@ -176,14 +177,12 @@ class TextFormatMessageToStringTests(TextFormatBase):
     self.CompareToGoldenText(
         self.RemoveRedundantZeros(text_format.MessageToString(message)),
         'repeated_float: 0\n'
-        # This should be 0.8
-        'repeated_float: 0.80000001\n'
+        'repeated_float: 0.8\n'
         'repeated_float: 1\n'
         'repeated_float: 1.2\n'
         'repeated_float: 1.23\n'
         'repeated_float: 1.234\n'
-        # This should be 1.2345
-        'repeated_float: 1.2345001\n'
+        'repeated_float: 1.2345\n'
         'repeated_float: 1.23456\n'
         # Note that these don't use scientific notation.
         'repeated_float: 12000000000\n'
@@ -400,8 +399,7 @@ class TextFormatMessageToStringTests(TextFormatBase):
     # 32-bit 1.2 is noisy when extended to 64-bit:
     #  >>> struct.unpack('f', struct.pack('f', 1.2))[0]
     #  1.2000000476837158
-    # TODO(jieluo): change to 1.2 with cl/241634942.
-    message.payload.optional_float = 1.2000000476837158
+    message.payload.optional_float = 1.2
     formatted_fields = ['optional_float: 1.2',
                         'optional_double: -3.45678901234568e-6',
                         'repeated_float: -5642', 'repeated_double: 7.89e-5']
@@ -422,7 +420,7 @@ class TextFormatMessageToStringTests(TextFormatBase):
         'payload {{\n  {0}\n  {1}\n  {2}\n  {3}\n}}\n'.format(
             *formatted_fields))
 
-    # Test default float_format has 8 valid digits.
+    # Test default float_format will automatic print shortest float.
     message.payload.optional_float = 1.2345678912
     message.payload.optional_double = 1.2345678912
     formatted_fields = ['optional_float: 1.2345679',
@@ -434,6 +432,17 @@ class TextFormatMessageToStringTests(TextFormatBase):
         'payload {{\n  {0}\n  {1}\n  {2}\n  {3}\n}}\n'.format(
             *formatted_fields))
 
+    message.Clear()
+    message.payload.optional_float = 1.1000000000011
+    self.assertEqual(text_format.MessageToString(message),
+                     'payload {\n  optional_float: 1.1\n}\n')
+    message.payload.optional_float = 1.00000075e-36
+    self.assertEqual(text_format.MessageToString(message),
+                     'payload {\n  optional_float: 1.00000075e-36\n}\n')
+    message.payload.optional_float = 12345678912345e+11
+    self.assertEqual(text_format.MessageToString(message),
+                     'payload {\n  optional_float: 1.234568e+24\n}\n')
+
   def testMessageToString(self, message_module):
     message = message_module.ForeignMessage()
     message.c = 123
@@ -1850,6 +1859,24 @@ class Proto3Tests(unittest.TestCase):
       text_format.Merge(text, message)
     self.assertEqual(str(e.exception), '3:11 : Expected "}".')
 
+  def testProto3Optional(self):
+    msg = test_proto3_optional_pb2.TestProto3Optional()
+    self.assertEqual(text_format.MessageToString(msg), '')
+    msg.optional_int32 = 0
+    msg.optional_float = 0.0
+    msg.optional_string = ''
+    msg.optional_nested_message.bb = 0
+    text = ('optional_int32: 0\n'
+            'optional_float: 0.0\n'
+            'optional_string: ""\n'
+            'optional_nested_message {\n'
+            '  bb: 0\n'
+            '}\n')
+    self.assertEqual(text_format.MessageToString(msg), text)
+    msg2 = test_proto3_optional_pb2.TestProto3Optional()
+    text_format.Parse(text, msg2)
+    self.assertEqual(text_format.MessageToString(msg2), text)
+
 
 class TokenizerTest(unittest.TestCase):
 

python/google/protobuf/internal/type_checkers.py

@@ -72,6 +72,26 @@ def TruncateToFourByteFloat(original):
     return struct.unpack('<f', struct.pack('<f', original))[0]
 
 
+def ToShortestFloat(original):
+  """Returns the shortest float that has same value in wire."""
+  # Return the original value if it is not truncated. This may happen
+  # if someone mixes this code with an old protobuf runtime.
+  # TODO(jieluo): Remove it after maybe 2022.
+  if TruncateToFourByteFloat(original) != original:
+    return original
+  # All 4 byte floats have between 6 and 9 significant digits, so we
+  # start with 6 as the lower bound.
+  # It has to be iterative because use '.9g' directly can not get rid
+  # of the noises for most values. For example if set a float_field=0.9
+  # use '.9g' will print 0.899999976.
+  precision = 6
+  rounded = float('{0:.{1}g}'.format(original, precision))
+  while TruncateToFourByteFloat(rounded) != original:
+    precision += 1
+    rounded = float('{0:.{1}g}'.format(original, precision))
+  return rounded
+
+
 def SupportsOpenEnums(field_descriptor):
   return field_descriptor.containing_type.syntax == "proto3"
 

python/google/protobuf/json_format.py

@@ -283,25 +283,6 @@ class _Printer(object):
 
   def _FieldToJsonObject(self, field, value):
     """Converts field value according to Proto3 JSON Specification."""
-
-    def _ToShortestFloat(original):
-      """Returns the shortest float that has same value in wire."""
-      # Return the original value if it is not truncated. This may happen
-      # if someone mixes this code with an old protobuf runtime.
-      if type_checkers.TruncateToFourByteFloat(original) != original:
-        return original
-      # All 4 byte floats have between 6 and 9 significant digits, so we
-      # start with 6 as the lower bound.
-      # It has to be iterative because use '.9g' directly can not get rid
-      # of the noises for most values. For example if set a float_field=0.9
-      # use '.9g' will print 0.899999976.
-      precision = 6
-      rounded = float('{0:.{1}g}'.format(original, precision))
-      while type_checkers.TruncateToFourByteFloat(rounded) != original:
-        precision += 1
-        rounded = float('{0:.{1}g}'.format(original, precision))
-      return rounded
-
     if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
       return self._MessageToJsonObject(value)
     elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM:
@@ -337,7 +318,7 @@ class _Printer(object):
         if self.float_format:
           return float(format(value, self.float_format))
         else:
-          return _ToShortestFloat(value)
+          return type_checkers.ToShortestFloat(value)
 
     return value
 

python/google/protobuf/text_format.py

@@ -158,9 +158,9 @@ def MessageToString(
       determined by the extension number. By default, use the field number
       order.
     float_format (str): If set, use this to specify float field formatting
-      (per the "Format Specification Mini-Language"); otherwise, 8 valid
-      digits is used (default '.8g'). Also affect double field if
-      double_format is not set but float_format is set.
+      (per the "Format Specification Mini-Language"); otherwise, shortest float
+      that has same value in wire will be printed. Also affect double field
+      if double_format is not set but float_format is set.
     double_format (str): If set, use this to specify double field formatting
       (per the "Format Specification Mini-Language"); if it is not set but
       float_format is set, use float_format. Otherwise, use ``str()``
@@ -367,9 +367,9 @@ class _Printer(object):
         defined in source code instead of the field number. By default, use the
         field number order.
       float_format: If set, use this to specify float field formatting
-        (per the "Format Specification Mini-Language"); otherwise, 8 valid
-        digits is used (default '.8g'). Also affect double field if
-        double_format is not set but float_format is set.
+        (per the "Format Specification Mini-Language"); otherwise, shortest
+        float that has same value in wire will be printed. Also affect double
+        field if double_format is not set but float_format is set.
       double_format: If set, use this to specify double field formatting
         (per the "Format Specification Mini-Language"); if it is not set but
         float_format is set, use float_format. Otherwise, str() is used.
@@ -630,7 +630,7 @@ class _Printer(object):
       if self.float_format is not None:
         out.write('{1:{0}}'.format(self.float_format, value))
       else:
-        out.write(str(float(format(value, '.8g'))))
+        out.write(str(type_checkers.ToShortestFloat(value)))
     elif (field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_DOUBLE and
           self.double_format is not None):
       out.write('{1:{0}}'.format(self.double_format, value))

src/google/protobuf/arena_unittest.cc

@@ -58,6 +58,9 @@
 #include <google/protobuf/stubs/strutil.h>
 
 
+// Must be included last
+#include <google/protobuf/port_def.inc>
+
 using proto2_arena_unittest::ArenaMessage;
 using protobuf_unittest::TestAllExtensions;
 using protobuf_unittest::TestAllTypes;

src/google/protobuf/compiler/cpp/cpp_map_field.cc

@@ -29,6 +29,7 @@
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <google/protobuf/compiler/cpp/cpp_map_field.h>
+
 #include <google/protobuf/compiler/cpp/cpp_helpers.h>
 #include <google/protobuf/io/printer.h>
 #include <google/protobuf/wire_format.h>

src/google/protobuf/compiler/java/java_enum.cc

@@ -142,9 +142,13 @@ void EnumGenerator::Generate(io::Printer* printer) {
     vars["number"] = StrCat(descriptor_->value(i)->number());
     vars["{"] = "";
     vars["}"] = "";
+    vars["deprecation"] = descriptor_->value(i)->options().deprecated()
+                              ? "@java.lang.Deprecated "
+                              : "";
     WriteEnumValueDocComment(printer, descriptor_->value(i));
     printer->Print(vars,
-                   "public static final int ${$$name$_VALUE$}$ = $number$;\n");
+                   "$deprecation$public static final int ${$$name$_VALUE$}$ = "
+                   "$number$;\n");
     printer->Annotate("{", "}", descriptor_->value(i));
   }
   printer->Print("\n");

src/google/protobuf/compiler/java/java_enum_lite.cc

@@ -124,9 +124,13 @@ void EnumLiteGenerator::Generate(io::Printer* printer) {
     vars["number"] = StrCat(descriptor_->value(i)->number());
     vars["{"] = "";
     vars["}"] = "";
+    vars["deprecation"] = descriptor_->value(i)->options().deprecated()
+                              ? "@java.lang.Deprecated "
+                              : "";
     WriteEnumValueDocComment(printer, descriptor_->value(i));
     printer->Print(vars,
-                   "public static final int ${$$name$_VALUE$}$ = $number$;\n");
+                   "$deprecation$public static final int ${$$name$_VALUE$}$ = "
+                   "$number$;\n");
     printer->Annotate("{", "}", descriptor_->value(i));
   }
   printer->Print("\n");

src/google/protobuf/compiler/java/java_field.cc

@@ -251,6 +251,7 @@ void SetCommonFieldVariables(const FieldDescriptor* descriptor,
   (*variables)["disambiguated_reason"] = info->disambiguated_reason;
   (*variables)["constant_name"] = FieldConstantName(descriptor);
   (*variables)["number"] = StrCat(descriptor->number());
+  (*variables)["kt_dsl_builder"] = "_builder";
   // These variables are placeholders to pick out the beginning and ends of
   // identifiers for annotations (when doing so with existing variables would
   // be ambiguous or impossible). They should never be set to anything but the

src/google/protobuf/compiler/java/java_helpers.cc

@@ -428,6 +428,7 @@ const char* BoxedPrimitiveTypeName(const FieldDescriptor* descriptor) {
   return BoxedPrimitiveTypeName(GetJavaType(descriptor));
 }
 
+
 std::string GetOneofStoredType(const FieldDescriptor* field) {
   const JavaType javaType = GetJavaType(field);
   switch (javaType) {

src/google/protobuf/compiler/java/java_helpers.h

@@ -226,6 +226,7 @@ const char* PrimitiveTypeName(JavaType type);
 // types.
 const char* BoxedPrimitiveTypeName(JavaType type);
 
+
 // Get the name of the java enum constant representing this type. E.g.,
 // "INT32" for FieldDescriptor::TYPE_INT32. The enum constant's full
 // name is "com.google.protobuf.WireFormat.FieldType.INT32".

src/google/protobuf/compiler/java/java_primitive_field.cc

@@ -296,6 +296,7 @@ void ImmutablePrimitiveFieldGenerator::GenerateBuilderMembers(
                  "}\n");
 }
 
+
 void ImmutablePrimitiveFieldGenerator::GenerateFieldBuilderInitializationCode(
     io::Printer* printer) const {
   // noop for primitives

src/google/protobuf/compiler/java/java_primitive_field_lite.cc

@@ -32,6 +32,8 @@
 //  Based on original Protocol Buffers design by
 //  Sanjay Ghemawat, Jeff Dean, and others.
 
+#include <google/protobuf/compiler/java/java_primitive_field_lite.h>
+
 #include <map>
 #include <string>
 
@@ -41,7 +43,6 @@
 #include <google/protobuf/compiler/java/java_doc_comment.h>
 #include <google/protobuf/compiler/java/java_helpers.h>
 #include <google/protobuf/compiler/java/java_name_resolver.h>
-#include <google/protobuf/compiler/java/java_primitive_field_lite.h>
 #include <google/protobuf/io/printer.h>
 #include <google/protobuf/wire_format.h>
 #include <google/protobuf/stubs/strutil.h>
@@ -300,6 +301,7 @@ void ImmutablePrimitiveFieldLiteGenerator::GenerateBuilderMembers(
   printer->Annotate("{", "}", descriptor_);
 }
 
+
 void ImmutablePrimitiveFieldLiteGenerator::GenerateFieldInfo(
     io::Printer* printer, std::vector<uint16>* output) const {
   WriteIntToUtf16CharSequence(descriptor_->number(), output);

src/google/protobuf/descriptor.cc

@@ -1720,6 +1720,18 @@ const EnumValueDescriptor* Descriptor::FindEnumValueByName(
   }
 }
 
+const FieldDescriptor* Descriptor::map_key() const {
+  if (!options().map_entry()) return nullptr;
+  GOOGLE_DCHECK_EQ(field_count(), 2);
+  return field(0);
+}
+
+const FieldDescriptor* Descriptor::map_value() const {
+  if (!options().map_entry()) return nullptr;
+  GOOGLE_DCHECK_EQ(field_count(), 2);
+  return field(1);
+}
+
 const EnumValueDescriptor* EnumDescriptor::FindValueByName(
     const std::string& key) const {
   Symbol result =
@@ -6222,8 +6234,8 @@ bool DescriptorBuilder::ValidateMapEntry(FieldDescriptor* field,
     return false;
   }
 
-  const FieldDescriptor* key = message->field(0);
-  const FieldDescriptor* value = message->field(1);
+  const FieldDescriptor* key = message->map_key();
+  const FieldDescriptor* value = message->map_value();
   if (key->label() != FieldDescriptor::LABEL_OPTIONAL || key->number() != 1 ||
       key->name() != "key") {
     return false;

src/google/protobuf/descriptor.h

@@ -487,6 +487,16 @@ class PROTOBUF_EXPORT Descriptor {
   // |*out_location| unchanged iff location information was not available.
   bool GetSourceLocation(SourceLocation* out_location) const;
 
+  // Maps --------------------------------------------------------------
+
+  // Returns the FieldDescriptor for the "key" field. If this isn't a map entry
+  // field, returns nullptr.
+  const FieldDescriptor* map_key() const;
+
+  // Returns the FieldDescriptor for the "value" field. If this isn't a map
+  // entry field, returns nullptr.
+  const FieldDescriptor* map_value() const;
+
  private:
   typedef MessageOptions OptionsType;
 
@@ -693,7 +703,7 @@ class PROTOBUF_EXPORT FieldDescriptor {
   // .proto file. Excludes singular proto3 fields that do not have a label.
   bool has_optional_keyword() const;
 
-  // Returns true if this field tracks presence, ie. does the message
+  // Returns true if this field tracks presence, ie. does the field
   // distinguish between "unset" and "present with default value."
   // This includes required, optional, and oneof fields. It excludes maps,
   // repeated fields, and singular proto3 fields without "optional".

src/google/protobuf/descriptor_unittest.cc

@@ -999,6 +999,22 @@ TEST_F(DescriptorTest, IsMap) {
   EXPECT_TRUE(map_->message_type()->options().map_entry());
 }
 
+TEST_F(DescriptorTest, GetMap) {
+  const Descriptor* map_desc = map_->message_type();
+  const FieldDescriptor* map_key = map_desc->map_key();
+  ASSERT_TRUE(map_key != nullptr);
+  EXPECT_EQ(map_key->name(), "key");
+  EXPECT_EQ(map_key->number(), 1);
+
+  const FieldDescriptor* map_value = map_desc->map_value();
+  ASSERT_TRUE(map_value != nullptr);
+  EXPECT_EQ(map_value->name(), "value");
+  EXPECT_EQ(map_value->number(), 2);
+
+  EXPECT_EQ(message_->map_key(), nullptr);
+  EXPECT_EQ(message_->map_value(), nullptr);
+}
+
 TEST_F(DescriptorTest, FieldHasDefault) {
   EXPECT_FALSE(foo_->has_default_value());
   EXPECT_FALSE(bar_->has_default_value());
@@ -7184,9 +7200,9 @@ class SourceLocationTest : public testing::Test {
   DescriptorPool pool_;
 
   // tag number of all custom options in above test file
-  static const int kCustomOptionFieldNumber = 10101;
+  static constexpr int kCustomOptionFieldNumber = 10101;
   // tag number of field "a" in message type "A" in above test file
-  static const int kAFieldNumber = 1;
+  static constexpr int kAFieldNumber = 1;
 };
 
 // TODO(adonovan): implement support for option fields and for

src/google/protobuf/extension_set_unittest.cc

@@ -821,7 +821,7 @@ TEST(ExtensionSetTest, SpaceUsedExcludingSelf) {
     const int old_capacity =                                                  \
         message.GetRepeatedExtension(unittest::repeated_##type##_extension)   \
             .Capacity();                                                      \
-    EXPECT_GE(old_capacity, kMinRepeatedFieldAllocationSize);                 \
+    EXPECT_GE(old_capacity, kRepeatedFieldLowerClampLimit);                   \
     for (int i = 0; i < 16; ++i) {                                            \
       message.AddExtension(unittest::repeated_##type##_extension, value);     \
     }                                                                         \
@@ -864,7 +864,7 @@ TEST(ExtensionSetTest, SpaceUsedExcludingSelf) {
       message.AddExtension(unittest::repeated_string_extension, value);
     }
     min_expected_size +=
-        (sizeof(value) + value.size()) * (16 - kMinRepeatedFieldAllocationSize);
+        (sizeof(value) + value.size()) * (16 - kRepeatedFieldLowerClampLimit);
     EXPECT_LE(min_expected_size, message.SpaceUsed());
   }
   // Repeated messages
@@ -880,7 +880,7 @@ TEST(ExtensionSetTest, SpaceUsedExcludingSelf) {
           ->CopyFrom(prototype);
     }
     min_expected_size +=
-        (16 - kMinRepeatedFieldAllocationSize) * prototype.SpaceUsed();
+        (16 - kRepeatedFieldLowerClampLimit) * prototype.SpaceUsed();
     EXPECT_LE(min_expected_size, message.SpaceUsed());
   }
 }

src/google/protobuf/implicit_weak_message.h

@@ -100,7 +100,7 @@ template <typename ImplicitWeakType>
 class ImplicitWeakTypeHandler {
  public:
   typedef MessageLite Type;
-  static const bool Moveable = false;
+  static constexpr bool Moveable = false;
 
   static inline MessageLite* NewFromPrototype(const MessageLite* prototype,
                                               Arena* arena = NULL) {

src/google/protobuf/io/coded_stream_unittest.cc

@@ -132,7 +132,7 @@ namespace {
 class CodedStreamTest : public testing::Test {
  protected:
   // Buffer used during most of the tests. This assumes tests run sequentially.
-  static const int kBufferSize = 1024 * 64;
+  static constexpr int kBufferSize = 1024 * 64;
   static uint8 buffer_[kBufferSize];
 };
 

src/google/protobuf/map_entry.h

@@ -153,9 +153,9 @@ template <typename Derived, typename K, typename V,
 struct DeconstructMapEntry<MapEntry<Derived, K, V, key, value, default_enum> > {
   typedef K Key;
   typedef V Value;
-  static const WireFormatLite::FieldType kKeyFieldType = key;
-  static const WireFormatLite::FieldType kValueFieldType = value;
-  static const int default_enum_value = default_enum;
+  static constexpr WireFormatLite::FieldType kKeyFieldType = key;
+  static constexpr WireFormatLite::FieldType kValueFieldType = value;
+  static constexpr int default_enum_value = default_enum;
 };
 
 }  // namespace internal

src/google/protobuf/map_field.cc

@@ -196,8 +196,7 @@ bool DynamicMapField::ContainsMapKey(const MapKey& map_key) const {
 }
 
 void DynamicMapField::AllocateMapValue(MapValueRef* map_val) {
-  const FieldDescriptor* val_des =
-      default_entry_->GetDescriptor()->FindFieldByName("value");
+  const FieldDescriptor* val_des = default_entry_->GetDescriptor()->map_value();
   map_val->SetType(val_des->cpp_type());
   // Allocate memory for the MapValueRef, and initialize to
   // default value.
@@ -300,7 +299,7 @@ void DynamicMapField::MergeFrom(const MapFieldBase& other) {
 
     // Copy map value
     const FieldDescriptor* field_descriptor =
-        default_entry_->GetDescriptor()->FindFieldByName("value");
+        default_entry_->GetDescriptor()->map_value();
     switch (field_descriptor->cpp_type()) {
       case FieldDescriptor::CPPTYPE_INT32: {
         map_val->SetInt32Value(other_it->second.GetInt32Value());
@@ -360,10 +359,8 @@ void DynamicMapField::Swap(MapFieldBase* other) {
 
 void DynamicMapField::SyncRepeatedFieldWithMapNoLock() const {
   const Reflection* reflection = default_entry_->GetReflection();
-  const FieldDescriptor* key_des =
-      default_entry_->GetDescriptor()->FindFieldByName("key");
-  const FieldDescriptor* val_des =
-      default_entry_->GetDescriptor()->FindFieldByName("value");
+  const FieldDescriptor* key_des = default_entry_->GetDescriptor()->map_key();
+  const FieldDescriptor* val_des = default_entry_->GetDescriptor()->map_value();
   if (MapFieldBase::repeated_field_ == NULL) {
     if (MapFieldBase::arena_ == NULL) {
       MapFieldBase::repeated_field_ = new RepeatedPtrField<Message>();
@@ -448,10 +445,8 @@ void DynamicMapField::SyncRepeatedFieldWithMapNoLock() const {
 void DynamicMapField::SyncMapWithRepeatedFieldNoLock() const {
   Map<MapKey, MapValueRef>* map = &const_cast<DynamicMapField*>(this)->map_;
   const Reflection* reflection = default_entry_->GetReflection();
-  const FieldDescriptor* key_des =
-      default_entry_->GetDescriptor()->FindFieldByName("key");
-  const FieldDescriptor* val_des =
-      default_entry_->GetDescriptor()->FindFieldByName("value");
+  const FieldDescriptor* key_des = default_entry_->GetDescriptor()->map_key();
+  const FieldDescriptor* val_des = default_entry_->GetDescriptor()->map_value();
   // DynamicMapField owns map values. Need to delete them before clearing
   // the map.
   if (MapFieldBase::arena_ == nullptr) {

src/google/protobuf/map_field.h

@@ -443,7 +443,7 @@ class MapField : public TypeDefinedMapFieldBase<Key, T> {
   // different exposed type in Map's api and repeated field's api. For
   // details see the comment in the implementation of
   // SyncMapWithRepeatedFieldNoLock.
-  static const bool kIsValueEnum = ValueTypeHandler::kIsEnum;
+  static constexpr bool kIsValueEnum = ValueTypeHandler::kIsEnum;
   typedef typename MapIf<kIsValueEnum, T, const T&>::type CastValueType;
 
  public:

src/google/protobuf/map_field_test.cc

@@ -107,8 +107,8 @@ class MapFieldBasePrimitiveTest : public ::testing::Test {
     map_descriptor_ = unittest::TestMap::descriptor()
                           ->FindFieldByName("map_int32_int32")
                           ->message_type();
-    key_descriptor_ = map_descriptor_->FindFieldByName("key");
-    value_descriptor_ = map_descriptor_->FindFieldByName("value");
+    key_descriptor_ = map_descriptor_->map_key();
+    value_descriptor_ = map_descriptor_->map_value();
 
     // Build map field
     map_field_.reset(new MapFieldType);

src/google/protobuf/map_test_util.h

@@ -238,6 +238,51 @@ inline MapReflectionTester::MapReflectionTester(
   EXPECT_FALSE(map_int32_enum_val_ == nullptr);
   EXPECT_FALSE(map_int32_foreign_message_key_ == nullptr);
   EXPECT_FALSE(map_int32_foreign_message_val_ == nullptr);
+
+  std::vector<const FieldDescriptor*> all_map_descriptors = {
+      map_int32_int32_key_,
+      map_int32_int32_val_,
+      map_int64_int64_key_,
+      map_int64_int64_val_,
+      map_uint32_uint32_key_,
+      map_uint32_uint32_val_,
+      map_uint64_uint64_key_,
+      map_uint64_uint64_val_,
+      map_sint32_sint32_key_,
+      map_sint32_sint32_val_,
+      map_sint64_sint64_key_,
+      map_sint64_sint64_val_,
+      map_fixed32_fixed32_key_,
+      map_fixed32_fixed32_val_,
+      map_fixed64_fixed64_key_,
+      map_fixed64_fixed64_val_,
+      map_sfixed32_sfixed32_key_,
+      map_sfixed32_sfixed32_val_,
+      map_sfixed64_sfixed64_key_,
+      map_sfixed64_sfixed64_val_,
+      map_int32_float_key_,
+      map_int32_float_val_,
+      map_int32_double_key_,
+      map_int32_double_val_,
+      map_bool_bool_key_,
+      map_bool_bool_val_,
+      map_string_string_key_,
+      map_string_string_val_,
+      map_int32_bytes_key_,
+      map_int32_bytes_val_,
+      map_int32_enum_key_,
+      map_int32_enum_val_,
+      map_int32_foreign_message_key_,
+      map_int32_foreign_message_val_};
+  for (const FieldDescriptor* fdesc : all_map_descriptors) {
+    GOOGLE_CHECK(fdesc->containing_type() != nullptr) << fdesc->name();
+    if (fdesc->name() == "key") {
+      EXPECT_EQ(fdesc->containing_type()->map_key(), fdesc);
+    } else {
+      EXPECT_EQ(fdesc->name(), "value");
+      EXPECT_EQ(fdesc->containing_type()->map_value(), fdesc);
+    }
+  }
 }
 
 // Shorthand to get a FieldDescriptor for a field of unittest::TestMap.

src/google/protobuf/map_type_handler.h

@@ -155,13 +155,13 @@ class MapTypeHandler<WireFormatLite::TYPE_MESSAGE, Type> {
   typedef typename MapWireFieldTypeTraits<WireFormatLite::TYPE_MESSAGE,
                                           Type>::TypeOnMemory TypeOnMemory;
   // Corresponding wire type for field type.
-  static const WireFormatLite::WireType kWireType =
+  static constexpr WireFormatLite::WireType kWireType =
       MapWireFieldTypeTraits<WireFormatLite::TYPE_MESSAGE, Type>::kWireType;
   // Whether wire type is for message.
-  static const bool kIsMessage =
+  static constexpr bool kIsMessage =
       MapWireFieldTypeTraits<WireFormatLite::TYPE_MESSAGE, Type>::kIsMessage;
   // Whether wire type is for enum.
-  static const bool kIsEnum =
+  static constexpr bool kIsEnum =
       MapWireFieldTypeTraits<WireFormatLite::TYPE_MESSAGE, Type>::kIsEnum;
 
   // Functions used in parsing and serialization. ===================

src/google/protobuf/message.h

@@ -222,6 +222,9 @@ bool CreateUnknownEnumValues(const FieldDescriptor* field);
 // optimized for speed will want to override these with faster implementations,
 // but classes optimized for code size may be happy with keeping them.  See
 // the optimize_for option in descriptor.proto.
+//
+// Users must not derive from this class. Only the protocol compiler and
+// the internal library are allowed to create subclasses.
 class PROTOBUF_EXPORT Message : public MessageLite {
  public:
   inline Message() {}

src/google/protobuf/message_lite.h

@@ -182,6 +182,9 @@ PROTOBUF_EXPORT size_t StringSpaceUsedExcludingSelfLong(const std::string& str);
 // is best when you only have a small number of message types linked
 // into your binary, in which case the size of the protocol buffers
 // runtime itself is the biggest problem.
+//
+// Users must not derive from this class. Only the protocol compiler and
+// the internal library are allowed to create subclasses.
 class PROTOBUF_EXPORT MessageLite {
  public:
   inline MessageLite() {}

src/google/protobuf/metadata_lite.h

@@ -139,8 +139,8 @@ class InternalMetadata {
     // ptr_ is a Container*.
     kTagContainer = 1,
   };
-  static const intptr_t kPtrTagMask = 1;
-  static const intptr_t kPtrValueMask = ~kPtrTagMask;
+  static constexpr intptr_t kPtrTagMask = 1;
+  static constexpr intptr_t kPtrValueMask = ~kPtrTagMask;
 
   // Accessors for pointer tag and pointer value.
   PROTOBUF_ALWAYS_INLINE int PtrTag() const {

src/google/protobuf/reflection.h

@@ -471,7 +471,7 @@ class RepeatedFieldRefIterator
 // RepeatedFieldAccessor type, etc.
 template <typename T>
 struct PrimitiveTraits {
-  static const bool is_primitive = false;
+  static constexpr bool is_primitive = false;
 };
 #define DEFINE_PRIMITIVE(TYPE, type)                 \
   template <>                                        \
@@ -497,7 +497,8 @@ struct RefTypeTraits<
   typedef T AccessorValueType;
   typedef T IteratorValueType;
   typedef T* IteratorPointerType;
-  static const FieldDescriptor::CppType cpp_type = PrimitiveTraits<T>::cpp_type;
+  static constexpr FieldDescriptor::CppType cpp_type =
+      PrimitiveTraits<T>::cpp_type;
   static const Descriptor* GetMessageFieldDescriptor() { return NULL; }
 };
 
@@ -510,7 +511,7 @@ struct RefTypeTraits<
   typedef int32 AccessorValueType;
   typedef T IteratorValueType;
   typedef int32* IteratorPointerType;
-  static const FieldDescriptor::CppType cpp_type =
+  static constexpr FieldDescriptor::CppType cpp_type =
       FieldDescriptor::CPPTYPE_ENUM;
   static const Descriptor* GetMessageFieldDescriptor() { return NULL; }
 };
@@ -523,7 +524,7 @@ struct RefTypeTraits<
   typedef std::string AccessorValueType;
   typedef const std::string IteratorValueType;
   typedef const std::string* IteratorPointerType;
-  static const FieldDescriptor::CppType cpp_type =
+  static constexpr FieldDescriptor::CppType cpp_type =
       FieldDescriptor::CPPTYPE_STRING;
   static const Descriptor* GetMessageFieldDescriptor() { return NULL; }
 };
@@ -547,7 +548,7 @@ struct RefTypeTraits<
   typedef Message AccessorValueType;
   typedef const T& IteratorValueType;
   typedef const T* IteratorPointerType;
-  static const FieldDescriptor::CppType cpp_type =
+  static constexpr FieldDescriptor::CppType cpp_type =
       FieldDescriptor::CPPTYPE_MESSAGE;
   static const Descriptor* GetMessageFieldDescriptor() {
     return MessageDescriptorGetter<T>::get();

src/google/protobuf/repeated_field.cc

@@ -56,7 +56,7 @@ void** RepeatedPtrFieldBase::InternalExtend(int extend_amount) {
   }
   Rep* old_rep = rep_;
   Arena* arena = GetArena();
-  new_size = std::max(kMinRepeatedFieldAllocationSize,
+  new_size = std::max(internal::kRepeatedFieldLowerClampLimit,
                       std::max(total_size_ * 2, new_size));
   GOOGLE_CHECK_LE(new_size, (std::numeric_limits<size_t>::max() - kRepHeaderSize) /
                          sizeof(old_rep->elements[0]))

src/google/protobuf/repeated_field.h

@@ -66,6 +66,7 @@
 #include <type_traits>
 
 
+// Must be included last.
 #include <google/protobuf/port_def.inc>
 
 #ifdef SWIG
@@ -85,7 +86,16 @@ namespace internal {
 
 class MergePartialFromCodedStreamHelper;
 
-static const int kMinRepeatedFieldAllocationSize = 4;
+// kRepeatedFieldLowerClampLimit is the smallest size that will be allocated
+// when growing a repeated field.
+constexpr int kRepeatedFieldLowerClampLimit = 4;
+
+// kRepeatedFieldUpperClampLimit is the lowest signed integer value that
+// overflows when multiplied by 2 (which is undefined behavior). Sizes above
+// this will clamp to the maximum int value instead of following exponential
+// growth when growing a repeated field.
+constexpr int kRepeatedFieldUpperClampLimit =
+    (std::numeric_limits<int>::max() / 2) + 1;
 
 // A utility function for logging that doesn't need any template types.
 void LogIndexOutOfBounds(int index, int size);
@@ -309,7 +319,7 @@ class RepeatedField final {
   inline void InternalSwap(RepeatedField* other);
 
  private:
-  static const int kInitialSize = 0;
+  static constexpr int kInitialSize = 0;
   // A note on the representation here (see also comment below for
   // RepeatedPtrFieldBase's struct Rep):
   //
@@ -390,6 +400,84 @@ class RepeatedField final {
       }
     }
   }
+
+  // This class is a performance wrapper around RepeatedField::Add(const T&)
+  // function. In general unless a RepeatedField is a local stack variable LLVM
+  // has a hard time optimizing Add. The machine code tends to be
+  // loop:
+  // mov %size, dword ptr [%repeated_field]       // load
+  // cmp %size, dword ptr [%repeated_field + 4]
+  // jae fallback
+  // mov %buffer, qword ptr [%repeated_field + 8]
+  // mov dword [%buffer + %size * 4], %value
+  // inc %size                                    // increment
+  // mov dword ptr [%repeated_field], %size       // store
+  // jmp loop
+  //
+  // This puts a load/store in each iteration of the important loop variable
+  // size. It's a pretty bad compile that happens even in simple cases, but
+  // largely the presence of the fallback path disturbs the compilers mem-to-reg
+  // analysis.
+  //
+  // This class takes ownership of a repeated field for the duration of it's
+  // lifetime. The repeated field should not be accessed during this time, ie.
+  // only access through this class is allowed. This class should always be a
+  // function local stack variable. Intended use
+  //
+  // void AddSequence(const int* begin, const int* end, RepeatedField<int>* out)
+  // {
+  //   RepeatedFieldAdder<int> adder(out);  // Take ownership of out
+  //   for (auto it = begin; it != end; ++it) {
+  //     adder.Add(*it);
+  //   }
+  // }
+  //
+  // Typically due to the fact adder is a local stack variable. The compiler
+  // will be successful in mem-to-reg transformation and the machine code will
+  // be loop: cmp %size, %capacity jae fallback mov dword ptr [%buffer + %size *
+  // 4], %val inc %size jmp loop
+  //
+  // The first version executes at 7 cycles per iteration while the second
+  // version near 1 or 2 cycles.
+  class FastAdder {
+   public:
+    explicit FastAdder(RepeatedField* rf) : repeated_field_(rf) {
+      if (kIsPod) {
+        index_ = repeated_field_->current_size_;
+        capacity_ = repeated_field_->total_size_;
+        buffer_ = repeated_field_->unsafe_elements();
+      }
+    }
+    ~FastAdder() {
+      if (kIsPod) repeated_field_->current_size_ = index_;
+    }
+
+    void Add(const Element& val) {
+      if (kIsPod) {
+        if (index_ == capacity_) {
+          repeated_field_->current_size_ = index_;
+          repeated_field_->Reserve(index_ + 1);
+          capacity_ = repeated_field_->total_size_;
+          buffer_ = repeated_field_->unsafe_elements();
+        }
+        buffer_[index_++] = val;
+      } else {
+        repeated_field_->Add(val);
+      }
+    }
+
+   private:
+    constexpr static bool kIsPod = std::is_pod<Element>::value;
+    RepeatedField* repeated_field_;
+    int index_;
+    int capacity_;
+    Element* buffer_;
+
+    GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(FastAdder);
+  };
+
+  friend class TestRepeatedFieldHelper;
+  friend class ::google::protobuf::internal::ParseContext;
 };
 
 template <typename Element>
@@ -629,7 +717,7 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase {
   inline Arena* GetArena() const { return arena_; }
 
  private:
-  static const int kInitialSize = 0;
+  static constexpr int kInitialSize = 0;
   // A few notes on internal representation:
   //
   // We use an indirected approach, with struct Rep, to keep
@@ -648,7 +736,7 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase {
     int allocated_size;
     void* elements[1];
   };
-  static const size_t kRepHeaderSize = sizeof(Rep) - sizeof(void*);
+  static constexpr size_t kRepHeaderSize = sizeof(Rep) - sizeof(void*);
   Rep* rep_;
 
   template <typename TypeHandler>
@@ -1240,14 +1328,19 @@ inline void RepeatedField<Element>::Set(int index, const Element& value) {
 
 template <typename Element>
 inline void RepeatedField<Element>::Add(const Element& value) {
-  if (current_size_ == total_size_) Reserve(total_size_ + 1);
-  elements()[current_size_++] = value;
+  uint32 size = current_size_;
+  if (static_cast<int>(size) == total_size_) Reserve(total_size_ + 1);
+  elements()[size] = value;
+  current_size_ = size + 1;
 }
 
 template <typename Element>
 inline Element* RepeatedField<Element>::Add() {
-  if (current_size_ == total_size_) Reserve(total_size_ + 1);
-  return &elements()[current_size_++];
+  uint32 size = current_size_;
+  if (static_cast<int>(size) == total_size_) Reserve(total_size_ + 1);
+  auto ptr = &elements()[size];
+  current_size_ = size + 1;
+  return ptr;
 }
 
 template <typename Element>
@@ -1269,9 +1362,8 @@ inline void RepeatedField<Element>::Add(Iter begin, Iter end) {
     std::copy(begin, end, elements() + size());
     current_size_ = reserve + size();
   } else {
-    for (; begin != end; ++begin) {
-      Add(*begin);
-    }
+    FastAdder fast_adder(this);
+    for (; begin != end; ++begin) fast_adder.Add(*begin);
   }
 }
 
@@ -1425,6 +1517,30 @@ inline size_t RepeatedField<Element>::SpaceUsedExcludingSelfLong() const {
   return total_size_ > 0 ? (total_size_ * sizeof(Element) + kRepHeaderSize) : 0;
 }
 
+namespace internal {
+// Returns the new size for a reserved field based on its 'total_size' and the
+// requested 'new_size'. The result is clamped to the closed interval:
+//   [internal::kMinRepeatedFieldAllocationSize,
+//    std::numeric_limits<int>::max()]
+// Requires:
+//     new_size > total_size &&
+//     (total_size == 0 ||
+//      total_size >= kRepeatedFieldLowerClampLimit)
+inline int CalculateReserveSize(int total_size, int new_size) {
+  if (new_size < kRepeatedFieldLowerClampLimit) {
+    // Clamp to smallest allowed size.
+    return kRepeatedFieldLowerClampLimit;
+  }
+  if (total_size < kRepeatedFieldUpperClampLimit) {
+    return std::max(total_size * 2, new_size);
+  } else {
+    // Clamp to largest allowed size.
+    GOOGLE_DCHECK_GT(new_size, kRepeatedFieldUpperClampLimit);
+    return std::numeric_limits<int>::max();
+  }
+}
+}  // namespace internal
+
 // Avoid inlining of Reserve(): new, copy, and delete[] lead to a significant
 // amount of code bloat.
 template <typename Element>
@@ -1433,8 +1549,7 @@ void RepeatedField<Element>::Reserve(int new_size) {
   Rep* old_rep = total_size_ > 0 ? rep() : NULL;
   Rep* new_rep;
   Arena* arena = GetArena();
-  new_size = std::max(internal::kMinRepeatedFieldAllocationSize,
-                      std::max(total_size_ * 2, new_size));
+  new_size = internal::CalculateReserveSize(total_size_, new_size);
   GOOGLE_DCHECK_LE(
       static_cast<size_t>(new_size),
       (std::numeric_limits<size_t>::max() - kRepHeaderSize) / sizeof(Element))
@@ -1448,6 +1563,10 @@ void RepeatedField<Element>::Reserve(int new_size) {
   }
   new_rep->arena = arena;
   int old_total_size = total_size_;
+  // Already known: new_size >= internal::kMinRepeatedFieldAllocationSize
+  // Maintain invariant:
+  //     total_size_ == 0 ||
+  //     total_size_ >= internal::kMinRepeatedFieldAllocationSize
   total_size_ = new_size;
   arena_or_elements_ = new_rep->elements;
   // Invoke placement-new on newly allocated elements. We shouldn't have to do

src/google/protobuf/repeated_field_unittest.cc

@@ -38,6 +38,8 @@
 #include <google/protobuf/repeated_field.h>
 
 #include <algorithm>
+#include <cstdlib>
+#include <iterator>
 #include <limits>
 #include <list>
 #include <sstream>
@@ -53,6 +55,9 @@
 #include <gtest/gtest.h>
 #include <google/protobuf/stubs/stl_util.h>
 
+// Must be included last.
+#include <google/protobuf/port_def.inc>
+
 namespace google {
 namespace protobuf {
 namespace {
@@ -268,6 +273,68 @@ TEST(RepeatedField, Resize) {
   EXPECT_TRUE(field.empty());
 }
 
+TEST(RepeatedField, ReserveNothing) {
+  RepeatedField<int> field;
+  EXPECT_EQ(0, field.Capacity());
+
+  field.Reserve(-1);
+  EXPECT_EQ(0, field.Capacity());
+}
+
+TEST(RepeatedField, ReserveLowerClamp) {
+  const int clamped_value = internal::CalculateReserveSize(0, 1);
+  EXPECT_EQ(internal::kRepeatedFieldLowerClampLimit, clamped_value);
+  EXPECT_EQ(clamped_value, internal::CalculateReserveSize(clamped_value, 2));
+}
+
+TEST(RepeatedField, ReserveGrowth) {
+  // Make sure the field capacity doubles in size on repeated reservation.
+  for (int size = internal::kRepeatedFieldLowerClampLimit, i = 0; i < 4;
+       ++i, size *= 2) {
+    EXPECT_EQ(size * 2, internal::CalculateReserveSize(size, size + 1));
+  }
+}
+
+TEST(RepeatedField, ReserveLarge) {
+  const int old_size = 10;
+  // This is a size we won't get by doubling:
+  const int new_size = old_size * 3 + 1;
+
+  // Reserving more than 2x current capacity should grow directly to that size.
+  EXPECT_EQ(new_size, internal::CalculateReserveSize(old_size, new_size));
+}
+
+TEST(RepeatedField, ReserveHuge) {
+  // Largest value that does not clamp to the large limit:
+  constexpr int non_clamping_limit = std::numeric_limits<int>::max() / 2;
+  ASSERT_LT(2 * non_clamping_limit, std::numeric_limits<int>::max());
+  EXPECT_LT(internal::CalculateReserveSize(non_clamping_limit,
+                                           non_clamping_limit + 1),
+            std::numeric_limits<int>::max());
+
+  // Smallest size that *will* clamp to the upper limit:
+  constexpr int min_clamping_size = std::numeric_limits<int>::max() / 2 + 1;
+  EXPECT_EQ(
+      internal::CalculateReserveSize(min_clamping_size, min_clamping_size + 1),
+      std::numeric_limits<int>::max());
+
+#ifdef PROTOBUF_TEST_ALLOW_LARGE_ALLOC
+  // The rest of this test may allocate several GB of memory, so it is only
+  // built if explicitly requested.
+  RepeatedField<int> huge_field;
+
+  // Reserve a size for huge_field that will clamp.
+  huge_field.Reserve(min_clamping_size);
+  EXPECT_GE(huge_field.Capacity(), min_clamping_size);
+  ASSERT_LT(huge_field.Capacity(), std::numeric_limits<int>::max() - 1);
+
+  // Allocation may return more memory than we requested. However, the updated
+  // size must still be clamped to a valid range.
+  huge_field.Reserve(huge_field.Capacity() + 1);
+  EXPECT_EQ(huge_field.Capacity(), std::numeric_limits<int>::max());
+#endif  // PROTOBUF_TEST_ALLOW_LARGE_ALLOC
+}
+
 TEST(RepeatedField, MergeFrom) {
   RepeatedField<int> source, destination;
   source.Add(4);

src/google/protobuf/util/internal/datapiece.cc

@@ -384,9 +384,8 @@ bool DataPiece::DecodeBase64(StringPiece src, std::string* dest) const {
   if (Base64Unescape(src, dest)) {
     if (use_strict_base64_decoding_) {
       std::string encoded;
-      Base64Escape(
-          reinterpret_cast<const unsigned char*>(dest->data()), dest->length(),
-          &encoded, false);
+      Base64Escape(reinterpret_cast<const unsigned char*>(dest->data()),
+                         dest->length(), &encoded, false);
       StringPiece src_no_padding = StringPiece(src).substr(
           0, HasSuffixString(src, "=") ? src.find_last_not_of('=') + 1
                                       : src.length());

src/google/protobuf/util/internal/json_escaping.h

@@ -44,34 +44,34 @@ class JsonEscaping {
   // The minimum value of a unicode high-surrogate code unit in the utf-16
   // encoding. A high-surrogate is also known as a leading-surrogate.
   // See http://www.unicode.org/glossary/#high_surrogate_code_unit
-  static const uint16 kMinHighSurrogate = 0xd800;
+  static constexpr uint16 kMinHighSurrogate = 0xd800;
 
   // The maximum value of a unicide high-surrogate code unit in the utf-16
   // encoding. A high-surrogate is also known as a leading-surrogate.
   // See http://www.unicode.org/glossary/#high_surrogate_code_unit
-  static const uint16 kMaxHighSurrogate = 0xdbff;
+  static constexpr uint16 kMaxHighSurrogate = 0xdbff;
 
   // The minimum value of a unicode low-surrogate code unit in the utf-16
   // encoding. A low-surrogate is also known as a trailing-surrogate.
   // See http://www.unicode.org/glossary/#low_surrogate_code_unit
-  static const uint16 kMinLowSurrogate = 0xdc00;
+  static constexpr uint16 kMinLowSurrogate = 0xdc00;
 
   // The maximum value of a unicode low-surrogate code unit in the utf-16
   // encoding. A low-surrogate is also known as a trailing surrogate.
   // See http://www.unicode.org/glossary/#low_surrogate_code_unit
-  static const uint16 kMaxLowSurrogate = 0xdfff;
+  static constexpr uint16 kMaxLowSurrogate = 0xdfff;
 
   // The minimum value of a unicode supplementary code point.
   // See http://www.unicode.org/glossary/#supplementary_code_point
-  static const uint32 kMinSupplementaryCodePoint = 0x010000;
+  static constexpr uint32 kMinSupplementaryCodePoint = 0x010000;
 
   // The minimum value of a unicode code point.
   // See http://www.unicode.org/glossary/#code_point
-  static const uint32 kMinCodePoint = 0x000000;
+  static constexpr uint32 kMinCodePoint = 0x000000;
 
   // The maximum value of a unicode code point.
   // See http://www.unicode.org/glossary/#code_point
-  static const uint32 kMaxCodePoint = 0x10ffff;
+  static constexpr uint32 kMaxCodePoint = 0x10ffff;
 
   JsonEscaping() {}
   virtual ~JsonEscaping() {}

src/google/protobuf/util/internal/protostream_objectsource.cc

@@ -537,6 +537,11 @@ Status ProtoStreamObjectSource::RenderStruct(const ProtoStreamObjectSource* os,
   ow->StartObject(field_name);
   while (tag != 0) {
     field = os->FindAndVerifyField(type, tag);
+    if (field == nullptr) {
+      WireFormat::SkipField(os->stream_, tag, nullptr);
+      tag = os->stream_->ReadTag();
+      continue;
+    }
     // google.protobuf.Struct has only one field that is a map. Hence we use
     // RenderMap to render that field.
     if (os->IsMap(*field)) {

src/google/protobuf/util/time_util_test.cc

@@ -119,10 +119,10 @@ TEST(TimeUtilTest, DurationStringFormat) {
   // Duration must support range from -315,576,000,000s to +315576000000s
   // which includes negative values.
   EXPECT_TRUE(TimeUtil::FromString("315576000000.999999999s", &d));
-  EXPECT_EQ(315576000000LL, d.seconds());
+  EXPECT_EQ(int64{315576000000}, d.seconds());
   EXPECT_EQ(999999999, d.nanos());
   EXPECT_TRUE(TimeUtil::FromString("-315576000000.999999999s", &d));
-  EXPECT_EQ(-315576000000LL, d.seconds());
+  EXPECT_EQ(int64{-315576000000}, d.seconds());
   EXPECT_EQ(-999999999, d.nanos());
 }
 
@@ -278,20 +278,22 @@ TEST(TimeUtilTest, DurationOperators) {
   // Multiplication should not overflow if the result fits into the supported
   // range of Duration (intermediate result may be larger than int64).
   EXPECT_EQ("315575999684.424s",
-            TimeUtil::ToString((one_second - one_nano) * 315576000000LL));
+            TimeUtil::ToString((one_second - one_nano) * int64{315576000000}));
   EXPECT_EQ("-315575999684.424s",
-            TimeUtil::ToString((one_nano - one_second) * 315576000000LL));
-  EXPECT_EQ("-315575999684.424s",
-            TimeUtil::ToString((one_second - one_nano) * (-315576000000LL)));
+            TimeUtil::ToString((one_nano - one_second) * int64{315576000000}));
+  EXPECT_EQ("-315575999684.424s", TimeUtil::ToString((one_second - one_nano) *
+                                                     (int64{-315576000000})));
 
   // Test / and %
   EXPECT_EQ("0.999999999s", TimeUtil::ToString(a / 2));
   EXPECT_EQ("-0.999999999s", TimeUtil::ToString(b / 2));
-  Duration large = TimeUtil::SecondsToDuration(315576000000LL) - one_nano;
+  Duration large = TimeUtil::SecondsToDuration(int64{315576000000}) - one_nano;
   // We have to handle division with values beyond 64 bits.
-  EXPECT_EQ("0.999999999s", TimeUtil::ToString(large / 315576000000LL));
-  EXPECT_EQ("-0.999999999s", TimeUtil::ToString((-large) / 315576000000LL));
-  EXPECT_EQ("-0.999999999s", TimeUtil::ToString(large / (-315576000000LL)));
+  EXPECT_EQ("0.999999999s", TimeUtil::ToString(large / int64{315576000000}));
+  EXPECT_EQ("-0.999999999s",
+            TimeUtil::ToString((-large) / int64{315576000000}));
+  EXPECT_EQ("-0.999999999s",
+            TimeUtil::ToString(large / (int64{-315576000000})));
   Duration large2 = large + one_nano;
   EXPECT_EQ(large, large % large2);
   EXPECT_EQ(-large, (-large) % large2);

src/google/protobuf/wire_format_lite.h

@@ -156,9 +156,9 @@ class PROTOBUF_EXPORT WireFormatLite {
   }
 
   // Number of bits in a tag which identify the wire type.
-  static const int kTagTypeBits = 3;
+  static constexpr int kTagTypeBits = 3;
   // Mask for those bits.
-  static const uint32 kTagTypeMask = (1 << kTagTypeBits) - 1;
+  static constexpr uint32 kTagTypeMask = (1 << kTagTypeBits) - 1;
 
   // Helper functions for encoding and decoding tags.  (Inlined below and in
   // _inl.h)
@@ -210,9 +210,9 @@ class PROTOBUF_EXPORT WireFormatLite {
   //       required string message = 3;
   //     }
   //   }
-  static const int kMessageSetItemNumber = 1;
-  static const int kMessageSetTypeIdNumber = 2;
-  static const int kMessageSetMessageNumber = 3;
+  static constexpr int kMessageSetItemNumber = 1;
+  static constexpr int kMessageSetTypeIdNumber = 2;
+  static constexpr int kMessageSetMessageNumber = 3;
   static const int kMessageSetItemStartTag = GOOGLE_PROTOBUF_WIRE_FORMAT_MAKE_TAG(
       kMessageSetItemNumber, WireFormatLite::WIRETYPE_START_GROUP);
   static const int kMessageSetItemEndTag = GOOGLE_PROTOBUF_WIRE_FORMAT_MAKE_TAG(
@@ -685,13 +685,13 @@ class PROTOBUF_EXPORT WireFormatLite {
   static size_t EnumSize(const RepeatedField<int>& value);
 
   // These types always have the same size.
-  static const size_t kFixed32Size = 4;
-  static const size_t kFixed64Size = 8;
-  static const size_t kSFixed32Size = 4;
-  static const size_t kSFixed64Size = 8;
-  static const size_t kFloatSize = 4;
-  static const size_t kDoubleSize = 8;
-  static const size_t kBoolSize = 1;
+  static constexpr size_t kFixed32Size = 4;
+  static constexpr size_t kFixed64Size = 8;
+  static constexpr size_t kSFixed32Size = 4;
+  static constexpr size_t kSFixed64Size = 8;
+  static constexpr size_t kFloatSize = 4;
+  static constexpr size_t kDoubleSize = 8;
+  static constexpr size_t kBoolSize = 1;
 
   static inline size_t StringSize(const std::string& value);
   static inline size_t BytesSize(const std::string& value);