Merge pull request #7630 from haberman/sync-stage

Integrate from Piper for C++, Java, and Python

CHANGES.txt

@@ -11,16 +11,25 @@ Unreleased Changes
   * Added more Windows macros to proto whitelist.
   * Arena constructors for map entry messages are now marked "explicit"
     (for regular messages they were already explicit).
+  * Fix subtle aliasing bug in RepeatedField::Add
+  * Fix mismatch between MapEntry ByteSize and Serialize with respect to unset
+    fields.
 
   Python:
-  * Reject lowercase t for Timestamp json format. Fixes a conformance test.
+  * JSON format conformance fixes:
+    * Reject lowercase t for Timestamp json format.
+    * Print full_name directly for extensions (no camelCase).
+    * Reject boolean values for integer fields.
+    * Reject NaN, Infinity, -Infinity that is not quoted.
+    * Base64 fixes for bytes fields: accept URL-safe base64 and missing padding.
+  * Bugfix for fields/files named "async" or "await".
   * Improved the error message when AttributeError is returned from __getattr__
     in EnumTypeWrapper.
-  * Json format will print full_name directly for extensions.
 
   Java:
   * Fixed a bug where setting optional proto3 enums with setFooValue() would
     not mark the value as present.
+  * Add Subtract function to FieldMaskUtil.
 
   C#:
   * Dropped support for netstandard1.0 (replaced by support for netstandard1.1).

conformance/failure_list_python.txt

@@ -1,30 +0,0 @@
-Recommended.Proto3.JsonInput.BytesFieldBase64Url.JsonOutput
-Recommended.Proto3.JsonInput.BytesFieldBase64Url.ProtobufOutput
-Recommended.Proto3.JsonInput.DoubleFieldInfinityNotQuoted
-Recommended.Proto3.JsonInput.DoubleFieldNanNotQuoted
-Recommended.Proto3.JsonInput.DoubleFieldNegativeInfinityNotQuoted
-Recommended.Proto3.JsonInput.FloatFieldInfinityNotQuoted
-Recommended.Proto3.JsonInput.FloatFieldNanNotQuoted
-Recommended.Proto3.JsonInput.FloatFieldNegativeInfinityNotQuoted
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.BOOL[0].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.BYTES[0].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.DOUBLE[0].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.ENUM[0].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.FIXED32[0].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.FIXED64[0].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.FLOAT[0].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.INT32[0].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.INT32[6].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.INT64[0].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.SFIXED32[0].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.SFIXED64[0].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.SINT32[0].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.SINT64[0].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.STRING[0].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.UINT32[0].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.UINT32[5].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.UINT64[0].ProtobufOutput
-Required.Proto3.JsonInput.DoubleFieldTooSmall
-Required.Proto3.JsonInput.FloatFieldTooLarge
-Required.Proto3.JsonInput.FloatFieldTooSmall
-Required.Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingIntegersGotBool

conformance/failure_list_python_cpp.txt

@@ -6,16 +6,3 @@
 #
 # TODO(haberman): insert links to corresponding bugs tracking the issue.
 # Should we use GitHub issues or the Google-internal bug tracker?
-
-Recommended.Proto3.JsonInput.BytesFieldBase64Url.JsonOutput
-Recommended.Proto3.JsonInput.BytesFieldBase64Url.ProtobufOutput
-Recommended.Proto3.JsonInput.DoubleFieldInfinityNotQuoted
-Recommended.Proto3.JsonInput.DoubleFieldNanNotQuoted
-Recommended.Proto3.JsonInput.DoubleFieldNegativeInfinityNotQuoted
-Recommended.Proto3.JsonInput.FloatFieldInfinityNotQuoted
-Recommended.Proto3.JsonInput.FloatFieldNanNotQuoted
-Recommended.Proto3.JsonInput.FloatFieldNegativeInfinityNotQuoted
-Required.Proto3.JsonInput.DoubleFieldTooSmall
-Required.Proto3.JsonInput.FloatFieldTooLarge
-Required.Proto3.JsonInput.FloatFieldTooSmall
-Required.Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingIntegersGotBool

java/util/src/main/java/com/google/protobuf/util/FieldMaskTree.java

@@ -30,6 +30,7 @@
 
 package com.google.protobuf.util;
 
+import com.google.common.base.Splitter;
 import com.google.errorprone.annotations.CanIgnoreReturnValue;
 import com.google.protobuf.Descriptors.Descriptor;
 import com.google.protobuf.Descriptors.FieldDescriptor;
@@ -66,7 +67,7 @@ final class FieldMaskTree {
   private static final String FIELD_PATH_SEPARATOR_REGEX = "\\.";
 
   private static final class Node {
-    final SortedMap<String, Node> children = new TreeMap<String, Node>();
+    final SortedMap<String, Node> children = new TreeMap<>();
   }
 
   private final Node root = new Node();
@@ -97,6 +98,7 @@ final class FieldMaskTree {
    * to add is a sub-path of an existing leaf node, nothing will be changed in the tree.
    */
   @CanIgnoreReturnValue
+  @SuppressWarnings("StringSplitter")
   FieldMaskTree addFieldPath(String path) {
     String[] parts = path.split(FIELD_PATH_SEPARATOR_REGEX);
     if (parts.length == 0) {
@@ -134,6 +136,45 @@ final class FieldMaskTree {
     return this;
   }
 
+  /**
+   * Remove {@code path} from the tree.
+   *
+   * <p>When removing a field path from the tree, all sub-paths will be removed. That is, after
+   * removing "foo.bar" from the tree, "foo.bar.baz" will be removed. Likewise, if the field path to
+   * remove is a non-exist sub-path, nothing will be changed.
+   */
+  @CanIgnoreReturnValue
+  FieldMaskTree removeFieldPath(String path) {
+    List<String> parts = Splitter.onPattern(FIELD_PATH_SEPARATOR_REGEX).splitToList(path);
+    if (parts.isEmpty()) {
+      return this;
+    }
+    Node node = root;
+    for (int i = 0; i < parts.size(); i++) {
+      String key = parts.get(i);
+      if (!node.children.containsKey(key)) {
+        // Path does not exist.
+        return this;
+      }
+      if (i == parts.size() - 1) {
+        // Remove path.
+        node.children.remove(key);
+        return this;
+      }
+      node = node.children.get(key);
+    }
+    return this;
+  }
+
+  /** Removes all field paths in {@code mask} from this tree. */
+  @CanIgnoreReturnValue
+  FieldMaskTree removeFromFieldMask(FieldMask mask) {
+    for (String path : mask.getPathsList()) {
+      removeFieldPath(path);
+    }
+    return this;
+  }
+
   /**
    * Converts this tree to a FieldMask.
    */
@@ -141,7 +182,7 @@ final class FieldMaskTree {
     if (root.children.isEmpty()) {
       return FieldMask.getDefaultInstance();
     }
-    List<String> paths = new ArrayList<String>();
+    List<String> paths = new ArrayList<>();
     getFieldPaths(root, "", paths);
     return FieldMask.newBuilder().addAllPaths(paths).build();
   }
@@ -186,7 +227,7 @@ final class FieldMaskTree {
     }
     // We found a matching node for the path. All leaf children of this matching
     // node is in the intersection.
-    List<String> paths = new ArrayList<String>();
+    List<String> paths = new ArrayList<>();
     getFieldPaths(node, path, paths);
     for (String value : paths) {
       output.addFieldPath(value);

java/util/src/main/java/com/google/protobuf/util/FieldMaskUtil.java

@@ -276,6 +276,16 @@ public final class FieldMaskUtil {
     return maskTree.toFieldMask();
   }
 
+  /** Subtracts {@code secondMask} and {@code otherMasks} from {@code firstMask}. */
+  public static FieldMask subtract(
+      FieldMask firstMask, FieldMask secondMask, FieldMask... otherMasks) {
+    FieldMaskTree maskTree = new FieldMaskTree(firstMask).removeFromFieldMask(secondMask);
+    for (FieldMask mask : otherMasks) {
+      maskTree.removeFromFieldMask(mask);
+    }
+    return maskTree.toFieldMask();
+  }
+
   /**
    * Calculates the intersection of two FieldMasks.
    */

java/util/src/test/java/com/google/protobuf/util/FieldMaskTreeTest.java

@@ -30,6 +30,8 @@
 
 package com.google.protobuf.util;
 
+import static com.google.common.truth.Truth.assertThat;
+
 import com.google.protobuf.DynamicMessage;
 import com.google.protobuf.Message;
 import com.google.protobuf.UninitializedMessageException;
@@ -43,9 +45,9 @@ import junit.framework.TestCase;
 public class FieldMaskTreeTest extends TestCase {
   public void testAddFieldPath() throws Exception {
     FieldMaskTree tree = new FieldMaskTree();
-    assertEquals("", tree.toString());
+    assertThat(tree.toString()).isEmpty();
     tree.addFieldPath("");
-    assertEquals("", tree.toString());
+    assertThat(tree.toString()).isEmpty();
     // New branch.
     tree.addFieldPath("foo");
     assertEquals("foo", tree.toString());
@@ -73,15 +75,45 @@ public class FieldMaskTreeTest extends TestCase {
     assertEquals("bar,foo", tree.toString());
   }
 
+  public void testRemoveFieldPath() throws Exception {
+    String initialTreeString = "bar.baz,bar.quz.bar,foo";
+    FieldMaskTree tree = new FieldMaskTree(FieldMaskUtil.fromString(initialTreeString));
+    // Empty path.
+    tree.removeFieldPath("");
+    assertEquals(initialTreeString, tree.toString());
+    // Non-exist sub-path of an existing leaf.
+    tree.removeFieldPath("foo.bar");
+    assertEquals(initialTreeString, tree.toString());
+    // Non-exist path.
+    tree.removeFieldPath("bar.foo");
+    assertEquals(initialTreeString, tree.toString());
+    // Match an existing leaf node.
+    tree.removeFieldPath("foo");
+    assertEquals("bar.baz,bar.quz.bar", tree.toString());
+    // Match sub-path of an existing leaf node.
+    tree.removeFieldPath("bar.quz.bar");
+    assertEquals("bar.baz,bar.quz", tree.toString());
+    // Match a non-leaf node.
+    tree.removeFieldPath("bar");
+    assertThat(tree.toString()).isEmpty();
+  }
+
+  public void testRemoveFromFieldMask() throws Exception {
+    FieldMaskTree tree = new FieldMaskTree(FieldMaskUtil.fromString("foo,bar.baz,bar.quz"));
+    assertEquals("bar.baz,bar.quz,foo", tree.toString());
+    tree.removeFromFieldMask(FieldMaskUtil.fromString("foo.bar,bar"));
+    assertEquals("foo", tree.toString());
+  }
+
   public void testIntersectFieldPath() throws Exception {
     FieldMaskTree tree = new FieldMaskTree(FieldMaskUtil.fromString("foo,bar.baz,bar.quz"));
     FieldMaskTree result = new FieldMaskTree();
     // Empty path.
     tree.intersectFieldPath("", result);
-    assertEquals("", result.toString());
+    assertThat(result.toString()).isEmpty();
     // Non-exist path.
     tree.intersectFieldPath("quz", result);
-    assertEquals("", result.toString());
+    assertThat(result.toString()).isEmpty();
     // Sub-path of an existing leaf.
     tree.intersectFieldPath("foo.bar", result);
     assertEquals("foo.bar", result.toString());

java/util/src/test/java/com/google/protobuf/util/FieldMaskUtilTest.java

@@ -224,6 +224,24 @@ public class FieldMaskUtilTest extends TestCase {
     assertEquals("bar,foo", FieldMaskUtil.toString(result));
   }
 
+  public void testSubstract() throws Exception {
+    // Only test a simple case here and expect
+    // {@link FieldMaskTreeTest#testRemoveFieldPath} to cover all scenarios.
+    FieldMask mask1 = FieldMaskUtil.fromString("foo,bar.baz,bar.quz");
+    FieldMask mask2 = FieldMaskUtil.fromString("foo.bar,bar");
+    FieldMask result = FieldMaskUtil.subtract(mask1, mask2);
+    assertEquals("foo", FieldMaskUtil.toString(result));
+  }
+
+  public void testSubstract_usingVarArgs() throws Exception {
+    FieldMask mask1 = FieldMaskUtil.fromString("foo,bar.baz,bar.quz.bar");
+    FieldMask mask2 = FieldMaskUtil.fromString("foo.bar,bar.baz.quz");
+    FieldMask mask3 = FieldMaskUtil.fromString("bar.quz");
+    FieldMask mask4 = FieldMaskUtil.fromString("foo,bar.baz");
+    FieldMask result = FieldMaskUtil.subtract(mask1, mask2, mask3, mask4);
+    assertEquals("bar", FieldMaskUtil.toString(result));
+  }
+
   public void testIntersection() throws Exception {
     // Only test a simple case here and expect
     // {@link FieldMaskTreeTest#testIntersectFieldPath} to cover all scenarios.

python/google/protobuf/internal/decoder.py

@@ -209,7 +209,8 @@ def _SimpleDecoder(wire_type, decode_value):
         _DecodeVarint()
   """
 
-  def SpecificDecoder(field_number, is_repeated, is_packed, key, new_default):
+  def SpecificDecoder(field_number, is_repeated, is_packed, key, new_default,
+                      clear_if_default=False):
     if is_packed:
       local_DecodeVarint = _DecodeVarint
       def DecodePackedField(buffer, pos, end, message, field_dict):
@@ -249,10 +250,13 @@ def _SimpleDecoder(wire_type, decode_value):
       return DecodeRepeatedField
     else:
       def DecodeField(buffer, pos, end, message, field_dict):
-        (field_dict[key], pos) = decode_value(buffer, pos)
+        (new_value, pos) = decode_value(buffer, pos)
         if pos > end:
-          del field_dict[key]  # Discard corrupt value.
           raise _DecodeError('Truncated message.')
+        if clear_if_default and not new_value:
+          field_dict.pop(key, None)
+        else:
+          field_dict[key] = new_value
         return pos
       return DecodeField
 
@@ -383,7 +387,9 @@ def _DoubleDecoder():
   return _SimpleDecoder(wire_format.WIRETYPE_FIXED64, InnerDecode)
 
 
-def EnumDecoder(field_number, is_repeated, is_packed, key, new_default):
+def EnumDecoder(field_number, is_repeated, is_packed, key, new_default,
+                clear_if_default=False):
+  """Returns a decoder for enum field."""
   enum_type = key.enum_type
   if is_packed:
     local_DecodeVarint = _DecodeVarint
@@ -498,6 +504,9 @@ def EnumDecoder(field_number, is_repeated, is_packed, key, new_default):
       (enum_value, pos) = _DecodeSignedVarint32(buffer, pos)
       if pos > end:
         raise _DecodeError('Truncated message.')
+      if clear_if_default and not enum_value:
+        field_dict.pop(key, None)
+        return pos
       # pylint: disable=protected-access
       if enum_value in enum_type.values_by_number:
         field_dict[key] = enum_value
@@ -550,7 +559,7 @@ BoolDecoder = _ModifiedDecoder(
 
 
 def StringDecoder(field_number, is_repeated, is_packed, key, new_default,
-                  is_strict_utf8=False):
+                  is_strict_utf8=False, clear_if_default=False):
   """Returns a decoder for a string field."""
 
   local_DecodeVarint = _DecodeVarint
@@ -604,12 +613,16 @@ def StringDecoder(field_number, is_repeated, is_packed, key, new_default,
       new_pos = pos + size
       if new_pos > end:
         raise _DecodeError('Truncated string.')
-      field_dict[key] = _ConvertToUnicode(buffer[pos:new_pos])
+      if clear_if_default and not size:
+        field_dict.pop(key, None)
+      else:
+        field_dict[key] = _ConvertToUnicode(buffer[pos:new_pos])
       return new_pos
     return DecodeField
 
 
-def BytesDecoder(field_number, is_repeated, is_packed, key, new_default):
+def BytesDecoder(field_number, is_repeated, is_packed, key, new_default,
+                 clear_if_default=False):
   """Returns a decoder for a bytes field."""
 
   local_DecodeVarint = _DecodeVarint
@@ -641,7 +654,10 @@ def BytesDecoder(field_number, is_repeated, is_packed, key, new_default):
       new_pos = pos + size
       if new_pos > end:
         raise _DecodeError('Truncated string.')
-      field_dict[key] = buffer[pos:new_pos].tobytes()
+      if clear_if_default and not size:
+        field_dict.pop(key, None)
+      else:
+        field_dict[key] = buffer[pos:new_pos].tobytes()
       return new_pos
     return DecodeField
 

python/google/protobuf/internal/encoder.py

@@ -372,14 +372,15 @@ def MapSizer(field_descriptor, is_message_map):
 def _VarintEncoder():
   """Return an encoder for a basic varint value (does not include tag)."""
 
+  local_int2byte = six.int2byte
   def EncodeVarint(write, value, unused_deterministic=None):
     bits = value & 0x7f
     value >>= 7
     while value:
-      write(six.int2byte(0x80|bits))
+      write(local_int2byte(0x80|bits))
       bits = value & 0x7f
       value >>= 7
-    return write(six.int2byte(bits))
+    return write(local_int2byte(bits))
 
   return EncodeVarint
 
@@ -388,16 +389,17 @@ def _SignedVarintEncoder():
   """Return an encoder for a basic signed varint value (does not include
   tag)."""
 
+  local_int2byte = six.int2byte
   def EncodeSignedVarint(write, value, unused_deterministic=None):
     if value < 0:
       value += (1 << 64)
     bits = value & 0x7f
     value >>= 7
     while value:
-      write(six.int2byte(0x80|bits))
+      write(local_int2byte(0x80|bits))
       bits = value & 0x7f
       value >>= 7
-    return write(six.int2byte(bits))
+    return write(local_int2byte(bits))
 
   return EncodeSignedVarint
 

python/google/protobuf/internal/json_format_test.py

@@ -830,10 +830,14 @@ class JsonFormatTest(JsonFormatBase):
 
   def testParseDoubleToFloat(self):
     message = json_format_proto3_pb2.TestMessage()
-    text = ('{"repeatedFloatValue": [3.4028235e+39, 1.4028235e-39]\n}')
+    text = ('{"repeatedDoubleValue": [3.4028235e+39, 1.4028235e-39]\n}')
     json_format.Parse(text, message)
-    self.assertEqual(message.repeated_float_value[0], float('inf'))
-    self.assertAlmostEqual(message.repeated_float_value[1], 1.4028235e-39)
+    self.assertEqual(message.repeated_double_value[0], 3.4028235e+39)
+    self.assertEqual(message.repeated_double_value[1], 1.4028235e-39)
+    text = ('{"repeatedFloatValue": [3.4028235e+39, 1.4028235e-39]\n}')
+    self.CheckError(text,
+                    'Failed to parse repeatedFloatValue field: '
+                    'Float value too large.')
 
   def testFloatPrecision(self):
     message = json_format_proto3_pb2.TestMessage()
@@ -898,6 +902,23 @@ class JsonFormatTest(JsonFormatBase):
         'for enum type protobuf_unittest.TestAllTypes.NestedEnum.',
         json_format.Parse, '{"optionalNestedEnum": 12345}', message)
 
+  def testBytes(self):
+    message = json_format_proto3_pb2.TestMessage()
+    # Test url base64
+    text = '{"bytesValue": "-_"}'
+    json_format.Parse(text, message)
+    self.assertEqual(message.bytes_value, b'\xfb')
+    # Test padding
+    text = '{"bytesValue": "AQI="}'
+    json_format.Parse(text, message)
+    self.assertEqual(message.bytes_value, b'\x01\x02')
+    text = '{"bytesValue": "AQI"}'
+    json_format.Parse(text, message)
+    self.assertEqual(message.bytes_value, b'\x01\x02')
+    text = '{"bytesValue": "AQI*"}'
+    json_format.Parse(text, message)
+    self.assertEqual(message.bytes_value, b'\x01\x02')
+
   def testParseBadIdentifer(self):
     self.CheckError('{int32Value: 1}',
                     (r'Failed to load JSON: Expecting property name'
@@ -948,6 +969,9 @@ class JsonFormatTest(JsonFormatBase):
     self.CheckError('{"int32Value": "1 "}',
                     'Failed to parse int32Value field: '
                     'Couldn\'t parse integer: "1 ".')
+    self.CheckError('{"int32Value": false}',
+                    'Failed to parse int32Value field: Bool value False '
+                    'is not acceptable for integer field.')
     self.CheckError('{"int32Value": 12345678901234567890}',
                     'Failed to parse int32Value field: Value out of range: '
                     '12345678901234567890.')
@@ -959,12 +983,24 @@ class JsonFormatTest(JsonFormatBase):
     self.CheckError('{"floatValue": "nan"}',
                     'Failed to parse floatValue field: Couldn\'t '
                     'parse float "nan", use "NaN" instead.')
-
-  def testInvalidBytesValue(self):
-    self.CheckError('{"bytesValue": "AQI"}',
-                    'Failed to parse bytesValue field: Incorrect padding.')
-    self.CheckError('{"bytesValue": "AQI*"}',
-                    'Failed to parse bytesValue field: Incorrect padding.')
+    self.CheckError('{"floatValue": NaN}',
+                    'Failed to parse floatValue field: Couldn\'t '
+                    'parse NaN, use quoted "NaN" instead.')
+    self.CheckError('{"floatValue": Infinity}',
+                    'Failed to parse floatValue field: Couldn\'t parse Infinity'
+                    ' or value too large, use quoted "Infinity" instead.')
+    self.CheckError('{"floatValue": -Infinity}',
+                    'Failed to parse floatValue field: Couldn\'t parse '
+                    '-Infinity or value too small, '
+                    'use quoted "-Infinity" instead.')
+    self.CheckError('{"doubleValue": -1.89769e+308}',
+                    'Failed to parse doubleValue field: Couldn\'t parse '
+                    '-Infinity or value too small, '
+                    'use quoted "-Infinity" instead.')
+    self.CheckError('{"floatValue": 3.4028235e+39}',
+                    'Failed to parse floatValue field: Float value too large.')
+    self.CheckError('{"floatValue": -3.502823e+38}',
+                    'Failed to parse floatValue field: Float value too small.')
 
   def testInvalidRepeated(self):
     self.CheckError('{"repeatedInt32Value": 12345}',

python/google/protobuf/internal/message_test.py

@@ -1675,6 +1675,17 @@ class Proto3Test(unittest.TestCase):
     self.assertEqual(False, message.optional_bool)
     self.assertEqual(0, message.optional_nested_message.bb)
 
+  def testProto3ParserDropDefaultScalar(self):
+    message_proto2 = unittest_pb2.TestAllTypes()
+    message_proto2.optional_int32 = 0
+    message_proto2.optional_string = ''
+    message_proto2.optional_bytes = b''
+    self.assertEqual(len(message_proto2.ListFields()), 3)
+
+    message_proto3 = unittest_proto3_arena_pb2.TestAllTypes()
+    message_proto3.ParseFromString(message_proto2.SerializeToString())
+    self.assertEqual(len(message_proto3.ListFields()), 0)
+
   def testProto3Optional(self):
     msg = test_proto3_optional_pb2.TestProto3Optional()
     self.assertFalse(msg.HasField('optional_int32'))

python/google/protobuf/internal/python_message.py

@@ -295,6 +295,7 @@ def _AttachFieldHelpers(cls, field_descriptor):
   is_repeated = (field_descriptor.label == _FieldDescriptor.LABEL_REPEATED)
   is_packable = (is_repeated and
                  wire_format.IsTypePackable(field_descriptor.type))
+  is_proto3 = field_descriptor.containing_type.syntax == 'proto3'
   if not is_packable:
     is_packed = False
   elif field_descriptor.containing_type.syntax == 'proto2':
@@ -333,8 +334,12 @@ def _AttachFieldHelpers(cls, field_descriptor):
       decode_type = _FieldDescriptor.TYPE_INT32
 
     oneof_descriptor = None
+    clear_if_default = False
     if field_descriptor.containing_oneof is not None:
       oneof_descriptor = field_descriptor
+    elif (is_proto3 and not is_repeated and
+          field_descriptor.cpp_type != _FieldDescriptor.CPPTYPE_MESSAGE):
+      clear_if_default = True
 
     if is_map_entry:
       is_message_map = _IsMessageMapField(field_descriptor)
@@ -347,11 +352,17 @@ def _AttachFieldHelpers(cls, field_descriptor):
       field_decoder = decoder.StringDecoder(
           field_descriptor.number, is_repeated, is_packed,
           field_descriptor, field_descriptor._default_constructor,
-          is_strict_utf8_check)
-    else:
+          is_strict_utf8_check, clear_if_default)
+    elif field_descriptor.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE:
       field_decoder = type_checkers.TYPE_TO_DECODER[decode_type](
           field_descriptor.number, is_repeated, is_packed,
           field_descriptor, field_descriptor._default_constructor)
+    else:
+      field_decoder = type_checkers.TYPE_TO_DECODER[decode_type](
+          field_descriptor.number, is_repeated, is_packed,
+          # pylint: disable=protected-access
+          field_descriptor, field_descriptor._default_constructor,
+          clear_if_default)
 
     cls._decoders_by_tag[tag_bytes] = (field_decoder, oneof_descriptor)
 

python/google/protobuf/json_format.py

@@ -195,7 +195,6 @@ class _Printer(object):
     self.preserving_proto_field_name = preserving_proto_field_name
     self.use_integers_for_enums = use_integers_for_enums
     self.descriptor_pool = descriptor_pool
-    # TODO(jieluo): change the float precision default to 8 valid digits.
     if float_precision:
       self.float_format = '.{}g'.format(float_precision)
     else:
@@ -720,12 +719,18 @@ def _ConvertScalarFieldValue(value, field, require_str=False):
   if field.cpp_type in _INT_TYPES:
     return _ConvertInteger(value)
   elif field.cpp_type in _FLOAT_TYPES:
-    return _ConvertFloat(value)
+    return _ConvertFloat(value, field)
   elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL:
     return _ConvertBool(value, require_str)
   elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING:
     if field.type == descriptor.FieldDescriptor.TYPE_BYTES:
-      return base64.b64decode(value)
+      if isinstance(value, six.text_type):
+        encoded = value.encode('utf-8')
+      else:
+        encoded = value
+      # Add extra padding '='
+      padded_value = encoded + b'=' * (4 - len(encoded) % 4)
+      return base64.urlsafe_b64decode(padded_value)
     else:
       # Checking for unpaired surrogates appears to be unreliable,
       # depending on the specific Python version, so we check manually.
@@ -769,11 +774,32 @@ def _ConvertInteger(value):
   if isinstance(value, six.text_type) and value.find(' ') != -1:
     raise ParseError('Couldn\'t parse integer: "{0}".'.format(value))
 
+  if isinstance(value, bool):
+    raise ParseError('Bool value {0} is not acceptable for '
+                     'integer field.'.format(value))
+
   return int(value)
 
 
-def _ConvertFloat(value):
+def _ConvertFloat(value, field):
   """Convert an floating point number."""
+  if isinstance(value, float):
+    if math.isnan(value):
+      raise ParseError('Couldn\'t parse NaN, use quoted "NaN" instead.')
+    if math.isinf(value):
+      if value > 0:
+        raise ParseError('Couldn\'t parse Infinity or value too large, '
+                         'use quoted "Infinity" instead.')
+      else:
+        raise ParseError('Couldn\'t parse -Infinity or value too small, '
+                         'use quoted "-Infinity" instead.')
+    if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_FLOAT:
+      # pylint: disable=protected-access
+      if value > type_checkers._FLOAT_MAX:
+        raise ParseError('Float value too large')
+      # pylint: disable=protected-access
+      if value < type_checkers._FLOAT_MIN:
+        raise ParseError('Float value too small')
   if value == 'nan':
     raise ParseError('Couldn\'t parse float "nan", use "NaN" instead.')
   try:

python/google/protobuf/pyext/descriptor.cc

@@ -1690,7 +1690,7 @@ static PyObject* FindMethodByName(PyBaseDescriptor *self, PyObject* arg) {
   }
 
   const MethodDescriptor* method_descriptor =
-      _GetDescriptor(self)->FindMethodByName(std::string(name, name_size));
+      _GetDescriptor(self)->FindMethodByName(StringParam(name, name_size));
   if (method_descriptor == NULL) {
     PyErr_Format(PyExc_KeyError, "Couldn't find method %.200s", name);
     return NULL;

python/google/protobuf/pyext/descriptor.h

@@ -41,6 +41,9 @@ namespace google {
 namespace protobuf {
 namespace python {
 
+// Should match the type of ConstStringParam.
+using StringParam = std::string;
+
 extern PyTypeObject PyMessageDescriptor_Type;
 extern PyTypeObject PyFieldDescriptor_Type;
 extern PyTypeObject PyEnumDescriptor_Type;

python/google/protobuf/pyext/descriptor_containers.cc

@@ -81,9 +81,9 @@ struct PyContainer;
 typedef int (*CountMethod)(PyContainer* self);
 typedef const void* (*GetByIndexMethod)(PyContainer* self, int index);
 typedef const void* (*GetByNameMethod)(PyContainer* self,
-                                       const std::string& name);
+                                       ConstStringParam name);
 typedef const void* (*GetByCamelcaseNameMethod)(PyContainer* self,
-                                                const std::string& name);
+                                                ConstStringParam name);
 typedef const void* (*GetByNumberMethod)(PyContainer* self, int index);
 typedef PyObject* (*NewObjectFromItemMethod)(const void* descriptor);
 typedef const std::string& (*GetItemNameMethod)(const void* descriptor);
@@ -183,7 +183,7 @@ static bool _GetItemByKey(PyContainer* self, PyObject* key, const void** item) {
           return false;
         }
         *item = self->container_def->get_by_name_fn(
-            self, std::string(name, name_size));
+            self, StringParam(name, name_size));
         return true;
       }
     case PyContainer::KIND_BYCAMELCASENAME:
@@ -200,7 +200,7 @@ static bool _GetItemByKey(PyContainer* self, PyObject* key, const void** item) {
           return false;
         }
         *item = self->container_def->get_by_camelcase_name_fn(
-            self, std::string(camelcase_name, name_size));
+            self, StringParam(camelcase_name, name_size));
         return true;
       }
     case PyContainer::KIND_BYNUMBER:
@@ -962,12 +962,12 @@ static int Count(PyContainer* self) {
   return GetDescriptor(self)->field_count();
 }
 
-static const void* GetByName(PyContainer* self, const std::string& name) {
+static const void* GetByName(PyContainer* self, ConstStringParam name) {
   return GetDescriptor(self)->FindFieldByName(name);
 }
 
 static const void* GetByCamelcaseName(PyContainer* self,
-                                      const std::string& name) {
+                                      ConstStringParam name) {
   return GetDescriptor(self)->FindFieldByCamelcaseName(name);
 }
 
@@ -1040,7 +1040,7 @@ static int Count(PyContainer* self) {
   return GetDescriptor(self)->nested_type_count();
 }
 
-static const void* GetByName(PyContainer* self, const std::string& name) {
+static const void* GetByName(PyContainer* self, ConstStringParam name) {
   return GetDescriptor(self)->FindNestedTypeByName(name);
 }
 
@@ -1092,7 +1092,7 @@ static int Count(PyContainer* self) {
   return GetDescriptor(self)->enum_type_count();
 }
 
-static const void* GetByName(PyContainer* self, const std::string& name) {
+static const void* GetByName(PyContainer* self, ConstStringParam name) {
   return GetDescriptor(self)->FindEnumTypeByName(name);
 }
 
@@ -1155,7 +1155,7 @@ static int Count(PyContainer* self) {
   return count;
 }
 
-static const void* GetByName(PyContainer* self, const std::string& name) {
+static const void* GetByName(PyContainer* self, ConstStringParam name) {
   return GetDescriptor(self)->FindEnumValueByName(name);
 }
 
@@ -1215,7 +1215,7 @@ static int Count(PyContainer* self) {
   return GetDescriptor(self)->extension_count();
 }
 
-static const void* GetByName(PyContainer* self, const std::string& name) {
+static const void* GetByName(PyContainer* self, ConstStringParam name) {
   return GetDescriptor(self)->FindExtensionByName(name);
 }
 
@@ -1267,7 +1267,7 @@ static int Count(PyContainer* self) {
   return GetDescriptor(self)->oneof_decl_count();
 }
 
-static const void* GetByName(PyContainer* self, const std::string& name) {
+static const void* GetByName(PyContainer* self, ConstStringParam name) {
   return GetDescriptor(self)->FindOneofByName(name);
 }
 
@@ -1333,7 +1333,7 @@ static const void* GetByIndex(PyContainer* self, int index) {
   return GetDescriptor(self)->value(index);
 }
 
-static const void* GetByName(PyContainer* self, const std::string& name) {
+static const void* GetByName(PyContainer* self, ConstStringParam name) {
   return GetDescriptor(self)->FindValueByName(name);
 }
 
@@ -1454,7 +1454,7 @@ static int Count(PyContainer* self) {
   return GetDescriptor(self)->method_count();
 }
 
-static const void* GetByName(PyContainer* self, const std::string& name) {
+static const void* GetByName(PyContainer* self, ConstStringParam name) {
   return GetDescriptor(self)->FindMethodByName(name);
 }
 
@@ -1516,7 +1516,7 @@ static int Count(PyContainer* self) {
   return GetDescriptor(self)->message_type_count();
 }
 
-static const void* GetByName(PyContainer* self, const std::string& name) {
+static const void* GetByName(PyContainer* self, ConstStringParam name) {
   return GetDescriptor(self)->FindMessageTypeByName(name);
 }
 
@@ -1564,7 +1564,7 @@ static int Count(PyContainer* self) {
   return GetDescriptor(self)->enum_type_count();
 }
 
-static const void* GetByName(PyContainer* self, const std::string& name) {
+static const void* GetByName(PyContainer* self, ConstStringParam name) {
   return GetDescriptor(self)->FindEnumTypeByName(name);
 }
 
@@ -1612,7 +1612,7 @@ static int Count(PyContainer* self) {
   return GetDescriptor(self)->extension_count();
 }
 
-static const void* GetByName(PyContainer* self, const std::string& name) {
+static const void* GetByName(PyContainer* self, ConstStringParam name) {
   return GetDescriptor(self)->FindExtensionByName(name);
 }
 
@@ -1660,7 +1660,7 @@ static int Count(PyContainer* self) {
   return GetDescriptor(self)->service_count();
 }
 
-static const void* GetByName(PyContainer* self, const std::string& name) {
+static const void* GetByName(PyContainer* self, ConstStringParam name) {
   return GetDescriptor(self)->FindServiceByName(name);
 }
 

python/google/protobuf/pyext/descriptor_pool.cc

@@ -240,7 +240,7 @@ static PyObject* FindMessageByName(PyObject* self, PyObject* arg) {
 
   const Descriptor* message_descriptor =
       reinterpret_cast<PyDescriptorPool*>(self)->pool->FindMessageTypeByName(
-          std::string(name, name_size));
+          StringParam(name, name_size));
 
   if (message_descriptor == NULL) {
     return SetErrorFromCollector(
@@ -264,7 +264,7 @@ static PyObject* FindFileByName(PyObject* self, PyObject* arg) {
 
   PyDescriptorPool* py_pool = reinterpret_cast<PyDescriptorPool*>(self);
   const FileDescriptor* file_descriptor =
-      py_pool->pool->FindFileByName(std::string(name, name_size));
+      py_pool->pool->FindFileByName(StringParam(name, name_size));
 
   if (file_descriptor == NULL) {
     return SetErrorFromCollector(py_pool->error_collector, name, "file");
@@ -280,7 +280,7 @@ PyObject* FindFieldByName(PyDescriptorPool* self, PyObject* arg) {
   }
 
   const FieldDescriptor* field_descriptor =
-      self->pool->FindFieldByName(std::string(name, name_size));
+      self->pool->FindFieldByName(StringParam(name, name_size));
   if (field_descriptor == NULL) {
     return SetErrorFromCollector(self->error_collector, name, "field");
   }
@@ -301,7 +301,7 @@ PyObject* FindExtensionByName(PyDescriptorPool* self, PyObject* arg) {
   }
 
   const FieldDescriptor* field_descriptor =
-      self->pool->FindExtensionByName(std::string(name, name_size));
+      self->pool->FindExtensionByName(StringParam(name, name_size));
   if (field_descriptor == NULL) {
     return SetErrorFromCollector(self->error_collector, name,
                                  "extension field");
@@ -323,7 +323,7 @@ PyObject* FindEnumTypeByName(PyDescriptorPool* self, PyObject* arg) {
   }
 
   const EnumDescriptor* enum_descriptor =
-      self->pool->FindEnumTypeByName(std::string(name, name_size));
+      self->pool->FindEnumTypeByName(StringParam(name, name_size));
   if (enum_descriptor == NULL) {
     return SetErrorFromCollector(self->error_collector, name, "enum");
   }
@@ -344,7 +344,7 @@ PyObject* FindOneofByName(PyDescriptorPool* self, PyObject* arg) {
   }
 
   const OneofDescriptor* oneof_descriptor =
-      self->pool->FindOneofByName(std::string(name, name_size));
+      self->pool->FindOneofByName(StringParam(name, name_size));
   if (oneof_descriptor == NULL) {
     return SetErrorFromCollector(self->error_collector, name, "oneof");
   }
@@ -366,7 +366,7 @@ static PyObject* FindServiceByName(PyObject* self, PyObject* arg) {
 
   const ServiceDescriptor* service_descriptor =
       reinterpret_cast<PyDescriptorPool*>(self)->pool->FindServiceByName(
-          std::string(name, name_size));
+          StringParam(name, name_size));
   if (service_descriptor == NULL) {
     return SetErrorFromCollector(
         reinterpret_cast<PyDescriptorPool*>(self)->error_collector, name,
@@ -386,7 +386,7 @@ static PyObject* FindMethodByName(PyObject* self, PyObject* arg) {
 
   const MethodDescriptor* method_descriptor =
       reinterpret_cast<PyDescriptorPool*>(self)->pool->FindMethodByName(
-          std::string(name, name_size));
+          StringParam(name, name_size));
   if (method_descriptor == NULL) {
     return SetErrorFromCollector(
         reinterpret_cast<PyDescriptorPool*>(self)->error_collector, name,
@@ -406,7 +406,7 @@ static PyObject* FindFileContainingSymbol(PyObject* self, PyObject* arg) {
 
   const FileDescriptor* file_descriptor =
       reinterpret_cast<PyDescriptorPool*>(self)->pool->FindFileContainingSymbol(
-          std::string(name, name_size));
+          StringParam(name, name_size));
   if (file_descriptor == NULL) {
     return SetErrorFromCollector(
         reinterpret_cast<PyDescriptorPool*>(self)->error_collector, name,

python/google/protobuf/pyext/extension_dict.cc

@@ -240,11 +240,11 @@ PyObject* _FindExtensionByName(ExtensionDict* self, PyObject* arg) {
 
   PyDescriptorPool* pool = cmessage::GetFactoryForMessage(self->parent)->pool;
   const FieldDescriptor* message_extension =
-      pool->pool->FindExtensionByName(std::string(name, name_size));
+      pool->pool->FindExtensionByName(StringParam(name, name_size));
   if (message_extension == NULL) {
     // Is is the name of a message set extension?
     const Descriptor* message_descriptor =
-        pool->pool->FindMessageTypeByName(std::string(name, name_size));
+        pool->pool->FindMessageTypeByName(StringParam(name, name_size));
     if (message_descriptor && message_descriptor->extension_count() > 0) {
       const FieldDescriptor* extension = message_descriptor->extension(0);
       if (extension->is_extension() &&

python/google/protobuf/pyext/message.cc

@@ -243,7 +243,7 @@ static PyObject* New(PyTypeObject* type,
     }
     message_descriptor =
         GetDefaultDescriptorPool()->pool->FindMessageTypeByName(
-            std::string(full_name, name_size));
+            StringParam(full_name, name_size));
     if (message_descriptor == nullptr) {
       PyErr_Format(PyExc_KeyError,
                    "Can not find message descriptor %s "
@@ -984,7 +984,7 @@ static PyObject* GetIntegerEnumValue(const FieldDescriptor& descriptor,
       return NULL;
     }
     const EnumValueDescriptor* enum_value_descriptor =
-        enum_descriptor->FindValueByName(std::string(enum_label, size));
+        enum_descriptor->FindValueByName(StringParam(enum_label, size));
     if (enum_value_descriptor == NULL) {
       PyErr_Format(PyExc_ValueError, "unknown enum label \"%s\"", enum_label);
       return NULL;
@@ -1399,7 +1399,7 @@ int HasFieldByDescriptor(CMessage* self,
 }
 
 const FieldDescriptor* FindFieldWithOneofs(const Message* message,
-                                           const std::string& field_name,
+                                           ConstStringParam field_name,
                                            bool* in_oneof) {
   *in_oneof = false;
   const Descriptor* descriptor = message->GetDescriptor();
@@ -1455,7 +1455,7 @@ PyObject* HasField(CMessage* self, PyObject* arg) {
   Message* message = self->message;
   bool is_in_oneof;
   const FieldDescriptor* field_descriptor =
-      FindFieldWithOneofs(message, std::string(field_name, size), &is_in_oneof);
+      FindFieldWithOneofs(message, StringParam(field_name, size), &is_in_oneof);
   if (field_descriptor == NULL) {
     if (!is_in_oneof) {
       PyErr_Format(PyExc_ValueError, "Protocol message %s has no field %s.",
@@ -1632,7 +1632,7 @@ PyObject* ClearField(CMessage* self, PyObject* arg) {
   AssureWritable(self);
   bool is_in_oneof;
   const FieldDescriptor* field_descriptor = FindFieldWithOneofs(
-      self->message, std::string(field_name, field_size), &is_in_oneof);
+      self->message, StringParam(field_name, field_size), &is_in_oneof);
   if (field_descriptor == NULL) {
     if (is_in_oneof) {
       // We gave the name of a oneof, and none of its fields are set.
@@ -2033,13 +2033,12 @@ static PyObject* WhichOneof(CMessage* self, PyObject* arg) {
   char *name_data;
   if (PyString_AsStringAndSize(arg, &name_data, &name_size) < 0)
     return NULL;
-  std::string oneof_name = std::string(name_data, name_size);
   const OneofDescriptor* oneof_desc =
-      self->message->GetDescriptor()->FindOneofByName(oneof_name);
+      self->message->GetDescriptor()->FindOneofByName(
+          StringParam(name_data, name_size));
   if (oneof_desc == NULL) {
     PyErr_Format(PyExc_ValueError,
-                 "Protocol message has no oneof \"%s\" field.",
-                 oneof_name.c_str());
+                 "Protocol message has no oneof \"%s\" field.", name_data);
     return NULL;
   }
   const FieldDescriptor* field_in_oneof =

src/google/protobuf/any.pb.h

@@ -69,7 +69,7 @@ PROTOBUF_NAMESPACE_OPEN
 class PROTOBUF_EXPORT Any PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.Any) */ {
  public:
-  inline Any() : Any(nullptr) {};
+  inline Any() : Any(nullptr) {}
   virtual ~Any();
 
   Any(const Any& from);

src/google/protobuf/api.pb.h

@@ -79,7 +79,7 @@ PROTOBUF_NAMESPACE_OPEN
 class PROTOBUF_EXPORT Api PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.Api) */ {
  public:
-  inline Api() : Api(nullptr) {};
+  inline Api() : Api(nullptr) {}
   virtual ~Api();
 
   Api(const Api& from);
@@ -332,7 +332,7 @@ class PROTOBUF_EXPORT Api PROTOBUF_FINAL :
 class PROTOBUF_EXPORT Method PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.Method) */ {
  public:
-  inline Method() : Method(nullptr) {};
+  inline Method() : Method(nullptr) {}
   virtual ~Method();
 
   Method(const Method& from);
@@ -565,7 +565,7 @@ class PROTOBUF_EXPORT Method PROTOBUF_FINAL :
 class PROTOBUF_EXPORT Mixin PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.Mixin) */ {
  public:
-  inline Mixin() : Mixin(nullptr) {};
+  inline Mixin() : Mixin(nullptr) {}
   virtual ~Mixin();
 
   Mixin(const Mixin& from);

src/google/protobuf/compiler/command_line_interface.cc

@@ -1104,6 +1104,7 @@ PopulateSingleSimpleDescriptorDatabase(const std::string& descriptor_set_name) {
 
 bool CommandLineInterface::AllowProto3Optional(
     const FileDescriptor& file) const {
+  // If the --experimental_allow_proto3_optional flag was set, we allow.
   if (allow_proto3_optional_) return true;
 
   // Whitelist all ads protos. Ads is an early adopter of this feature.

src/google/protobuf/compiler/command_line_interface_unittest.cc

@@ -2373,48 +2373,50 @@ TEST_F(CommandLineInterfaceTest, MissingValueAtEndError) {
 }
 
 TEST_F(CommandLineInterfaceTest, Proto3OptionalDisallowed) {
-  CreateTempFile("foo.proto",
+  CreateTempFile("google/foo.proto",
                  "syntax = \"proto3\";\n"
                  "message Foo {\n"
                  "  optional int32 i = 1;\n"
                  "}\n");
 
-  Run("protocol_compiler --proto_path=$tmpdir foo.proto -odescriptor.pb");
+  Run("protocol_compiler --proto_path=$tmpdir google/foo.proto "
+      "-odescriptor.pb");
 
   ExpectErrorSubstring("--experimental_allow_proto3_optional was not set");
 }
 
 TEST_F(CommandLineInterfaceTest, Proto3OptionalDisallowedDescriptor) {
-  CreateTempFile("foo.proto",
+  CreateTempFile("google/foo.proto",
                  "syntax = \"proto3\";\n"
                  "message Foo {\n"
                  "  optional int32 i = 1;\n"
                  "}\n");
 
   Run("protocol_compiler --experimental_allow_proto3_optional "
-      "--proto_path=$tmpdir foo.proto "
+      "--proto_path=$tmpdir google/foo.proto "
       " -o$tmpdir/descriptor.pb");
   ExpectNoErrors();
 
-  Run("protocol_compiler --descriptor_set_in=$tmpdir/descriptor.pb foo.proto "
-      "--test_out=$tmpdir");
+  Run("protocol_compiler --descriptor_set_in=$tmpdir/descriptor.pb"
+      " google/foo.proto --test_out=$tmpdir");
   ExpectErrorSubstring("--experimental_allow_proto3_optional was not set");
 }
 
 TEST_F(CommandLineInterfaceTest, Proto3OptionalDisallowedGenCode) {
-  CreateTempFile("foo.proto",
+  CreateTempFile("google/foo.proto",
                  "syntax = \"proto3\";\n"
                  "message Foo {\n"
                  "  optional int32 i = 1;\n"
                  "}\n");
 
-  Run("protocol_compiler --proto_path=$tmpdir foo.proto --test_out=$tmpdir");
+  Run("protocol_compiler --proto_path=$tmpdir google/foo.proto "
+      "--test_out=$tmpdir");
 
   ExpectErrorSubstring("--experimental_allow_proto3_optional was not set");
 }
 
 TEST_F(CommandLineInterfaceTest, Proto3OptionalDisallowedNoCodegenSupport) {
-  CreateTempFile("foo.proto",
+  CreateTempFile("google/foo.proto",
                  "syntax = \"proto3\";\n"
                  "message Foo {\n"
                  "  optional int32 i = 1;\n"
@@ -2425,7 +2427,7 @@ TEST_F(CommandLineInterfaceTest, Proto3OptionalDisallowedNoCodegenSupport) {
                                      CodeGenerator::FEATURE_PROTO3_OPTIONAL);
 
   Run("protocol_compiler --experimental_allow_proto3_optional "
-      "--proto_path=$tmpdir foo.proto --no_proto3_optional_out=$tmpdir");
+      "--proto_path=$tmpdir google/foo.proto --no_proto3_optional_out=$tmpdir");
 
   ExpectErrorSubstring(
       "code generator --no_proto3_optional_out hasn't been updated to support "
@@ -2433,14 +2435,14 @@ TEST_F(CommandLineInterfaceTest, Proto3OptionalDisallowedNoCodegenSupport) {
 }
 
 TEST_F(CommandLineInterfaceTest, Proto3OptionalAllowWithFlag) {
-  CreateTempFile("foo.proto",
+  CreateTempFile("google/foo.proto",
                  "syntax = \"proto3\";\n"
                  "message Foo {\n"
                  "  optional int32 i = 1;\n"
                  "}\n");
 
   Run("protocol_compiler --experimental_allow_proto3_optional "
-      "--proto_path=$tmpdir foo.proto --test_out=$tmpdir");
+      "--proto_path=$tmpdir google/foo.proto --test_out=$tmpdir");
   ExpectNoErrors();
 }
 

src/google/protobuf/compiler/cpp/cpp_message.cc

@@ -1123,7 +1123,7 @@ void MessageGenerator::GenerateClassDefinition(io::Printer* printer) {
   format.Indent();
 
   if (SupportsArenas(descriptor_)) {
-    format("inline $classname$() : $classname$(nullptr) {};\n");
+    format("inline $classname$() : $classname$(nullptr) {}\n");
   } else {
     format("$classname$();\n");
   }

src/google/protobuf/compiler/main.cc

@@ -28,8 +28,6 @@
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-// Author: kenton@google.com (Kenton Varda)
-
 #include <google/protobuf/compiler/cpp/cpp_generator.h>
 #include <google/protobuf/compiler/java/java_generator.h>
 #include <google/protobuf/compiler/js/js_generator.h>
@@ -87,10 +85,10 @@ int ProtobufMain(int argc, char* argv[]) {
   cli.RegisterGenerator("--csharp_out", "--csharp_opt", &csharp_generator,
                         "Generate C# source file.");
 
-  // Objective C
+  // Objective-C
   objectivec::ObjectiveCGenerator objc_generator;
   cli.RegisterGenerator("--objc_out", "--objc_opt", &objc_generator,
-                        "Generate Objective C header and source.");
+                        "Generate Objective-C header and source.");
 
   // JavaScript
   js::Generator js_generator;

src/google/protobuf/compiler/plugin.pb.h

@@ -115,7 +115,7 @@ inline bool CodeGeneratorResponse_Feature_Parse(
 class PROTOC_EXPORT Version PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.compiler.Version) */ {
  public:
-  inline Version() : Version(nullptr) {};
+  inline Version() : Version(nullptr) {}
   virtual ~Version();
 
   Version(const Version& from);
@@ -316,7 +316,7 @@ class PROTOC_EXPORT Version PROTOBUF_FINAL :
 class PROTOC_EXPORT CodeGeneratorRequest PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.compiler.CodeGeneratorRequest) */ {
  public:
-  inline CodeGeneratorRequest() : CodeGeneratorRequest(nullptr) {};
+  inline CodeGeneratorRequest() : CodeGeneratorRequest(nullptr) {}
   virtual ~CodeGeneratorRequest();
 
   CodeGeneratorRequest(const CodeGeneratorRequest& from);
@@ -538,7 +538,7 @@ class PROTOC_EXPORT CodeGeneratorRequest PROTOBUF_FINAL :
 class PROTOC_EXPORT CodeGeneratorResponse_File PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.compiler.CodeGeneratorResponse.File) */ {
  public:
-  inline CodeGeneratorResponse_File() : CodeGeneratorResponse_File(nullptr) {};
+  inline CodeGeneratorResponse_File() : CodeGeneratorResponse_File(nullptr) {}
   virtual ~CodeGeneratorResponse_File();
 
   CodeGeneratorResponse_File(const CodeGeneratorResponse_File& from);
@@ -738,7 +738,7 @@ class PROTOC_EXPORT CodeGeneratorResponse_File PROTOBUF_FINAL :
 class PROTOC_EXPORT CodeGeneratorResponse PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.compiler.CodeGeneratorResponse) */ {
  public:
-  inline CodeGeneratorResponse() : CodeGeneratorResponse(nullptr) {};
+  inline CodeGeneratorResponse() : CodeGeneratorResponse(nullptr) {}
   virtual ~CodeGeneratorResponse();
 
   CodeGeneratorResponse(const CodeGeneratorResponse& from);

src/google/protobuf/compiler/python/python_generator.cc

@@ -106,11 +106,12 @@ std::string ModuleAlias(const std::string& filename) {
 
 // Keywords reserved by the Python language.
 const char* const kKeywords[] = {
-    "False",   "None",     "True",     "and",    "as",    "assert", "break",
-    "class",   "continue", "def",      "del",    "elif",  "else",   "except",
-    "finally", "for",      "from",     "global", "if",    "import", "in",
-    "is",      "lambda",   "nonlocal", "not",    "or",    "pass",   "raise",
-    "return",  "try",      "while",    "with",   "yield", "print",
+    "False",  "None",     "True",  "and",    "as",       "assert",
+    "async",  "await",    "break", "class",  "continue", "def",
+    "del",    "elif",     "else",  "except", "finally",  "for",
+    "from",   "global",   "if",    "import", "in",       "is",
+    "lambda", "nonlocal", "not",   "or",     "pass",     "raise",
+    "return", "try",      "while", "with",   "yield",    "print",
 };
 const char* const* kKeywordsEnd =
     kKeywords + (sizeof(kKeywords) / sizeof(kKeywords[0]));

src/google/protobuf/descriptor.pb.h

@@ -333,7 +333,7 @@ inline bool MethodOptions_IdempotencyLevel_Parse(
 class PROTOBUF_EXPORT FileDescriptorSet PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.FileDescriptorSet) */ {
  public:
-  inline FileDescriptorSet() : FileDescriptorSet(nullptr) {};
+  inline FileDescriptorSet() : FileDescriptorSet(nullptr) {}
   virtual ~FileDescriptorSet();
 
   FileDescriptorSet(const FileDescriptorSet& from);
@@ -486,7 +486,7 @@ class PROTOBUF_EXPORT FileDescriptorSet PROTOBUF_FINAL :
 class PROTOBUF_EXPORT FileDescriptorProto PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.FileDescriptorProto) */ {
  public:
-  inline FileDescriptorProto() : FileDescriptorProto(nullptr) {};
+  inline FileDescriptorProto() : FileDescriptorProto(nullptr) {}
   virtual ~FileDescriptorProto();
 
   FileDescriptorProto(const FileDescriptorProto& from);
@@ -880,7 +880,7 @@ class PROTOBUF_EXPORT FileDescriptorProto PROTOBUF_FINAL :
 class PROTOBUF_EXPORT DescriptorProto_ExtensionRange PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.DescriptorProto.ExtensionRange) */ {
  public:
-  inline DescriptorProto_ExtensionRange() : DescriptorProto_ExtensionRange(nullptr) {};
+  inline DescriptorProto_ExtensionRange() : DescriptorProto_ExtensionRange(nullptr) {}
   virtual ~DescriptorProto_ExtensionRange();
 
   DescriptorProto_ExtensionRange(const DescriptorProto_ExtensionRange& from);
@@ -1064,7 +1064,7 @@ class PROTOBUF_EXPORT DescriptorProto_ExtensionRange PROTOBUF_FINAL :
 class PROTOBUF_EXPORT DescriptorProto_ReservedRange PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.DescriptorProto.ReservedRange) */ {
  public:
-  inline DescriptorProto_ReservedRange() : DescriptorProto_ReservedRange(nullptr) {};
+  inline DescriptorProto_ReservedRange() : DescriptorProto_ReservedRange(nullptr) {}
   virtual ~DescriptorProto_ReservedRange();
 
   DescriptorProto_ReservedRange(const DescriptorProto_ReservedRange& from);
@@ -1228,7 +1228,7 @@ class PROTOBUF_EXPORT DescriptorProto_ReservedRange PROTOBUF_FINAL :
 class PROTOBUF_EXPORT DescriptorProto PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.DescriptorProto) */ {
  public:
-  inline DescriptorProto() : DescriptorProto(nullptr) {};
+  inline DescriptorProto() : DescriptorProto(nullptr) {}
   virtual ~DescriptorProto();
 
   DescriptorProto(const DescriptorProto& from);
@@ -1573,7 +1573,7 @@ class PROTOBUF_EXPORT DescriptorProto PROTOBUF_FINAL :
 class PROTOBUF_EXPORT ExtensionRangeOptions PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.ExtensionRangeOptions) */ {
  public:
-  inline ExtensionRangeOptions() : ExtensionRangeOptions(nullptr) {};
+  inline ExtensionRangeOptions() : ExtensionRangeOptions(nullptr) {}
   virtual ~ExtensionRangeOptions();
 
   ExtensionRangeOptions(const ExtensionRangeOptions& from);
@@ -1729,7 +1729,7 @@ class PROTOBUF_EXPORT ExtensionRangeOptions PROTOBUF_FINAL :
 class PROTOBUF_EXPORT FieldDescriptorProto PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.FieldDescriptorProto) */ {
  public:
-  inline FieldDescriptorProto() : FieldDescriptorProto(nullptr) {};
+  inline FieldDescriptorProto() : FieldDescriptorProto(nullptr) {}
   virtual ~FieldDescriptorProto();
 
   FieldDescriptorProto(const FieldDescriptorProto& from);
@@ -2162,7 +2162,7 @@ class PROTOBUF_EXPORT FieldDescriptorProto PROTOBUF_FINAL :
 class PROTOBUF_EXPORT OneofDescriptorProto PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.OneofDescriptorProto) */ {
  public:
-  inline OneofDescriptorProto() : OneofDescriptorProto(nullptr) {};
+  inline OneofDescriptorProto() : OneofDescriptorProto(nullptr) {}
   virtual ~OneofDescriptorProto();
 
   OneofDescriptorProto(const OneofDescriptorProto& from);
@@ -2338,7 +2338,7 @@ class PROTOBUF_EXPORT OneofDescriptorProto PROTOBUF_FINAL :
 class PROTOBUF_EXPORT EnumDescriptorProto_EnumReservedRange PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.EnumDescriptorProto.EnumReservedRange) */ {
  public:
-  inline EnumDescriptorProto_EnumReservedRange() : EnumDescriptorProto_EnumReservedRange(nullptr) {};
+  inline EnumDescriptorProto_EnumReservedRange() : EnumDescriptorProto_EnumReservedRange(nullptr) {}
   virtual ~EnumDescriptorProto_EnumReservedRange();
 
   EnumDescriptorProto_EnumReservedRange(const EnumDescriptorProto_EnumReservedRange& from);
@@ -2502,7 +2502,7 @@ class PROTOBUF_EXPORT EnumDescriptorProto_EnumReservedRange PROTOBUF_FINAL :
 class PROTOBUF_EXPORT EnumDescriptorProto PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.EnumDescriptorProto) */ {
  public:
-  inline EnumDescriptorProto() : EnumDescriptorProto(nullptr) {};
+  inline EnumDescriptorProto() : EnumDescriptorProto(nullptr) {}
   virtual ~EnumDescriptorProto();
 
   EnumDescriptorProto(const EnumDescriptorProto& from);
@@ -2746,7 +2746,7 @@ class PROTOBUF_EXPORT EnumDescriptorProto PROTOBUF_FINAL :
 class PROTOBUF_EXPORT EnumValueDescriptorProto PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.EnumValueDescriptorProto) */ {
  public:
-  inline EnumValueDescriptorProto() : EnumValueDescriptorProto(nullptr) {};
+  inline EnumValueDescriptorProto() : EnumValueDescriptorProto(nullptr) {}
   virtual ~EnumValueDescriptorProto();
 
   EnumValueDescriptorProto(const EnumValueDescriptorProto& from);
@@ -2937,7 +2937,7 @@ class PROTOBUF_EXPORT EnumValueDescriptorProto PROTOBUF_FINAL :
 class PROTOBUF_EXPORT ServiceDescriptorProto PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.ServiceDescriptorProto) */ {
  public:
-  inline ServiceDescriptorProto() : ServiceDescriptorProto(nullptr) {};
+  inline ServiceDescriptorProto() : ServiceDescriptorProto(nullptr) {}
   virtual ~ServiceDescriptorProto();
 
   ServiceDescriptorProto(const ServiceDescriptorProto& from);
@@ -3133,7 +3133,7 @@ class PROTOBUF_EXPORT ServiceDescriptorProto PROTOBUF_FINAL :
 class PROTOBUF_EXPORT MethodDescriptorProto PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.MethodDescriptorProto) */ {
  public:
-  inline MethodDescriptorProto() : MethodDescriptorProto(nullptr) {};
+  inline MethodDescriptorProto() : MethodDescriptorProto(nullptr) {}
   virtual ~MethodDescriptorProto();
 
   MethodDescriptorProto(const MethodDescriptorProto& from);
@@ -3383,7 +3383,7 @@ class PROTOBUF_EXPORT MethodDescriptorProto PROTOBUF_FINAL :
 class PROTOBUF_EXPORT FileOptions PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.FileOptions) */ {
  public:
-  inline FileOptions() : FileOptions(nullptr) {};
+  inline FileOptions() : FileOptions(nullptr) {}
   virtual ~FileOptions();
 
   FileOptions(const FileOptions& from);
@@ -3942,7 +3942,7 @@ class PROTOBUF_EXPORT FileOptions PROTOBUF_FINAL :
 class PROTOBUF_EXPORT MessageOptions PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.MessageOptions) */ {
  public:
-  inline MessageOptions() : MessageOptions(nullptr) {};
+  inline MessageOptions() : MessageOptions(nullptr) {}
   virtual ~MessageOptions();
 
   MessageOptions(const MessageOptions& from);
@@ -4159,7 +4159,7 @@ class PROTOBUF_EXPORT MessageOptions PROTOBUF_FINAL :
 class PROTOBUF_EXPORT FieldOptions PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.FieldOptions) */ {
  public:
-  inline FieldOptions() : FieldOptions(nullptr) {};
+  inline FieldOptions() : FieldOptions(nullptr) {}
   virtual ~FieldOptions();
 
   FieldOptions(const FieldOptions& from);
@@ -4470,7 +4470,7 @@ class PROTOBUF_EXPORT FieldOptions PROTOBUF_FINAL :
 class PROTOBUF_EXPORT OneofOptions PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.OneofOptions) */ {
  public:
-  inline OneofOptions() : OneofOptions(nullptr) {};
+  inline OneofOptions() : OneofOptions(nullptr) {}
   virtual ~OneofOptions();
 
   OneofOptions(const OneofOptions& from);
@@ -4626,7 +4626,7 @@ class PROTOBUF_EXPORT OneofOptions PROTOBUF_FINAL :
 class PROTOBUF_EXPORT EnumOptions PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.EnumOptions) */ {
  public:
-  inline EnumOptions() : EnumOptions(nullptr) {};
+  inline EnumOptions() : EnumOptions(nullptr) {}
   virtual ~EnumOptions();
 
   EnumOptions(const EnumOptions& from);
@@ -4813,7 +4813,7 @@ class PROTOBUF_EXPORT EnumOptions PROTOBUF_FINAL :
 class PROTOBUF_EXPORT EnumValueOptions PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.EnumValueOptions) */ {
  public:
-  inline EnumValueOptions() : EnumValueOptions(nullptr) {};
+  inline EnumValueOptions() : EnumValueOptions(nullptr) {}
   virtual ~EnumValueOptions();
 
   EnumValueOptions(const EnumValueOptions& from);
@@ -4985,7 +4985,7 @@ class PROTOBUF_EXPORT EnumValueOptions PROTOBUF_FINAL :
 class PROTOBUF_EXPORT ServiceOptions PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.ServiceOptions) */ {
  public:
-  inline ServiceOptions() : ServiceOptions(nullptr) {};
+  inline ServiceOptions() : ServiceOptions(nullptr) {}
   virtual ~ServiceOptions();
 
   ServiceOptions(const ServiceOptions& from);
@@ -5157,7 +5157,7 @@ class PROTOBUF_EXPORT ServiceOptions PROTOBUF_FINAL :
 class PROTOBUF_EXPORT MethodOptions PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.MethodOptions) */ {
  public:
-  inline MethodOptions() : MethodOptions(nullptr) {};
+  inline MethodOptions() : MethodOptions(nullptr) {}
   virtual ~MethodOptions();
 
   MethodOptions(const MethodOptions& from);
@@ -5376,7 +5376,7 @@ class PROTOBUF_EXPORT MethodOptions PROTOBUF_FINAL :
 class PROTOBUF_EXPORT UninterpretedOption_NamePart PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.UninterpretedOption.NamePart) */ {
  public:
-  inline UninterpretedOption_NamePart() : UninterpretedOption_NamePart(nullptr) {};
+  inline UninterpretedOption_NamePart() : UninterpretedOption_NamePart(nullptr) {}
   virtual ~UninterpretedOption_NamePart();
 
   UninterpretedOption_NamePart(const UninterpretedOption_NamePart& from);
@@ -5550,7 +5550,7 @@ class PROTOBUF_EXPORT UninterpretedOption_NamePart PROTOBUF_FINAL :
 class PROTOBUF_EXPORT UninterpretedOption PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.UninterpretedOption) */ {
  public:
-  inline UninterpretedOption() : UninterpretedOption(nullptr) {};
+  inline UninterpretedOption() : UninterpretedOption(nullptr) {}
   virtual ~UninterpretedOption();
 
   UninterpretedOption(const UninterpretedOption& from);
@@ -5817,7 +5817,7 @@ class PROTOBUF_EXPORT UninterpretedOption PROTOBUF_FINAL :
 class PROTOBUF_EXPORT SourceCodeInfo_Location PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.SourceCodeInfo.Location) */ {
  public:
-  inline SourceCodeInfo_Location() : SourceCodeInfo_Location(nullptr) {};
+  inline SourceCodeInfo_Location() : SourceCodeInfo_Location(nullptr) {}
   virtual ~SourceCodeInfo_Location();
 
   SourceCodeInfo_Location(const SourceCodeInfo_Location& from);
@@ -6071,7 +6071,7 @@ class PROTOBUF_EXPORT SourceCodeInfo_Location PROTOBUF_FINAL :
 class PROTOBUF_EXPORT SourceCodeInfo PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.SourceCodeInfo) */ {
  public:
-  inline SourceCodeInfo() : SourceCodeInfo(nullptr) {};
+  inline SourceCodeInfo() : SourceCodeInfo(nullptr) {}
   virtual ~SourceCodeInfo();
 
   SourceCodeInfo(const SourceCodeInfo& from);
@@ -6226,7 +6226,7 @@ class PROTOBUF_EXPORT SourceCodeInfo PROTOBUF_FINAL :
 class PROTOBUF_EXPORT GeneratedCodeInfo_Annotation PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.GeneratedCodeInfo.Annotation) */ {
  public:
-  inline GeneratedCodeInfo_Annotation() : GeneratedCodeInfo_Annotation(nullptr) {};
+  inline GeneratedCodeInfo_Annotation() : GeneratedCodeInfo_Annotation(nullptr) {}
   virtual ~GeneratedCodeInfo_Annotation();
 
   GeneratedCodeInfo_Annotation(const GeneratedCodeInfo_Annotation& from);
@@ -6437,7 +6437,7 @@ class PROTOBUF_EXPORT GeneratedCodeInfo_Annotation PROTOBUF_FINAL :
 class PROTOBUF_EXPORT GeneratedCodeInfo PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.GeneratedCodeInfo) */ {
  public:
-  inline GeneratedCodeInfo() : GeneratedCodeInfo(nullptr) {};
+  inline GeneratedCodeInfo() : GeneratedCodeInfo(nullptr) {}
   virtual ~GeneratedCodeInfo();
 
   GeneratedCodeInfo(const GeneratedCodeInfo& from);

src/google/protobuf/duration.pb.h

@@ -69,7 +69,7 @@ PROTOBUF_NAMESPACE_OPEN
 class PROTOBUF_EXPORT Duration PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.Duration) */ {
  public:
-  inline Duration() : Duration(nullptr) {};
+  inline Duration() : Duration(nullptr) {}
   virtual ~Duration();
 
   Duration(const Duration& from);

src/google/protobuf/empty.pb.h

@@ -69,7 +69,7 @@ PROTOBUF_NAMESPACE_OPEN
 class PROTOBUF_EXPORT Empty PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.Empty) */ {
  public:
-  inline Empty() : Empty(nullptr) {};
+  inline Empty() : Empty(nullptr) {}
   virtual ~Empty();
 
   Empty(const Empty& from);

src/google/protobuf/field_mask.pb.h

@@ -69,7 +69,7 @@ PROTOBUF_NAMESPACE_OPEN
 class PROTOBUF_EXPORT FieldMask PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.FieldMask) */ {
  public:
-  inline FieldMask() : FieldMask(nullptr) {};
+  inline FieldMask() : FieldMask(nullptr) {}
   virtual ~FieldMask();
 
   FieldMask(const FieldMask& from);

src/google/protobuf/map_entry_lite.h

@@ -266,13 +266,8 @@ class MapEntryImpl : public Base {
 
   size_t ByteSizeLong() const override {
     size_t size = 0;
-    size += has_key() ? kTagSize +
-                            static_cast<size_t>(KeyTypeHandler::ByteSize(key()))
-                      : 0;
-    size += has_value()
-                ? kTagSize +
-                      static_cast<size_t>(ValueTypeHandler::ByteSize(value()))
-                : 0;
+    size += kTagSize + static_cast<size_t>(KeyTypeHandler::ByteSize(key()));
+    size += kTagSize + static_cast<size_t>(ValueTypeHandler::ByteSize(value()));
     return size;
   }
 

src/google/protobuf/map_type_handler.h

@@ -240,7 +240,7 @@ class MapTypeHandler<WireFormatLite::TYPE_MESSAGE, Type> {
                                                int default_enum);             \
     static inline size_t SpaceUsedInMapEntryLong(const TypeOnMemory& value);  \
     static inline size_t SpaceUsedInMapLong(const TypeOnMemory& value);       \
-    static inline size_t SpaceUsedInMapLong(const std::string& value);        \
+    static inline size_t SpaceUsedInMapLong(ConstStringParam value);          \
     static inline void AssignDefaultValue(TypeOnMemory* value);               \
     static inline const MapEntryAccessorType& DefaultIfNotInitialized(        \
         const TypeOnMemory& value, const TypeOnMemory& default_value);        \
@@ -643,8 +643,8 @@ inline bool MapTypeHandler<WireFormatLite::TYPE_MESSAGE, Type>::IsInitialized(
   template <typename Type>                                                    \
   inline size_t                                                               \
   MapTypeHandler<WireFormatLite::TYPE_##FieldType, Type>::SpaceUsedInMapLong( \
-      const std::string& value) {                                             \
-    return sizeof(value);                                                     \
+      ConstStringParam value) {                                               \
+    return sizeof(std::string);                                               \
   }                                                                           \
   template <typename Type>                                                    \
   inline void MapTypeHandler<WireFormatLite::TYPE_##FieldType, Type>::Clear(  \

src/google/protobuf/repeated_field.h

@@ -439,43 +439,50 @@ class RepeatedField final {
   //
   // The first version executes at 7 cycles per iteration while the second
   // version near 1 or 2 cycles.
-  class FastAdder {
+  template <int = 0, bool = std::is_pod<Element>::value>
+  class FastAdderImpl {
    public:
-    explicit FastAdder(RepeatedField* rf) : repeated_field_(rf) {
-      if (kIsPod) {
-        index_ = repeated_field_->current_size_;
-        capacity_ = repeated_field_->total_size_;
-        buffer_ = repeated_field_->unsafe_elements();
-      }
-    }
-    ~FastAdder() {
-      if (kIsPod) repeated_field_->current_size_ = index_;
+    explicit FastAdderImpl(RepeatedField* rf) : repeated_field_(rf) {
+      index_ = repeated_field_->current_size_;
+      capacity_ = repeated_field_->total_size_;
+      buffer_ = repeated_field_->unsafe_elements();
     }
+    ~FastAdderImpl() { repeated_field_->current_size_ = index_; }
 
-    void Add(const Element& val) {
-      if (kIsPod) {
-        if (index_ == capacity_) {
-          repeated_field_->current_size_ = index_;
-          repeated_field_->Reserve(index_ + 1);
-          capacity_ = repeated_field_->total_size_;
-          buffer_ = repeated_field_->unsafe_elements();
-        }
-        buffer_[index_++] = val;
-      } else {
-        repeated_field_->Add(val);
+    void Add(Element val) {
+      if (index_ == capacity_) {
+        repeated_field_->current_size_ = index_;
+        repeated_field_->Reserve(index_ + 1);
+        capacity_ = repeated_field_->total_size_;
+        buffer_ = repeated_field_->unsafe_elements();
       }
+      buffer_[index_++] = val;
     }
 
    private:
-    constexpr static bool kIsPod = std::is_pod<Element>::value;
     RepeatedField* repeated_field_;
     int index_;
     int capacity_;
     Element* buffer_;
 
-    GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(FastAdder);
+    GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(FastAdderImpl);
   };
 
+  // FastAdder is a wrapper for adding fields. The specialization above handles
+  // POD types more efficiently than RepeatedField.
+  template <int I>
+  class FastAdderImpl<I, false> {
+   public:
+    explicit FastAdderImpl(RepeatedField* rf) : repeated_field_(rf) {}
+    void Add(const Element& val) { repeated_field_->Add(val); }
+
+   private:
+    RepeatedField* repeated_field_;
+    GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(FastAdderImpl);
+  };
+
+  using FastAdder = FastAdderImpl<>;
+
   friend class TestRepeatedFieldHelper;
   friend class ::google::protobuf::internal::ParseContext;
 };
@@ -1329,8 +1336,15 @@ inline void RepeatedField<Element>::Set(int index, const Element& value) {
 template <typename Element>
 inline void RepeatedField<Element>::Add(const Element& value) {
   uint32 size = current_size_;
-  if (static_cast<int>(size) == total_size_) Reserve(total_size_ + 1);
-  elements()[size] = value;
+  if (static_cast<int>(size) == total_size_) {
+    // value could reference an element of the array. Reserving new space will
+    // invalidate the reference. So we must make a copy first.
+    auto tmp = value;
+    Reserve(total_size_ + 1);
+    elements()[size] = std::move(tmp);
+  } else {
+    elements()[size] = value;
+  }
   current_size_ = size + 1;
 }
 

src/google/protobuf/repeated_field_unittest.cc

@@ -807,6 +807,14 @@ TEST(RepeatedField, ClearThenReserveMore) {
   // strings.
 }
 
+TEST(RepeatedField, TestSAddFromSelf) {
+  RepeatedField<int> field;
+  field.Add(0);
+  for (int i = 0; i < 1000; i++) {
+    field.Add(field[0]);
+  }
+}
+
 // ===================================================================
 // RepeatedPtrField tests.  These pretty much just mirror the RepeatedField
 // tests above.

src/google/protobuf/source_context.pb.h

@@ -69,7 +69,7 @@ PROTOBUF_NAMESPACE_OPEN
 class PROTOBUF_EXPORT SourceContext PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.SourceContext) */ {
  public:
-  inline SourceContext() : SourceContext(nullptr) {};
+  inline SourceContext() : SourceContext(nullptr) {}
   virtual ~SourceContext();
 
   SourceContext(const SourceContext& from);

src/google/protobuf/struct.pb.h

@@ -141,7 +141,7 @@ public:
 class PROTOBUF_EXPORT Struct PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.Struct) */ {
  public:
-  inline Struct() : Struct(nullptr) {};
+  inline Struct() : Struct(nullptr) {}
   virtual ~Struct();
 
   Struct(const Struct& from);
@@ -292,7 +292,7 @@ class PROTOBUF_EXPORT Struct PROTOBUF_FINAL :
 class PROTOBUF_EXPORT Value PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.Value) */ {
  public:
-  inline Value() : Value(nullptr) {};
+  inline Value() : Value(nullptr) {}
   virtual ~Value();
 
   Value(const Value& from);
@@ -547,7 +547,7 @@ class PROTOBUF_EXPORT Value PROTOBUF_FINAL :
 class PROTOBUF_EXPORT ListValue PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.ListValue) */ {
  public:
-  inline ListValue() : ListValue(nullptr) {};
+  inline ListValue() : ListValue(nullptr) {}
   virtual ~ListValue();
 
   ListValue(const ListValue& from);

src/google/protobuf/timestamp.pb.h

@@ -69,7 +69,7 @@ PROTOBUF_NAMESPACE_OPEN
 class PROTOBUF_EXPORT Timestamp PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.Timestamp) */ {
  public:
-  inline Timestamp() : Timestamp(nullptr) {};
+  inline Timestamp() : Timestamp(nullptr) {}
   virtual ~Timestamp();
 
   Timestamp(const Timestamp& from);

src/google/protobuf/type.pb.h

@@ -182,7 +182,7 @@ inline bool Syntax_Parse(
 class PROTOBUF_EXPORT Type PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.Type) */ {
  public:
-  inline Type() : Type(nullptr) {};
+  inline Type() : Type(nullptr) {}
   virtual ~Type();
 
   Type(const Type& from);
@@ -423,7 +423,7 @@ class PROTOBUF_EXPORT Type PROTOBUF_FINAL :
 class PROTOBUF_EXPORT Field PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.Field) */ {
  public:
-  inline Field() : Field(nullptr) {};
+  inline Field() : Field(nullptr) {}
   virtual ~Field();
 
   Field(const Field& from);
@@ -794,7 +794,7 @@ class PROTOBUF_EXPORT Field PROTOBUF_FINAL :
 class PROTOBUF_EXPORT Enum PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.Enum) */ {
  public:
-  inline Enum() : Enum(nullptr) {};
+  inline Enum() : Enum(nullptr) {}
   virtual ~Enum();
 
   Enum(const Enum& from);
@@ -1009,7 +1009,7 @@ class PROTOBUF_EXPORT Enum PROTOBUF_FINAL :
 class PROTOBUF_EXPORT EnumValue PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.EnumValue) */ {
  public:
-  inline EnumValue() : EnumValue(nullptr) {};
+  inline EnumValue() : EnumValue(nullptr) {}
   virtual ~EnumValue();
 
   EnumValue(const EnumValue& from);
@@ -1184,7 +1184,7 @@ class PROTOBUF_EXPORT EnumValue PROTOBUF_FINAL :
 class PROTOBUF_EXPORT Option PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.Option) */ {
  public:
-  inline Option() : Option(nullptr) {};
+  inline Option() : Option(nullptr) {}
   virtual ~Option();
 
   Option(const Option& from);

src/google/protobuf/util/internal/json_stream_parser.cc

@@ -82,8 +82,8 @@ inline bool IsAlphanumeric(char c) {
 
 // Indicates a character may not be part of an unquoted key.
 inline bool IsKeySeparator(char c) {
-  return (ascii_isspace(c) || c == '"' || c == '\'' || c == '{' || c == '}' ||
-          c == '[' || c == ']' || c == ':' || c == ',');
+  return (ascii_isspace(c) || c == '"' || c == '\'' || c == '{' ||
+          c == '}' || c == '[' || c == ']' || c == ':' || c == ',');
 }
 
 inline void ReplaceInvalidCodePoints(StringPiece str,

src/google/protobuf/util/internal/proto_writer.cc

@@ -406,7 +406,9 @@ std::string ProtoWriter::ProtoElement::ToString() const {
         now->parent()->parent_field_ != now->parent_field_) {
       std::string name = now->parent_field_->name();
       int i = 0;
-      while (i < name.size() && (ascii_isalnum(name[i]) || name[i] == '_')) ++i;
+      while (i < name.size() &&
+             (ascii_isalnum(name[i]) || name[i] == '_'))
+        ++i;
       if (i > 0 && i == name.size()) {  // safe field name
         if (loc.empty()) {
           loc = name;

src/google/protobuf/util/internal/utility.cc

@@ -270,7 +270,8 @@ std::string ToCamelCase(StringPiece input) {
       // 1) following a lowercase:   "...aB..."
       // 2) followed by a lowercase: "...ABc..."
       if (!result.empty() && is_cap &&
-          (!was_cap || (i + 1 < input.size() && ascii_islower(input[i + 1])))) {
+          (!was_cap ||
+           (i + 1 < input.size() && ascii_islower(input[i + 1])))) {
         first_word = false;
         result.push_back(input[i]);
       } else {
@@ -310,9 +311,9 @@ std::string ToSnakeCase(StringPiece input) {
       //    (e.g. "GoogleLAB" => "google_lab")
       // 4) Followed by a lowercase: "...ABc..." => "...a_bc..."
       //    (e.g. "GBike" => "g_bike")
-      if (was_not_underscore &&               //            case 1 out
-          (was_not_cap ||                     // case 2 in, case 3 out
-           (i + 1 < input.size() &&           //            case 3 out
+      if (was_not_underscore &&                     //            case 1 out
+          (was_not_cap ||                           // case 2 in, case 3 out
+           (i + 1 < input.size() &&                 //            case 3 out
             ascii_islower(input[i + 1])))) {  // case 4 in
         // We add an underscore for case 2 and case 4.
         result.push_back('_');

src/google/protobuf/wrappers.pb.h

@@ -101,7 +101,7 @@ PROTOBUF_NAMESPACE_OPEN
 class PROTOBUF_EXPORT DoubleValue PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.DoubleValue) */ {
  public:
-  inline DoubleValue() : DoubleValue(nullptr) {};
+  inline DoubleValue() : DoubleValue(nullptr) {}
   virtual ~DoubleValue();
 
   DoubleValue(const DoubleValue& from);
@@ -238,7 +238,7 @@ class PROTOBUF_EXPORT DoubleValue PROTOBUF_FINAL :
 class PROTOBUF_EXPORT FloatValue PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.FloatValue) */ {
  public:
-  inline FloatValue() : FloatValue(nullptr) {};
+  inline FloatValue() : FloatValue(nullptr) {}
   virtual ~FloatValue();
 
   FloatValue(const FloatValue& from);
@@ -375,7 +375,7 @@ class PROTOBUF_EXPORT FloatValue PROTOBUF_FINAL :
 class PROTOBUF_EXPORT Int64Value PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.Int64Value) */ {
  public:
-  inline Int64Value() : Int64Value(nullptr) {};
+  inline Int64Value() : Int64Value(nullptr) {}
   virtual ~Int64Value();
 
   Int64Value(const Int64Value& from);
@@ -512,7 +512,7 @@ class PROTOBUF_EXPORT Int64Value PROTOBUF_FINAL :
 class PROTOBUF_EXPORT UInt64Value PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.UInt64Value) */ {
  public:
-  inline UInt64Value() : UInt64Value(nullptr) {};
+  inline UInt64Value() : UInt64Value(nullptr) {}
   virtual ~UInt64Value();
 
   UInt64Value(const UInt64Value& from);
@@ -649,7 +649,7 @@ class PROTOBUF_EXPORT UInt64Value PROTOBUF_FINAL :
 class PROTOBUF_EXPORT Int32Value PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.Int32Value) */ {
  public:
-  inline Int32Value() : Int32Value(nullptr) {};
+  inline Int32Value() : Int32Value(nullptr) {}
   virtual ~Int32Value();
 
   Int32Value(const Int32Value& from);
@@ -786,7 +786,7 @@ class PROTOBUF_EXPORT Int32Value PROTOBUF_FINAL :
 class PROTOBUF_EXPORT UInt32Value PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.UInt32Value) */ {
  public:
-  inline UInt32Value() : UInt32Value(nullptr) {};
+  inline UInt32Value() : UInt32Value(nullptr) {}
   virtual ~UInt32Value();
 
   UInt32Value(const UInt32Value& from);
@@ -923,7 +923,7 @@ class PROTOBUF_EXPORT UInt32Value PROTOBUF_FINAL :
 class PROTOBUF_EXPORT BoolValue PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.BoolValue) */ {
  public:
-  inline BoolValue() : BoolValue(nullptr) {};
+  inline BoolValue() : BoolValue(nullptr) {}
   virtual ~BoolValue();
 
   BoolValue(const BoolValue& from);
@@ -1060,7 +1060,7 @@ class PROTOBUF_EXPORT BoolValue PROTOBUF_FINAL :
 class PROTOBUF_EXPORT StringValue PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.StringValue) */ {
  public:
-  inline StringValue() : StringValue(nullptr) {};
+  inline StringValue() : StringValue(nullptr) {}
   virtual ~StringValue();
 
   StringValue(const StringValue& from);
@@ -1204,7 +1204,7 @@ class PROTOBUF_EXPORT StringValue PROTOBUF_FINAL :
 class PROTOBUF_EXPORT BytesValue PROTOBUF_FINAL :
     public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.BytesValue) */ {
  public:
-  inline BytesValue() : BytesValue(nullptr) {};
+  inline BytesValue() : BytesValue(nullptr) {}
   virtual ~BytesValue();
 
   BytesValue(const BytesValue& from);