Down Integrate Internal Changes (#6634)

* Down integrate to GitHub

* Remove unintentional double assign

* Change ByteCountInt64 to int64_t

* Revert changes in binary_conformance_test.cc

* Revert unnecessary changes

java/core/src/main/java/com/google/protobuf/AbstractMessage.java

@@ -193,7 +193,7 @@ public abstract class AbstractMessage
     if (list.isEmpty()) {
       return Collections.emptyMap();
     }
-    Map result = new HashMap();
+    Map result = new HashMap<>();
     Iterator iterator = list.iterator();
     Message entry = (Message) iterator.next();
     Descriptors.Descriptor descriptor = entry.getDescriptorForType();

java/core/src/main/java/com/google/protobuf/ByteString.java

@@ -30,6 +30,10 @@
 
 package com.google.protobuf;
 
+import static com.google.protobuf.TextFormatEscaper.escapeBytes;
+import static java.lang.Integer.toHexString;
+import static java.lang.System.identityHashCode;
+
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -49,6 +53,7 @@ import java.util.Collections;
 import java.util.Comparator;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Locale;
 import java.util.NoSuchElementException;
 
 /**
@@ -1268,7 +1273,17 @@ public abstract class ByteString implements Iterable<Byte>, Serializable {
   @Override
   public final String toString() {
     return String.format(
-        "<ByteString@%s size=%d>", Integer.toHexString(System.identityHashCode(this)), size());
+        Locale.ROOT,
+        "<ByteString@%s size=%d contents=\"%s\">",
+        toHexString(identityHashCode(this)),
+        size(),
+        truncateAndEscapeForDisplay());
+  }
+
+  private String truncateAndEscapeForDisplay() {
+    final int limit = 50;
+
+    return size() <= limit ? escapeBytes(this) : escapeBytes(substring(0, limit - 3)) + "...";
   }
 
   /**

java/core/src/main/java/com/google/protobuf/FieldSet.java

@@ -526,7 +526,7 @@ final class FieldSet<T extends FieldSet.FieldDescriptorLite<T>> {
     if (descriptor.isRepeated()) {
       Object value = getField(descriptor);
       if (value == null) {
-        value = new ArrayList();
+        value = new ArrayList<>();
       }
       for (Object element : (List) otherValue) {
         ((List) value).add(cloneIfMutable(element));
@@ -1266,7 +1266,7 @@ final class FieldSet<T extends FieldSet.FieldDescriptorLite<T>> {
       if (descriptor.isRepeated()) {
         Object value = getField(descriptor);
         if (value == null) {
-          value = new ArrayList();
+          value = new ArrayList<>();
         }
         for (Object element : (List) otherValue) {
           ((List) value).add(FieldSet.cloneIfMutable(element));

java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java

@@ -1238,7 +1238,7 @@ public abstract class GeneratedMessageLite<
     Object fromFieldSetType(final Object value) {
       if (descriptor.isRepeated()) {
         if (descriptor.getLiteJavaType() == WireFormat.JavaType.ENUM) {
-          final List result = new ArrayList();
+          final List result = new ArrayList<>();
           for (final Object element : (List) value) {
             result.add(singularFromFieldSetType(element));
           }
@@ -1263,7 +1263,7 @@ public abstract class GeneratedMessageLite<
     Object toFieldSetType(final Object value) {
       if (descriptor.isRepeated()) {
         if (descriptor.getLiteJavaType() == WireFormat.JavaType.ENUM) {
-          final List result = new ArrayList();
+          final List result = new ArrayList<>();
           for (final Object element : (List) value) {
             result.add(singularToFieldSetType(element));
           }

java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java

@@ -158,7 +158,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage
 
   /**
    * Internal helper to return a modifiable map containing all the fields.
-   * The returned Map is modifialbe so that the caller can add additional
+   * The returned Map is modifiable so that the caller can add additional
    * extension fields to implement {@link #getAllFields()}.
    *
    * @param getBytesForString whether to generate ByteString for string fields
@@ -2853,7 +2853,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage
       @Override
       @SuppressWarnings("unchecked")
       public Object get(GeneratedMessageV3 message) {
-        List result = new ArrayList();
+        List result = new ArrayList<>();
         for (int i = 0; i < getRepeatedCount(message); i++) {
           result.add(getRepeated(message, i));
         }
@@ -2863,7 +2863,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage
       @Override
       @SuppressWarnings("unchecked")
       public Object get(Builder builder) {
-        List result = new ArrayList();
+        List result = new ArrayList<>();
         for (int i = 0; i < getRepeatedCount(builder); i++) {
           result.add(getRepeated(builder, i));
         }
@@ -3068,7 +3068,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage
       @Override
       @SuppressWarnings("unchecked")
       public Object get(final GeneratedMessageV3 message) {
-        final List newList = new ArrayList();
+        final List newList = new ArrayList<>();
         final int size = getRepeatedCount(message);
         for (int i = 0; i < size; i++) {
           newList.add(getRepeated(message, i));
@@ -3079,7 +3079,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage
       @Override
       @SuppressWarnings("unchecked")
       public Object get(final GeneratedMessageV3.Builder builder) {
-        final List newList = new ArrayList();
+        final List newList = new ArrayList<>();
         final int size = getRepeatedCount(builder);
         for (int i = 0; i < size; i++) {
           newList.add(getRepeated(builder, i));

java/core/src/main/java/com/google/protobuf/MapEntry.java

@@ -439,7 +439,7 @@ public final class MapEntry<K, V> extends AbstractMessage {
     @Override
     @SuppressWarnings("unchecked")
     public Builder<K, V> clone() {
-      return new Builder(metadata, key, value, hasKey, hasValue);
+      return new Builder<>(metadata, key, value, hasKey, hasValue);
     }
   }
 

java/core/src/main/java/com/google/protobuf/MapFieldLite.java

@@ -58,7 +58,7 @@ public final class MapFieldLite<K, V> extends LinkedHashMap<K, V> {
   }
 
   @SuppressWarnings({"rawtypes", "unchecked"})
-  private static final MapFieldLite EMPTY_MAP_FIELD = new MapFieldLite();
+  private static final MapFieldLite EMPTY_MAP_FIELD = new MapFieldLite<>();
 
   static {
     EMPTY_MAP_FIELD.makeImmutable();

java/core/src/test/java/com/google/protobuf/ByteStringTest.java

@@ -486,6 +486,25 @@ public class ByteStringTest extends TestCase {
         "copyToStringUtf8 must respect the charset", testString, byteString.toStringUtf8());
   }
 
+  public void testToString() {
+    String toString =
+        ByteString.copyFrom("Here are some bytes: \t\u00a1".getBytes(Internal.UTF_8)).toString();
+    assertTrue(toString, toString.contains("size=24"));
+    assertTrue(toString, toString.contains("contents=\"Here are some bytes: \\t\\302\\241\""));
+  }
+
+  public void testToString_long() {
+    String toString =
+        ByteString.copyFrom(
+                "123456789012345678901234567890123456789012345678901234567890"
+                    .getBytes(Internal.UTF_8))
+            .toString();
+    assertTrue(toString, toString.contains("size=60"));
+    assertTrue(
+        toString,
+        toString.contains("contents=\"12345678901234567890123456789012345678901234567...\""));
+  }
+
   public void testNewOutput_InitialCapacity() throws IOException {
     byte[] bytes = getTestBytes();
     ByteString.Output output = ByteString.newOutput(bytes.length + 100);

java/util/src/main/java/com/google/protobuf/util/JsonFormat.java

@@ -1945,6 +1945,14 @@ public class JsonFormat {
           return field.getEnumType().findValueByNumber(0);
         }
         return null;
+      } else if (json instanceof JsonObject) {
+        if (field.getType() != FieldDescriptor.Type.MESSAGE
+            && field.getType() != FieldDescriptor.Type.GROUP) {
+          // If the field type is primitive, but the json type is JsonObject rather than
+          // JsonElement, throw a type mismatch error.
+          throw new InvalidProtocolBufferException(
+              String.format("Invalid value: %s for expected type: %s", json, field.getType()));
+        }
       }
       switch (field.getType()) {
         case INT32:

java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java

@@ -1723,6 +1723,22 @@ public class JsonFormatTest extends TestCase {
     }
   }
 
+  // Test that an error is thrown if a nested JsonObject is parsed as a primitive field.
+  public void testJsonObjectForPrimitiveField() throws Exception {
+    TestAllTypes.Builder builder = TestAllTypes.newBuilder();
+    try {
+      mergeFromJson(
+          "{\n"
+              + "  \"optionalString\": {\n"
+              + "    \"invalidNestedString\": \"Hello world\"\n"
+              + "  }\n"
+              + "}\n",
+          builder);
+    } catch (InvalidProtocolBufferException e) {
+      // Expected.
+    }
+  }
+
   public void testSortedMapKeys() throws Exception {
     TestMap.Builder mapBuilder = TestMap.newBuilder();
     mapBuilder.putStringToInt32Map("\ud834\udd20", 3); // utf-8 F0 9D 84 A0

js/binary/decoder_test.js

@@ -141,8 +141,14 @@ function doTestSignedValue(readValue,
   }
 
   // Encoding values outside the valid range should assert.
-  assertThrows(function() {writeValue.call(encoder, lowerLimit * 1.1);});
-  assertThrows(function() {writeValue.call(encoder, upperLimit * 1.1);});
+  var pastLowerLimit = lowerLimit * 1.1;
+  var pastUpperLimit = upperLimit * 1.1;
+  if (pastLowerLimit !== -Infinity) {
+    expect(() => void writeValue.call(encoder, pastLowerLimit)).toThrow();
+  }
+  if (pastUpperLimit !== Infinity) {
+    expect(() => void writeValue.call(encoder, pastUpperLimit)).toThrow();
+  }
 }
 
 describe('binaryDecoderTest', function() {

js/binary/encoder.js

@@ -382,8 +382,10 @@ jspb.BinaryEncoder.prototype.writeInt64String = function(value) {
  * @param {number} value The value to write.
  */
 jspb.BinaryEncoder.prototype.writeFloat = function(value) {
-  goog.asserts.assert((value >= -jspb.BinaryConstants.FLOAT32_MAX) &&
-                      (value <= jspb.BinaryConstants.FLOAT32_MAX));
+  goog.asserts.assert(
+      value === Infinity || value === -Infinity || isNaN(value) ||
+      ((value >= -jspb.BinaryConstants.FLOAT32_MAX) &&
+       (value <= jspb.BinaryConstants.FLOAT32_MAX)));
   jspb.utils.splitFloat32(value);
   this.writeUint32(jspb.utils.split64Low);
 };
@@ -395,8 +397,10 @@ jspb.BinaryEncoder.prototype.writeFloat = function(value) {
  * @param {number} value The value to write.
  */
 jspb.BinaryEncoder.prototype.writeDouble = function(value) {
-  goog.asserts.assert((value >= -jspb.BinaryConstants.FLOAT64_MAX) &&
-                      (value <= jspb.BinaryConstants.FLOAT64_MAX));
+  goog.asserts.assert(
+      value === Infinity || value === -Infinity || isNaN(value) ||
+      ((value >= -jspb.BinaryConstants.FLOAT64_MAX) &&
+       (value <= jspb.BinaryConstants.FLOAT64_MAX)));
   jspb.utils.splitFloat64(value);
   this.writeUint32(jspb.utils.split64Low);
   this.writeUint32(jspb.utils.split64High);

js/binary/utils.js

@@ -255,8 +255,25 @@ jspb.utils.splitFloat64 = function(value) {
     return;
   }
 
-  var exp = Math.floor(Math.log(value) / Math.LN2);
-  if (exp == 1024) exp = 1023;
+  // Compute the least significant exponent needed to represent the magnitude of
+  // the value by repeadly dividing/multiplying by 2 until the magnitude
+  // crosses 2. While tempting to use log math to find the exponent, at the
+  // bounadaries of precision, the result can be off by one.
+  var maxDoubleExponent = 1023;
+  var minDoubleExponent = -1022;
+  var x = value;
+  var exp = 0;
+  if (x >= 2) {
+    while (x >= 2 && exp < maxDoubleExponent) {
+      exp++;
+      x = x / 2;
+    }
+  } else {
+    while (x < 1 && exp > minDoubleExponent) {
+      x = x * 2;
+      exp--;
+    }
+  }
   var mant = value * Math.pow(2, -exp);
 
   var mantHigh = (mant * jspb.BinaryConstants.TWO_TO_20) & 0xFFFFF;

js/binary/utils_test.js

@@ -341,6 +341,8 @@ describe('binaryUtilsTest', function() {
     var f32_eps = jspb.BinaryConstants.FLOAT32_EPS;
     var f32_min = jspb.BinaryConstants.FLOAT32_MIN;
     var f32_max = jspb.BinaryConstants.FLOAT32_MAX;
+    var f32_max_safe_int = jspb.utils.joinFloat32(0x4b7fffff, 0);
+    var f32_pi = Math.fround(Math.PI);
 
     // NaN.
     jspb.utils.splitFloat32(NaN);
@@ -382,6 +384,13 @@ describe('binaryUtilsTest', function() {
     test(f32_max, 0x7F7FFFFF);
     test(-f32_max, 0xFF7FFFFF);
 
+    // Positive and negative max_safe_int.
+    test(f32_max_safe_int, 0x4B7FFFFF);
+    test(-f32_max_safe_int, 0xCB7FFFFF);
+
+    // Pi.
+    test(f32_pi, 0x40490fdb);
+
     // Various positive values.
     var cursor = f32_eps * 10;
     while (cursor != Infinity) {
@@ -420,10 +429,12 @@ describe('binaryUtilsTest', function() {
     function test(x, opt_highBits, opt_lowBits) {
       jspb.utils.splitFloat64(x);
       if (goog.isDef(opt_highBits)) {
-        if (opt_highBits != jspb.utils.split64High) throw 'fail!';
+        var split64High = jspb.utils.split64High;
+        expect(opt_highBits.toString(16)).toEqual(split64High.toString(16));
       }
       if (goog.isDef(opt_lowBits)) {
-        if (opt_lowBits != jspb.utils.split64Low) throw 'fail!';
+        var split64Low = jspb.utils.split64Low;
+        expect(opt_lowBits.toString(16)).toEqual(split64Low.toString(16));
       }
       expect(
           jspb.utils.joinFloat64(jspb.utils.split64Low, jspb.utils.split64High))
@@ -438,6 +449,9 @@ describe('binaryUtilsTest', function() {
     test(0, 0x00000000, 0x00000000);
     test(-0, 0x80000000, 0x00000000);
 
+    test(1, 0x3FF00000, 0x00000000);
+    test(2, 0x40000000, 0x00000000);
+
     // Positive and negative epsilon.
     test(f64_eps, 0x00000000, 0x00000001);
     test(-f64_eps, 0x80000000, 0x00000001);
@@ -450,6 +464,19 @@ describe('binaryUtilsTest', function() {
     test(f64_max, 0x7FEFFFFF, 0xFFFFFFFF);
     test(-f64_max, 0xFFEFFFFF, 0xFFFFFFFF);
 
+    test(Number.MAX_SAFE_INTEGER, 0x433FFFFF, 0xFFFFFFFF);
+    test(Number.MIN_SAFE_INTEGER, 0xC33FFFFF, 0xFFFFFFFF);
+
+    // Test various edge cases with mantissa of all 1, all 0, or just the
+    // highest or lowest significant bit.
+    test(4503599627370497, 0x43300000, 0x00000001);
+    test(6755399441055744, 0x43380000, 0x00000000);
+    test(1.348269851146737e+308, 0x7FE80000, 0x00000000);
+    test(1.9999999999999998, 0x3FFFFFFF, 0xFFFFFFFF);
+    test(2.225073858507201e-308, 0x000FFFFF, 0xFFFFFFFF);
+    test(Math.PI, 0x400921fb, 0x54442d18);
+    test(jspb.BinaryConstants.FLOAT32_MIN, 0x38100000, 0x00000000);
+
     // Various positive values.
     var cursor = f64_eps * 10;
     while (cursor != Infinity) {

js/binary/writer_test.js

@@ -40,6 +40,7 @@
 
 goog.require('goog.crypt');
 goog.require('goog.testing.asserts');
+goog.require('jspb.BinaryConstants');
 goog.require('jspb.BinaryReader');
 goog.require('jspb.BinaryWriter');
 goog.require('jspb.utils');
@@ -319,4 +320,50 @@ describe('binaryWriterTest', function() {
       return c.zigzag;
     }));
   });
+
+  it('writes float32 fields', function() {
+    var testCases = [
+      0, 1, -1, jspb.BinaryConstants.FLOAT32_MIN,
+      -jspb.BinaryConstants.FLOAT32_MIN, jspb.BinaryConstants.FLOAT32_MAX,
+      -jspb.BinaryConstants.FLOAT32_MAX, 3.1415927410125732, Infinity,
+      -Infinity, NaN
+    ];
+    var writer = new jspb.BinaryWriter();
+    testCases.forEach(function(f) {
+      writer.writeFloat(1, f);
+    });
+    var reader = jspb.BinaryReader.alloc(writer.getResultBuffer());
+    testCases.forEach(function(f) {
+      reader.nextField();
+      expect(reader.getFieldNumber()).toEqual(1);
+      if (isNaN(f)) {
+        expect(isNaN(reader.readFloat())).toEqual(true);
+      } else {
+        expect(reader.readFloat()).toEqual(f);
+      }
+    });
+  });
+
+  it('writes double fields', function() {
+    var testCases = [
+      0, 1, -1, Number.MAX_SAFE_INTEGER, Number.MIN_SAFE_INTEGER,
+      Number.MAX_VALUE, Number.MIN_VALUE, jspb.BinaryConstants.FLOAT32_MIN,
+      -jspb.BinaryConstants.FLOAT32_MIN, jspb.BinaryConstants.FLOAT32_MAX,
+      -jspb.BinaryConstants.FLOAT32_MAX, Math.PI, Infinity, -Infinity, NaN
+    ];
+    var writer = new jspb.BinaryWriter();
+    testCases.forEach(function(f) {
+      writer.writeDouble(1, f);
+    });
+    var reader = jspb.BinaryReader.alloc(writer.getResultBuffer());
+    testCases.forEach(function(f) {
+      reader.nextField();
+      expect(reader.getFieldNumber()).toEqual(1);
+      if (isNaN(f)) {
+        expect(isNaN(reader.readDouble())).toEqual(true);
+      } else {
+        expect(reader.readDouble()).toEqual(f);
+      }
+    });
+  });
 });

js/compatibility_tests/v3.1.0/binary/decoder_test.js

@@ -139,9 +139,20 @@ function doTestSignedValue(readValue,
     assertEquals(inputValues[i], readValue.call(decoder));
   }
 
-  // Encoding values outside the valid range should assert.
-  assertThrows(function() {writeValue.call(encoder, lowerLimit * 1.1);});
-  assertThrows(function() {writeValue.call(encoder, upperLimit * 1.1);});
+  // Encoding values outside the valid range should assert, except for doubles
+  // whose range runs all the way to infinity.
+  var pastLowerLimit = lowerLimit * 1.1;
+  var pastUpperLimit = upperLimit * 1.1;
+  if (pastLowerLimit !== -Infinity) {
+    expect(function() {
+      writeValue.call(encoder, lowerLimit * 1.1);
+    }).toThrow();
+  }
+  if (pastUpperLimit !== Infinity) {
+    expect(function() {
+      writeValue.call(encoder, upperLimit * 1.1);
+    }).toThrow();
+  }
 }
 
 describe('binaryDecoderTest', function() {

js/message_test.js

@@ -32,7 +32,6 @@
 
 goog.setTestOnly();
 
-goog.require('goog.json');
 goog.require('goog.testing.PropertyReplacer');
 goog.require('goog.testing.asserts');
 goog.require('goog.userAgent');

python/google/protobuf/internal/containers.py

@@ -230,24 +230,24 @@ class BaseContainer(object):
       kwargs['cmp'] = kwargs.pop('sort_function')
     self._values.sort(*args, **kwargs)
 
+
 collections_abc.MutableSequence.register(BaseContainer)
 
 
 class RepeatedScalarFieldContainer(BaseContainer):
-
   """Simple, type-checked, list-like container for holding repeated scalars."""
 
   # Disallows assignment to other attributes.
   __slots__ = ['_type_checker']
 
   def __init__(self, message_listener, type_checker):
-    """
-    Args:
-      message_listener: A MessageListener implementation.
-        The RepeatedScalarFieldContainer will call this object's
-        Modified() method when it is modified.
+    """Args:
+
+      message_listener: A MessageListener implementation. The
+      RepeatedScalarFieldContainer will call this object's Modified() method
+      when it is modified.
       type_checker: A type_checkers.ValueChecker instance to run on elements
-        inserted into this container.
+      inserted into this container.
     """
     super(RepeatedScalarFieldContainer, self).__init__(message_listener)
     self._type_checker = type_checker
@@ -285,6 +285,7 @@ class RepeatedScalarFieldContainer(BaseContainer):
 
   def MergeFrom(self, other):
     """Appends the contents of another repeated field of the same type to this
+
     one. We do not check the types of the individual fields.
     """
     self._values.extend(other._values)
@@ -400,6 +401,7 @@ class RepeatedCompositeFieldContainer(BaseContainer):
 
   def extend(self, elem_seq):
     """Extends by appending the given sequence of elements of the same type
+
     as this one, copying each individual message.
     """
     message_class = self._message_descriptor._concrete_class
@@ -414,6 +416,7 @@ class RepeatedCompositeFieldContainer(BaseContainer):
 
   def MergeFrom(self, other):
     """Appends the contents of another repeated field of the same type to this
+
     one, copying each individual message.
     """
     self.extend(other._values)

python/google/protobuf/internal/message_test.py

@@ -209,6 +209,19 @@ class MessageTest(unittest.TestCase):
     unpickled_message = pickle.loads(pickled_message)
     self.assertEqual(unpickled_message, golden_message)
 
+  def testPickleNestedMessage(self, message_module):
+    golden_message = message_module.TestPickleNestedMessage.NestedMessage(bb=1)
+    pickled_message = pickle.dumps(golden_message)
+    unpickled_message = pickle.loads(pickled_message)
+    self.assertEqual(unpickled_message, golden_message)
+
+  def testPickleNestedNestedMessage(self, message_module):
+    cls = message_module.TestPickleNestedMessage.NestedMessage
+    golden_message = cls.NestedNestedMessage(cc=1)
+    pickled_message = pickle.dumps(golden_message)
+    unpickled_message = pickle.loads(pickled_message)
+    self.assertEqual(unpickled_message, golden_message)
+
   def testPositiveInfinity(self, message_module):
     if message_module is unittest_pb2:
       golden_data = (b'\x5D\x00\x00\x80\x7F'

python/google/protobuf/internal/python_message.py

@@ -1357,12 +1357,6 @@ def _AddWhichOneofMethod(message_descriptor, cls):
   cls.WhichOneof = WhichOneof
 
 
-def _AddReduceMethod(cls):
-  def __reduce__(self):  # pylint: disable=invalid-name
-    return (type(self), (), self.__getstate__())
-  cls.__reduce__ = __reduce__
-
-
 def _Clear(self):
   # Clear fields.
   self._fields = {}
@@ -1425,7 +1419,6 @@ def _AddMessageMethods(message_descriptor, cls):
   _AddIsInitializedMethod(message_descriptor, cls)
   _AddMergeFromMethod(cls)
   _AddWhichOneofMethod(message_descriptor, cls)
-  _AddReduceMethod(cls)
   # Adds methods which do not depend on cls.
   cls.Clear = _Clear
   cls.UnknownFields = _UnknownFields

python/google/protobuf/internal/text_format_test.py

@@ -868,6 +868,51 @@ class OnlyWorksWithProto2RightNowTests(TextFormatBase):
                                                  print_unknown_fields=True,
                                                  as_one_line=True))
 
+  def testPrintUnknownFieldsEmbeddedMessageInBytes(self):
+    inner_msg = unittest_pb2.TestAllTypes()
+    inner_msg.optional_int32 = 101
+    inner_msg.optional_double = 102.0
+    inner_msg.optional_string = u'hello'
+    inner_msg.optional_bytes = b'103'
+    inner_msg.optionalgroup.a = 104
+    inner_msg.optional_nested_message.bb = 105
+    inner_data = inner_msg.SerializeToString()
+    outer_message = unittest_pb2.TestAllTypes()
+    outer_message.optional_int32 = 101
+    outer_message.optional_bytes = inner_data
+    all_data = outer_message.SerializeToString()
+    empty_message = unittest_pb2.TestEmptyMessage()
+    empty_message.ParseFromString(all_data)
+
+    self.assertEqual('  1: 101\n'
+                     '  15 {\n'
+                     '    1: 101\n'
+                     '    12: 4636878028842991616\n'
+                     '    14: "hello"\n'
+                     '    15: "103"\n'
+                     '    16 {\n'
+                     '      17: 104\n'
+                     '    }\n'
+                     '    18 {\n'
+                     '      1: 105\n'
+                     '    }\n'
+                     '  }\n',
+                     text_format.MessageToString(empty_message,
+                                                 indent=2,
+                                                 print_unknown_fields=True))
+    self.assertEqual('1: 101 '
+                     '15 { '
+                     '1: 101 '
+                     '12: 4636878028842991616 '
+                     '14: "hello" '
+                     '15: "103" '
+                     '16 { 17: 104 } '
+                     '18 { 1: 105 } '
+                     '}',
+                     text_format.MessageToString(empty_message,
+                                                 print_unknown_fields=True,
+                                                 as_one_line=True))
+
   def testPrintInIndexOrder(self):
     message = unittest_pb2.TestFieldOrderings()
     # Fields are listed in index order instead of field number.

python/google/protobuf/message.py

@@ -317,3 +317,21 @@ class Message(object):
     if not isinstance(serialized, bytes):
       serialized = serialized.encode('latin1')
     self.ParseFromString(serialized)
+
+  def __reduce__(self):
+    message_descriptor = self.DESCRIPTOR
+    if message_descriptor.containing_type is None:
+      return type(self), (), self.__getstate__()
+    # the message type must be nested.
+    # Python does not pickle nested classes; use the symbol_database on the
+    # receiving end.
+    container = message_descriptor
+    return (_InternalConstructMessage, (container.full_name,),
+            self.__getstate__())
+
+
+def _InternalConstructMessage(full_name):
+  """Constructs a nested message."""
+  from google.protobuf import symbol_database  # pylint:disable=g-import-not-at-top
+
+  return symbol_database.Default().GetSymbol(full_name)()

python/google/protobuf/pyext/message.cc

@@ -791,7 +791,8 @@ bool CheckAndSetString(
   } else if (index < 0) {
     reflection->SetString(message, descriptor, std::move(value_string));
   } else {
-    reflection->SetRepeatedString(message, descriptor, index, std::move(value_string));
+    reflection->SetRepeatedString(message, descriptor, index,
+                                  std::move(value_string));
   }
   return true;
 }
@@ -1777,7 +1778,7 @@ static PyObject* SerializePartialToString(
 
 // Formats proto fields for ascii dumps using python formatting functions where
 // appropriate.
-class PythonFieldValuePrinter : public TextFormat::FieldValuePrinter {
+class PythonFieldValuePrinter : public TextFormat::FastFieldValuePrinter {
  public:
   // Python has some differences from C++ when printing floating point numbers.
   //
@@ -1789,8 +1790,12 @@ class PythonFieldValuePrinter : public TextFormat::FieldValuePrinter {
   //
   // We override floating point printing with the C-API function for printing
   // Python floats to ensure consistency.
-  string PrintFloat(float value) const { return PrintDouble(value); }
-  string PrintDouble(double value) const {
+  void PrintFloat(float val,
+                  TextFormat::BaseTextGenerator* generator) const override {
+    PrintDouble(val, generator);
+  }
+  void PrintDouble(double val,
+                   TextFormat::BaseTextGenerator* generator) const override {
     // This implementation is not highly optimized (it allocates two temporary
     // Python objects) but it is simple and portable.  If this is shown to be a
     // performance bottleneck, we can optimize it, but the results will likely
@@ -1799,17 +1804,17 @@ class PythonFieldValuePrinter : public TextFormat::FieldValuePrinter {
     //
     // (Though a valid question is: do we really want to make out output
     // dependent on the Python version?)
-    ScopedPyObjectPtr py_value(PyFloat_FromDouble(value));
+    ScopedPyObjectPtr py_value(PyFloat_FromDouble(val));
     if (!py_value.get()) {
-      return string();
+      return;
     }
 
     ScopedPyObjectPtr py_str(PyObject_Str(py_value.get()));
     if (!py_str.get()) {
-      return string();
+      return;
     }
 
-    return string(PyString_AsString(py_str.get()));
+    generator->PrintString(PyString_AsString(py_str.get()));
   }
 };
 
@@ -1980,7 +1985,7 @@ static PyObject* ParseFromString(CMessage* self, PyObject* arg) {
 }
 
 static PyObject* ByteSize(CMessage* self, PyObject* args) {
-  return PyLong_FromLong(self->message->ByteSize());
+  return PyLong_FromLong(self->message->ByteSizeLong());
 }
 
 PyObject* RegisterExtension(PyObject* cls, PyObject* extension_handle) {
@@ -2433,52 +2438,6 @@ PyObject* ToUnicode(CMessage* self) {
   return decoded;
 }
 
-PyObject* Reduce(CMessage* self) {
-  ScopedPyObjectPtr constructor(reinterpret_cast<PyObject*>(Py_TYPE(self)));
-  constructor.inc();
-  ScopedPyObjectPtr args(PyTuple_New(0));
-  if (args == NULL) {
-    return NULL;
-  }
-  ScopedPyObjectPtr state(PyDict_New());
-  if (state == NULL) {
-    return  NULL;
-  }
-  string contents;
-  self->message->SerializePartialToString(&contents);
-  ScopedPyObjectPtr serialized(
-      PyBytes_FromStringAndSize(contents.c_str(), contents.size()));
-  if (serialized == NULL) {
-    return NULL;
-  }
-  if (PyDict_SetItemString(state.get(), "serialized", serialized.get()) < 0) {
-    return NULL;
-  }
-  return Py_BuildValue("OOO", constructor.get(), args.get(), state.get());
-}
-
-PyObject* SetState(CMessage* self, PyObject* state) {
-  if (!PyDict_Check(state)) {
-    PyErr_SetString(PyExc_TypeError, "state not a dict");
-    return NULL;
-  }
-  PyObject* serialized = PyDict_GetItemString(state, "serialized");
-  if (serialized == NULL) {
-    return NULL;
-  }
-#if PY_MAJOR_VERSION >= 3
-  // On Python 3, using encoding='latin1' is required for unpickling
-  // protos pickled by Python 2.
-  if (!PyBytes_Check(serialized)) {
-    serialized = PyUnicode_AsEncodedString(serialized, "latin1", NULL);
-  }
-#endif
-  if (ScopedPyObjectPtr(ParseFromString(self, serialized)) == NULL) {
-    return NULL;
-  }
-  Py_RETURN_NONE;
-}
-
 // CMessage static methods:
 PyObject* _CheckCalledFromGeneratedFile(PyObject* unused,
                                         PyObject* unused_arg) {
@@ -2539,10 +2498,6 @@ static PyGetSetDef Getters[] = {
 static PyMethodDef Methods[] = {
   { "__deepcopy__", (PyCFunction)DeepCopy, METH_VARARGS,
     "Makes a deep copy of the class." },
-  { "__reduce__", (PyCFunction)Reduce, METH_NOARGS,
-    "Outputs picklable representation of the message." },
-  { "__setstate__", (PyCFunction)SetState, METH_O,
-    "Inputs picklable representation of the message." },
   { "__unicode__", (PyCFunction)ToUnicode, METH_NOARGS,
     "Outputs a unicode representation of the message." },
   { "ByteSize", (PyCFunction)ByteSize, METH_NOARGS,

python/google/protobuf/text_format.py

@@ -578,18 +578,45 @@ class _Printer(object):
       else:
         out.write(str(value))
     elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING:
-      out.write('\"')
-      if isinstance(value, six.text_type) and (six.PY2 or not self.as_utf8):
-        out_value = value.encode('utf-8')
-      else:
-        out_value = value
-      if field.type == descriptor.FieldDescriptor.TYPE_BYTES:
-        # We always need to escape all binary data in TYPE_BYTES fields.
-        out_as_utf8 = False
+      embedded_unknown_message = None
+      if self.print_unknown_fields:
+        try:
+          # If this field is parseable as a Message, it is probably
+          # an embedded message.
+          # pylint: disable=protected-access
+          (embedded_unknown_message, pos) = decoder._DecodeUnknownFieldSet(
+              memoryview(value), 0, len(value))
+          if pos != len(value):
+            embedded_unknown_message = None
+        except Exception:    # pylint: disable=broad-except
+          pass
+      if embedded_unknown_message:
+        if self.as_one_line:
+          out.write(' { ')
+        else:
+          out.write(' {\n')
+          self.indent += 2
+
+        self._PrintUnknownFields(embedded_unknown_message)
+
+        if self.as_one_line:
+          out.write('} ')
+        else:
+          self.indent -= 2
+          out.write(' ' * self.indent + '}')
       else:
-        out_as_utf8 = self.as_utf8
-      out.write(text_encoding.CEscape(out_value, out_as_utf8))
-      out.write('\"')
+        out.write('\"')
+        if isinstance(value, six.text_type) and (six.PY2 or not self.as_utf8):
+          out_value = value.encode('utf-8')
+        else:
+          out_value = value
+        if field.type == descriptor.FieldDescriptor.TYPE_BYTES:
+          # We always need to escape all binary data in TYPE_BYTES fields.
+          out_as_utf8 = False
+        else:
+          out_as_utf8 = self.as_utf8
+        out.write(text_encoding.CEscape(out_value, out_as_utf8))
+        out.write('\"')
     elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL:
       if value:
         out.write('true')

src/google/protobuf/api.pb.cc

@@ -383,7 +383,7 @@ failure:
   // repeated .google.protobuf.Method methods = 2;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_methods_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(2, this->_internal_methods(i), target, stream);
   }
@@ -391,7 +391,7 @@ failure:
   // repeated .google.protobuf.Option options = 3;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_options_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(3, this->_internal_options(i), target, stream);
   }
@@ -408,7 +408,7 @@ failure:
 
   // .google.protobuf.SourceContext source_context = 5;
   if (this->has_source_context()) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(
         5, _Internal::source_context(this), target, stream);
@@ -417,14 +417,14 @@ failure:
   // repeated .google.protobuf.Mixin mixins = 6;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_mixins_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(6, this->_internal_mixins(i), target, stream);
   }
 
   // .google.protobuf.Syntax syntax = 7;
   if (this->syntax() != 0) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteEnumToArray(
       7, this->_internal_syntax(), target);
   }
@@ -775,7 +775,7 @@ failure:
 
   // bool request_streaming = 3;
   if (this->request_streaming() != 0) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(3, this->_internal_request_streaming(), target);
   }
 
@@ -791,21 +791,21 @@ failure:
 
   // bool response_streaming = 5;
   if (this->response_streaming() != 0) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(5, this->_internal_response_streaming(), target);
   }
 
   // repeated .google.protobuf.Option options = 6;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_options_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(6, this->_internal_options(i), target, stream);
   }
 
   // .google.protobuf.Syntax syntax = 7;
   if (this->syntax() != 0) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteEnumToArray(
       7, this->_internal_syntax(), target);
   }

src/google/protobuf/arena.h

@@ -33,6 +33,7 @@
 #ifndef GOOGLE_PROTOBUF_ARENA_H__
 #define GOOGLE_PROTOBUF_ARENA_H__
 
+
 #include <limits>
 #include <type_traits>
 #include <utility>
@@ -359,14 +360,6 @@ class PROTOBUF_EXPORT alignas(8) Arena final {
   // may not include space used by other threads executing concurrently with
   // the call to this method.
   uint64 SpaceUsed() const { return impl_.SpaceUsed(); }
-  // DEPRECATED. Please use SpaceAllocated() and SpaceUsed().
-  //
-  // Combines SpaceAllocated and SpaceUsed. Returns a pair of
-  // <space_allocated, space_used>.
-  PROTOBUF_DEPRECATED_MSG("Please use SpaceAllocated() and SpaceUsed()")
-  std::pair<uint64, uint64> SpaceAllocatedAndUsed() const {
-    return std::make_pair(SpaceAllocated(), SpaceUsed());
-  }
 
   // Frees all storage allocated by this arena after calling destructors
   // registered with OwnDestructor() and freeing objects registered with Own().

src/google/protobuf/compiler/command_line_interface.cc

@@ -411,11 +411,11 @@ class CommandLineInterface::MemoryOutputStream
   virtual ~MemoryOutputStream();
 
   // implements ZeroCopyOutputStream ---------------------------------
-  virtual bool Next(void** data, int* size) { return inner_->Next(data, size); }
-  virtual void BackUp(int count) { inner_->BackUp(count); }
-  virtual int64 ByteCount() const {
-    return inner_->ByteCount();
+  bool Next(void** data, int* size) override {
+    return inner_->Next(data, size);
   }
+  void BackUp(int count) override { inner_->BackUp(count); }
+  int64_t ByteCount() const override { return inner_->ByteCount(); }
 
  private:
   // Checks to see if "filename_.meta" exists in directory_; if so, fixes the

src/google/protobuf/compiler/cpp/cpp_enum_field.cc

@@ -142,7 +142,7 @@ void EnumFieldGenerator::GenerateSerializeWithCachedSizesToArray(
     io::Printer* printer) const {
   Formatter format(printer, variables_);
   format(
-      "stream->EnsureSpace(&target);\n"
+      "target = stream->EnsureSpace(target);\n"
       "target = ::$proto_ns$::internal::WireFormatLite::WriteEnumToArray(\n"
       "  $number$, this->_internal_$name$(), target);\n");
 }
@@ -445,7 +445,7 @@ void RepeatedEnumFieldGenerator::GenerateSerializeWithCachedSizesToArray(
   } else {
     format(
         "for (int i = 0, n = this->_internal_$name$_size(); i < n; i++) {\n"
-        "  stream->EnsureSpace(&target);\n"
+        "  target = stream->EnsureSpace(target);\n"
         "  target = ::$proto_ns$::internal::WireFormatLite::WriteEnumToArray(\n"
         "      $number$, this->_internal_$name$(i), target);\n"
         "}\n");

src/google/protobuf/compiler/cpp/cpp_generator.h

@@ -62,7 +62,7 @@ class PROTOC_EXPORT CppGenerator : public CodeGenerator {
 
     // Use the open-source runtime with google3 #include paths.  We make these
     // absolute to avoid ambiguity, so the runtime will be #included like:
-    //   #include "third_party/protobuf/<...>/google/protobuf/message.h"
+    //   #include "third_party/protobuf/.../google/protobuf/message.h"
     kOpensourceGoogle3
   };
 

src/google/protobuf/compiler/cpp/cpp_message.cc

@@ -3461,7 +3461,7 @@ void MessageGenerator::GenerateSerializeOneField(io::Printer* printer,
 
       format("if (cached_has_bits & 0x$1$u) {\n", mask);
     } else {
-      format("if (has_$1$()) {\n", FieldName(field));
+      format("if (_internal_has_$1$()) {\n", FieldName(field));
     }
 
     format.Indent();

src/google/protobuf/compiler/cpp/cpp_message_field.cc

@@ -436,7 +436,7 @@ void MessageFieldGenerator::GenerateSerializeWithCachedSizesToArray(
     io::Printer* printer) const {
   Formatter format(printer, variables_);
   format(
-      "stream->EnsureSpace(&target);\n"
+      "target = stream->EnsureSpace(target);\n"
       "target = ::$proto_ns$::internal::WireFormatLite::\n"
       "  InternalWrite$declared_type$ToArray(\n"
       "    $number$, _Internal::$name$(this), target, stream);\n");
@@ -749,7 +749,7 @@ void RepeatedMessageFieldGenerator::GenerateSerializeWithCachedSizesToArray(
     format(
         "for (auto it = this->$name$_.pointer_begin(),\n"
         "          end = this->$name$_.pointer_end(); it < end; ++it) {\n"
-        "  stream->EnsureSpace(&target);\n"
+        "  target = stream->EnsureSpace(target);\n"
         "  target = ::$proto_ns$::internal::WireFormatLite::\n"
         "    InternalWrite$declared_type$ToArray($number$, **it, target, "
         "stream);\n"
@@ -760,7 +760,7 @@ void RepeatedMessageFieldGenerator::GenerateSerializeWithCachedSizesToArray(
         "    n = static_cast<unsigned int>(this->_internal_$name$_size()); i < "
         "n; i++) "
         "{\n"
-        "  stream->EnsureSpace(&target);\n"
+        "  target = stream->EnsureSpace(target);\n"
         "  target = ::$proto_ns$::internal::WireFormatLite::\n"
         "    InternalWrite$declared_type$ToArray($number$, "
         "this->_internal_$name$(i), target, "

src/google/protobuf/compiler/cpp/cpp_primitive_field.cc

@@ -198,7 +198,7 @@ void PrimitiveFieldGenerator::GenerateSerializeWithCachedSizesToArray(
     io::Printer* printer) const {
   Formatter format(printer, variables_);
   format(
-      "stream->EnsureSpace(&target);\n"
+      "target = stream->EnsureSpace(target);\n"
       "target = "
       "::$proto_ns$::internal::WireFormatLite::Write$declared_type$ToArray("
       "$number$, this->_internal_$name$(), target);\n");
@@ -423,7 +423,7 @@ void RepeatedPrimitiveFieldGenerator::GenerateSerializeWithCachedSizesToArray(
   } else {
     format(
         "for (int i = 0, n = this->_internal_$name$_size(); i < n; i++) {\n"
-        "  stream->EnsureSpace(&target);\n"
+        "  target = stream->EnsureSpace(target);\n"
         "  target = ::$proto_ns$::internal::WireFormatLite::"
         "Write$declared_type$ToArray($number$, this->_internal_$name$(i), "
         "target);\n"

src/google/protobuf/compiler/parser_unittest.cc

@@ -2039,7 +2039,7 @@ TEST_F(ParserValidationErrorTest, MethodOutputTypeError) {
 }
 
 
-TEST_F(ParserValidationErrorTest, ResovledUndefinedError) {
+TEST_F(ParserValidationErrorTest, ResolvedUndefinedError) {
   // Create another file which defines symbol ".base.bar".
   FileDescriptorProto other_file;
   other_file.set_name("base.proto");
@@ -2084,14 +2084,14 @@ TEST_F(ParserValidationErrorTest, ResovledUndefinedOptionError) {
   FieldDescriptorProto* field(message->add_field());
   field->set_name("foo");
   field->set_number(1);
-  field->set_label(FieldDescriptorProto_Label_LABEL_OPTIONAL);
-  field->set_type(FieldDescriptorProto_Type_TYPE_INT32);
+  field->set_label(FieldDescriptorProto::LABEL_OPTIONAL);
+  field->set_type(FieldDescriptorProto::TYPE_INT32);
 
   FieldDescriptorProto* extension(other_file.add_extension());
   extension->set_name("bar");
   extension->set_number(7672757);
-  extension->set_label(FieldDescriptorProto_Label_LABEL_OPTIONAL);
-  extension->set_type(FieldDescriptorProto_Type_TYPE_MESSAGE);
+  extension->set_label(FieldDescriptorProto::LABEL_OPTIONAL);
+  extension->set_type(FieldDescriptorProto::TYPE_MESSAGE);
   extension->set_type_name("Bar");
   extension->set_extendee("google.protobuf.FileOptions");
 

src/google/protobuf/compiler/plugin.pb.cc

@@ -358,19 +358,19 @@ failure:
   cached_has_bits = _has_bits_[0];
   // optional int32 major = 1;
   if (cached_has_bits & 0x00000002u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(1, this->_internal_major(), target);
   }
 
   // optional int32 minor = 2;
   if (cached_has_bits & 0x00000004u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(2, this->_internal_minor(), target);
   }
 
   // optional int32 patch = 3;
   if (cached_has_bits & 0x00000008u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(3, this->_internal_patch(), target);
   }
 
@@ -707,7 +707,7 @@ failure:
 
   // optional .google.protobuf.compiler.Version compiler_version = 3;
   if (cached_has_bits & 0x00000002u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(
         3, _Internal::compiler_version(this), target, stream);
@@ -716,7 +716,7 @@ failure:
   // repeated .google.protobuf.FileDescriptorProto proto_file = 15;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_proto_file_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(15, this->_internal_proto_file(i), target, stream);
   }
@@ -1287,7 +1287,7 @@ failure:
   // repeated .google.protobuf.compiler.CodeGeneratorResponse.File file = 15;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_file_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(15, this->_internal_file(i), target, stream);
   }

src/google/protobuf/descriptor.cc

@@ -802,9 +802,7 @@ DescriptorPool::Tables::Tables()
       symbols_by_name_(3),
       files_by_name_(3) {}
 
-DescriptorPool::Tables::~Tables() {
-  GOOGLE_DCHECK(checkpoints_.empty());
-}
+DescriptorPool::Tables::~Tables() { GOOGLE_DCHECK(checkpoints_.empty()); }
 
 FileDescriptorTables::FileDescriptorTables()
     // Initialize all the hash tables to start out with a small # of buckets.
@@ -1331,7 +1329,6 @@ const DescriptorPool* DescriptorPool::generated_pool() {
 }
 
 
-
 void DescriptorPool::InternalAddGeneratedFile(
     const void* encoded_file_descriptor, int size) {
   // So, this function is called in the process of initializing the
@@ -5013,11 +5010,10 @@ void DescriptorBuilder::BuildOneof(const OneofDescriptorProto& proto,
   // We need to fill these in later.
   result->field_count_ = 0;
   result->fields_ = nullptr;
+  result->options_ = nullptr;
 
   // Copy options.
-  if (!proto.has_options()) {
-    result->options_ = nullptr;  // Will set to default_instance later.
-  } else {
+  if (proto.has_options()) {
     AllocateOptions(proto.options(), result,
                     OneofDescriptorProto::kOptionsFieldNumber);
   }
@@ -5466,12 +5462,19 @@ void DescriptorBuilder::CrossLinkField(FieldDescriptor* field,
             field->number());
 
     if (extension_range == nullptr) {
-      AddError(field->full_name(), proto,
-               DescriptorPool::ErrorCollector::NUMBER,
-               strings::Substitute("\"$0\" does not declare $1 as an "
-                                   "extension number.",
-                                   field->containing_type()->full_name(),
-                                   field->number()));
+      // Set of valid extension numbers for MessageSet is different (< 2^32)
+      // from other extendees (< 2^29). If unknown deps are allowed, we may not
+      // have that information, and wrongly deem the extension as invalid.
+      auto skip_check = get_allow_unknown(pool_) &&
+                        proto.extendee() == "google.protobuf.bridge.MessageSet";
+      if (!skip_check) {
+        AddError(field->full_name(), proto,
+                 DescriptorPool::ErrorCollector::NUMBER,
+                 strings::Substitute("\"$0\" does not declare $1 as an "
+                                     "extension number.",
+                                     field->containing_type()->full_name(),
+                                     field->number()));
+      }
     }
   }
 

src/google/protobuf/descriptor.pb.cc

@@ -1495,7 +1495,7 @@ failure:
   // repeated .google.protobuf.FileDescriptorProto file = 1;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_file_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(1, this->_internal_file(i), target, stream);
   }
@@ -1982,7 +1982,7 @@ failure:
   // repeated .google.protobuf.DescriptorProto message_type = 4;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_message_type_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(4, this->_internal_message_type(i), target, stream);
   }
@@ -1990,7 +1990,7 @@ failure:
   // repeated .google.protobuf.EnumDescriptorProto enum_type = 5;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_enum_type_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(5, this->_internal_enum_type(i), target, stream);
   }
@@ -1998,7 +1998,7 @@ failure:
   // repeated .google.protobuf.ServiceDescriptorProto service = 6;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_service_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(6, this->_internal_service(i), target, stream);
   }
@@ -2006,14 +2006,14 @@ failure:
   // repeated .google.protobuf.FieldDescriptorProto extension = 7;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_extension_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(7, this->_internal_extension(i), target, stream);
   }
 
   // optional .google.protobuf.FileOptions options = 8;
   if (cached_has_bits & 0x00000008u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(
         8, _Internal::options(this), target, stream);
@@ -2021,7 +2021,7 @@ failure:
 
   // optional .google.protobuf.SourceCodeInfo source_code_info = 9;
   if (cached_has_bits & 0x00000010u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(
         9, _Internal::source_code_info(this), target, stream);
@@ -2029,13 +2029,13 @@ failure:
 
   // repeated int32 public_dependency = 10;
   for (int i = 0, n = this->_internal_public_dependency_size(); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(10, this->_internal_public_dependency(i), target);
   }
 
   // repeated int32 weak_dependency = 11;
   for (int i = 0, n = this->_internal_weak_dependency_size(); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(11, this->_internal_weak_dependency(i), target);
   }
 
@@ -2446,19 +2446,19 @@ failure:
   cached_has_bits = _has_bits_[0];
   // optional int32 start = 1;
   if (cached_has_bits & 0x00000002u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(1, this->_internal_start(), target);
   }
 
   // optional int32 end = 2;
   if (cached_has_bits & 0x00000004u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(2, this->_internal_end(), target);
   }
 
   // optional .google.protobuf.ExtensionRangeOptions options = 3;
   if (cached_has_bits & 0x00000001u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(
         3, _Internal::options(this), target, stream);
@@ -2724,13 +2724,13 @@ failure:
   cached_has_bits = _has_bits_[0];
   // optional int32 start = 1;
   if (cached_has_bits & 0x00000001u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(1, this->_internal_start(), target);
   }
 
   // optional int32 end = 2;
   if (cached_has_bits & 0x00000002u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(2, this->_internal_end(), target);
   }
 
@@ -3142,7 +3142,7 @@ failure:
   // repeated .google.protobuf.FieldDescriptorProto field = 2;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_field_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(2, this->_internal_field(i), target, stream);
   }
@@ -3150,7 +3150,7 @@ failure:
   // repeated .google.protobuf.DescriptorProto nested_type = 3;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_nested_type_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(3, this->_internal_nested_type(i), target, stream);
   }
@@ -3158,7 +3158,7 @@ failure:
   // repeated .google.protobuf.EnumDescriptorProto enum_type = 4;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_enum_type_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(4, this->_internal_enum_type(i), target, stream);
   }
@@ -3166,7 +3166,7 @@ failure:
   // repeated .google.protobuf.DescriptorProto.ExtensionRange extension_range = 5;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_extension_range_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(5, this->_internal_extension_range(i), target, stream);
   }
@@ -3174,14 +3174,14 @@ failure:
   // repeated .google.protobuf.FieldDescriptorProto extension = 6;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_extension_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(6, this->_internal_extension(i), target, stream);
   }
 
   // optional .google.protobuf.MessageOptions options = 7;
   if (cached_has_bits & 0x00000002u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(
         7, _Internal::options(this), target, stream);
@@ -3190,7 +3190,7 @@ failure:
   // repeated .google.protobuf.OneofDescriptorProto oneof_decl = 8;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_oneof_decl_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(8, this->_internal_oneof_decl(i), target, stream);
   }
@@ -3198,7 +3198,7 @@ failure:
   // repeated .google.protobuf.DescriptorProto.ReservedRange reserved_range = 9;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_reserved_range_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(9, this->_internal_reserved_range(i), target, stream);
   }
@@ -3530,7 +3530,7 @@ failure:
   // repeated .google.protobuf.UninterpretedOption uninterpreted_option = 999;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_uninterpreted_option_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(999, this->_internal_uninterpreted_option(i), target, stream);
   }
@@ -3973,20 +3973,20 @@ failure:
 
   // optional int32 number = 3;
   if (cached_has_bits & 0x00000040u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(3, this->_internal_number(), target);
   }
 
   // optional .google.protobuf.FieldDescriptorProto.Label label = 4;
   if (cached_has_bits & 0x00000100u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteEnumToArray(
       4, this->_internal_label(), target);
   }
 
   // optional .google.protobuf.FieldDescriptorProto.Type type = 5;
   if (cached_has_bits & 0x00000200u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteEnumToArray(
       5, this->_internal_type(), target);
   }
@@ -4013,7 +4013,7 @@ failure:
 
   // optional .google.protobuf.FieldOptions options = 8;
   if (cached_has_bits & 0x00000020u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(
         8, _Internal::options(this), target, stream);
@@ -4021,7 +4021,7 @@ failure:
 
   // optional int32 oneof_index = 9;
   if (cached_has_bits & 0x00000080u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(9, this->_internal_oneof_index(), target);
   }
 
@@ -4421,7 +4421,7 @@ failure:
 
   // optional .google.protobuf.OneofOptions options = 2;
   if (cached_has_bits & 0x00000002u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(
         2, _Internal::options(this), target, stream);
@@ -4676,13 +4676,13 @@ failure:
   cached_has_bits = _has_bits_[0];
   // optional int32 start = 1;
   if (cached_has_bits & 0x00000001u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(1, this->_internal_start(), target);
   }
 
   // optional int32 end = 2;
   if (cached_has_bits & 0x00000002u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(2, this->_internal_end(), target);
   }
 
@@ -5019,14 +5019,14 @@ failure:
   // repeated .google.protobuf.EnumValueDescriptorProto value = 2;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_value_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(2, this->_internal_value(i), target, stream);
   }
 
   // optional .google.protobuf.EnumOptions options = 3;
   if (cached_has_bits & 0x00000002u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(
         3, _Internal::options(this), target, stream);
@@ -5035,7 +5035,7 @@ failure:
   // repeated .google.protobuf.EnumDescriptorProto.EnumReservedRange reserved_range = 4;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_reserved_range_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(4, this->_internal_reserved_range(i), target, stream);
   }
@@ -5384,13 +5384,13 @@ failure:
 
   // optional int32 number = 2;
   if (cached_has_bits & 0x00000004u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(2, this->_internal_number(), target);
   }
 
   // optional .google.protobuf.EnumValueOptions options = 3;
   if (cached_has_bits & 0x00000002u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(
         3, _Internal::options(this), target, stream);
@@ -5714,14 +5714,14 @@ failure:
   // repeated .google.protobuf.MethodDescriptorProto method = 2;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_method_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(2, this->_internal_method(i), target, stream);
   }
 
   // optional .google.protobuf.ServiceOptions options = 3;
   if (cached_has_bits & 0x00000002u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(
         3, _Internal::options(this), target, stream);
@@ -6117,7 +6117,7 @@ failure:
 
   // optional .google.protobuf.MethodOptions options = 4;
   if (cached_has_bits & 0x00000008u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(
         4, _Internal::options(this), target, stream);
@@ -6125,13 +6125,13 @@ failure:
 
   // optional bool client_streaming = 5 [default = false];
   if (cached_has_bits & 0x00000010u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(5, this->_internal_client_streaming(), target);
   }
 
   // optional bool server_streaming = 6 [default = false];
   if (cached_has_bits & 0x00000020u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(6, this->_internal_server_streaming(), target);
   }
 
@@ -6775,14 +6775,14 @@ failure:
 
   // optional .google.protobuf.FileOptions.OptimizeMode optimize_for = 9 [default = SPEED];
   if (cached_has_bits & 0x00080000u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteEnumToArray(
       9, this->_internal_optimize_for(), target);
   }
 
   // optional bool java_multiple_files = 10 [default = false];
   if (cached_has_bits & 0x00000400u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(10, this->_internal_java_multiple_files(), target);
   }
 
@@ -6798,43 +6798,43 @@ failure:
 
   // optional bool cc_generic_services = 16 [default = false];
   if (cached_has_bits & 0x00002000u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(16, this->_internal_cc_generic_services(), target);
   }
 
   // optional bool java_generic_services = 17 [default = false];
   if (cached_has_bits & 0x00004000u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(17, this->_internal_java_generic_services(), target);
   }
 
   // optional bool py_generic_services = 18 [default = false];
   if (cached_has_bits & 0x00008000u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(18, this->_internal_py_generic_services(), target);
   }
 
   // optional bool java_generate_equals_and_hash = 20 [deprecated = true];
   if (cached_has_bits & 0x00000800u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(20, this->_internal_java_generate_equals_and_hash(), target);
   }
 
   // optional bool deprecated = 23 [default = false];
   if (cached_has_bits & 0x00020000u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(23, this->_internal_deprecated(), target);
   }
 
   // optional bool java_string_check_utf8 = 27 [default = false];
   if (cached_has_bits & 0x00001000u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(27, this->_internal_java_string_check_utf8(), target);
   }
 
   // optional bool cc_enable_arenas = 31 [default = false];
   if (cached_has_bits & 0x00040000u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(31, this->_internal_cc_enable_arenas(), target);
   }
 
@@ -6890,7 +6890,7 @@ failure:
 
   // optional bool php_generic_services = 42 [default = false];
   if (cached_has_bits & 0x00010000u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(42, this->_internal_php_generic_services(), target);
   }
 
@@ -6917,7 +6917,7 @@ failure:
   // repeated .google.protobuf.UninterpretedOption uninterpreted_option = 999;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_uninterpreted_option_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(999, this->_internal_uninterpreted_option(i), target, stream);
   }
@@ -7432,32 +7432,32 @@ failure:
   cached_has_bits = _has_bits_[0];
   // optional bool message_set_wire_format = 1 [default = false];
   if (cached_has_bits & 0x00000001u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(1, this->_internal_message_set_wire_format(), target);
   }
 
   // optional bool no_standard_descriptor_accessor = 2 [default = false];
   if (cached_has_bits & 0x00000002u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(2, this->_internal_no_standard_descriptor_accessor(), target);
   }
 
   // optional bool deprecated = 3 [default = false];
   if (cached_has_bits & 0x00000004u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(3, this->_internal_deprecated(), target);
   }
 
   // optional bool map_entry = 7;
   if (cached_has_bits & 0x00000008u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(7, this->_internal_map_entry(), target);
   }
 
   // repeated .google.protobuf.UninterpretedOption uninterpreted_option = 999;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_uninterpreted_option_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(999, this->_internal_uninterpreted_option(i), target, stream);
   }
@@ -7821,46 +7821,46 @@ failure:
   cached_has_bits = _has_bits_[0];
   // optional .google.protobuf.FieldOptions.CType ctype = 1 [default = STRING];
   if (cached_has_bits & 0x00000001u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteEnumToArray(
       1, this->_internal_ctype(), target);
   }
 
   // optional bool packed = 2;
   if (cached_has_bits & 0x00000002u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(2, this->_internal_packed(), target);
   }
 
   // optional bool deprecated = 3 [default = false];
   if (cached_has_bits & 0x00000008u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(3, this->_internal_deprecated(), target);
   }
 
   // optional bool lazy = 5 [default = false];
   if (cached_has_bits & 0x00000004u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(5, this->_internal_lazy(), target);
   }
 
   // optional .google.protobuf.FieldOptions.JSType jstype = 6 [default = JS_NORMAL];
   if (cached_has_bits & 0x00000020u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteEnumToArray(
       6, this->_internal_jstype(), target);
   }
 
   // optional bool weak = 10 [default = false];
   if (cached_has_bits & 0x00000010u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(10, this->_internal_weak(), target);
   }
 
   // repeated .google.protobuf.UninterpretedOption uninterpreted_option = 999;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_uninterpreted_option_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(999, this->_internal_uninterpreted_option(i), target, stream);
   }
@@ -8156,7 +8156,7 @@ failure:
   // repeated .google.protobuf.UninterpretedOption uninterpreted_option = 999;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_uninterpreted_option_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(999, this->_internal_uninterpreted_option(i), target, stream);
   }
@@ -8422,20 +8422,20 @@ failure:
   cached_has_bits = _has_bits_[0];
   // optional bool allow_alias = 2;
   if (cached_has_bits & 0x00000001u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(2, this->_internal_allow_alias(), target);
   }
 
   // optional bool deprecated = 3 [default = false];
   if (cached_has_bits & 0x00000002u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(3, this->_internal_deprecated(), target);
   }
 
   // repeated .google.protobuf.UninterpretedOption uninterpreted_option = 999;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_uninterpreted_option_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(999, this->_internal_uninterpreted_option(i), target, stream);
   }
@@ -8709,14 +8709,14 @@ failure:
   cached_has_bits = _has_bits_[0];
   // optional bool deprecated = 1 [default = false];
   if (cached_has_bits & 0x00000001u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(1, this->_internal_deprecated(), target);
   }
 
   // repeated .google.protobuf.UninterpretedOption uninterpreted_option = 999;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_uninterpreted_option_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(999, this->_internal_uninterpreted_option(i), target, stream);
   }
@@ -8975,14 +8975,14 @@ failure:
   cached_has_bits = _has_bits_[0];
   // optional bool deprecated = 33 [default = false];
   if (cached_has_bits & 0x00000001u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(33, this->_internal_deprecated(), target);
   }
 
   // repeated .google.protobuf.UninterpretedOption uninterpreted_option = 999;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_uninterpreted_option_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(999, this->_internal_uninterpreted_option(i), target, stream);
   }
@@ -9265,13 +9265,13 @@ failure:
   cached_has_bits = _has_bits_[0];
   // optional bool deprecated = 33 [default = false];
   if (cached_has_bits & 0x00000001u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(33, this->_internal_deprecated(), target);
   }
 
   // optional .google.protobuf.MethodOptions.IdempotencyLevel idempotency_level = 34 [default = IDEMPOTENCY_UNKNOWN];
   if (cached_has_bits & 0x00000002u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteEnumToArray(
       34, this->_internal_idempotency_level(), target);
   }
@@ -9279,7 +9279,7 @@ failure:
   // repeated .google.protobuf.UninterpretedOption uninterpreted_option = 999;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_uninterpreted_option_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(999, this->_internal_uninterpreted_option(i), target, stream);
   }
@@ -9561,7 +9561,7 @@ failure:
 
   // required bool is_extension = 2;
   if (cached_has_bits & 0x00000002u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(2, this->_internal_is_extension(), target);
   }
 
@@ -9915,7 +9915,7 @@ failure:
   // repeated .google.protobuf.UninterpretedOption.NamePart name = 2;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_name_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(2, this->_internal_name(i), target, stream);
   }
@@ -9933,19 +9933,19 @@ failure:
 
   // optional uint64 positive_int_value = 4;
   if (cached_has_bits & 0x00000008u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteUInt64ToArray(4, this->_internal_positive_int_value(), target);
   }
 
   // optional int64 negative_int_value = 5;
   if (cached_has_bits & 0x00000010u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(5, this->_internal_negative_int_value(), target);
   }
 
   // optional double double_value = 6;
   if (cached_has_bits & 0x00000020u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteDoubleToArray(6, this->_internal_double_value(), target);
   }
 
@@ -10634,7 +10634,7 @@ failure:
   // repeated .google.protobuf.SourceCodeInfo.Location location = 1;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_location_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(1, this->_internal_location(i), target, stream);
   }
@@ -10918,13 +10918,13 @@ failure:
 
   // optional int32 begin = 3;
   if (cached_has_bits & 0x00000002u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(3, this->_internal_begin(), target);
   }
 
   // optional int32 end = 4;
   if (cached_has_bits & 0x00000004u) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(4, this->_internal_end(), target);
   }
 
@@ -11183,7 +11183,7 @@ failure:
   // repeated .google.protobuf.GeneratedCodeInfo.Annotation annotation = 1;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_annotation_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(1, this->_internal_annotation(i), target, stream);
   }

src/google/protobuf/duration.pb.cc

@@ -203,13 +203,13 @@ failure:
 
   // int64 seconds = 1;
   if (this->seconds() != 0) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(1, this->_internal_seconds(), target);
   }
 
   // int32 nanos = 2;
   if (this->nanos() != 0) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(2, this->_internal_nanos(), target);
   }
 

src/google/protobuf/dynamic_message.h

@@ -45,6 +45,7 @@
 #include <google/protobuf/stubs/common.h>
 #include <google/protobuf/message.h>
 #include <google/protobuf/stubs/mutex.h>
+#include <google/protobuf/reflection.h>
 #include <google/protobuf/repeated_field.h>
 
 #ifdef SWIG
@@ -158,16 +159,13 @@ class PROTOBUF_EXPORT DynamicMapSorter {
   static std::vector<const Message*> Sort(const Message& message, int map_size,
                                           const Reflection* reflection,
                                           const FieldDescriptor* field) {
-    std::vector<const Message*> result(static_cast<size_t>(map_size));
-    const RepeatedPtrField<Message>& map_field =
-        reflection->GetRepeatedPtrField<Message>(message, field);
-    size_t i = 0;
-    for (RepeatedPtrField<Message>::const_pointer_iterator it =
-             map_field.pointer_begin();
-         it != map_field.pointer_end();) {
-      result[i++] = *it++;
+    std::vector<const Message*> result;
+    result.reserve(map_size);
+    RepeatedFieldRef<Message> map_field =
+        reflection->GetRepeatedFieldRef<Message>(message, field);
+    for (auto it = map_field.begin(); it != map_field.end(); ++it) {
+      result.push_back(&*it);
     }
-    GOOGLE_DCHECK_EQ(result.size(), i);
     MapEntryMessageComparator comparator(field->message_type());
     std::stable_sort(result.begin(), result.end(), comparator);
     // Complain if the keys aren't in ascending order.

src/google/protobuf/extension_set.cc

@@ -1676,7 +1676,7 @@ size_t ExtensionSet::Extension::ByteSize(int number) const {
 #undef HANDLE_TYPE
       case WireFormatLite::TYPE_MESSAGE: {
         if (is_lazy) {
-          size_t size = lazymessage_value->ByteSize();
+          size_t size = lazymessage_value->ByteSizeLong();
           result += io::CodedOutputStream::VarintSize32(size) + size;
         } else {
           result += WireFormatLite::MessageSize(*message_value);
@@ -1931,7 +1931,7 @@ uint8* ExtensionSet::Extension::InternalSerializeFieldWithCachedSizesToArray(
     if (is_packed) {
       if (cached_size == 0) return target;
 
-      stream->EnsureSpace(&target);
+      target = stream->EnsureSpace(target);
       target = WireFormatLite::WriteTagToArray(
           number, WireFormatLite::WIRETYPE_LENGTH_DELIMITED, target);
       target = WireFormatLite::WriteInt32NoTagToArray(cached_size, target);
@@ -1940,7 +1940,7 @@ uint8* ExtensionSet::Extension::InternalSerializeFieldWithCachedSizesToArray(
 #define HANDLE_TYPE(UPPERCASE, CAMELCASE, LOWERCASE)                 \
   case WireFormatLite::TYPE_##UPPERCASE:                             \
     for (int i = 0; i < repeated_##LOWERCASE##_value->size(); i++) { \
-      stream->EnsureSpace(&target);                                  \
+      target = stream->EnsureSpace(target);                          \
       target = WireFormatLite::Write##CAMELCASE##NoTagToArray(       \
           repeated_##LOWERCASE##_value->Get(i), target);             \
     }                                                                \
@@ -1974,7 +1974,7 @@ uint8* ExtensionSet::Extension::InternalSerializeFieldWithCachedSizesToArray(
 #define HANDLE_TYPE(UPPERCASE, CAMELCASE, LOWERCASE)                 \
   case WireFormatLite::TYPE_##UPPERCASE:                             \
     for (int i = 0; i < repeated_##LOWERCASE##_value->size(); i++) { \
-      stream->EnsureSpace(&target);                                  \
+      target = stream->EnsureSpace(target);                          \
       target = WireFormatLite::Write##CAMELCASE##ToArray(            \
           number, repeated_##LOWERCASE##_value->Get(i), target);     \
     }                                                                \
@@ -1998,7 +1998,7 @@ uint8* ExtensionSet::Extension::InternalSerializeFieldWithCachedSizesToArray(
 #define HANDLE_TYPE(UPPERCASE, CAMELCASE, LOWERCASE)                 \
   case WireFormatLite::TYPE_##UPPERCASE:                             \
     for (int i = 0; i < repeated_##LOWERCASE##_value->size(); i++) { \
-      stream->EnsureSpace(&target);                                  \
+      target = stream->EnsureSpace(target);                          \
       target = stream->WriteString(                                  \
           number, repeated_##LOWERCASE##_value->Get(i), target);     \
     }                                                                \
@@ -2009,7 +2009,7 @@ uint8* ExtensionSet::Extension::InternalSerializeFieldWithCachedSizesToArray(
 #define HANDLE_TYPE(UPPERCASE, CAMELCASE, LOWERCASE)                     \
   case WireFormatLite::TYPE_##UPPERCASE:                                 \
     for (int i = 0; i < repeated_##LOWERCASE##_value->size(); i++) {     \
-      stream->EnsureSpace(&target);                                      \
+      target = stream->EnsureSpace(target);                              \
       target = WireFormatLite::InternalWrite##CAMELCASE##ToArray(        \
           number, repeated_##LOWERCASE##_value->Get(i), target, stream); \
     }                                                                    \
@@ -2024,7 +2024,7 @@ uint8* ExtensionSet::Extension::InternalSerializeFieldWithCachedSizesToArray(
     switch (real_type(type)) {
 #define HANDLE_TYPE(UPPERCASE, CAMELCASE, VALUE)                               \
   case WireFormatLite::TYPE_##UPPERCASE:                                       \
-    stream->EnsureSpace(&target);                                              \
+    target = stream->EnsureSpace(target);                                      \
     target = WireFormatLite::Write##CAMELCASE##ToArray(number, VALUE, target); \
     break
 
@@ -2045,14 +2045,14 @@ uint8* ExtensionSet::Extension::InternalSerializeFieldWithCachedSizesToArray(
 #undef HANDLE_TYPE
 #define HANDLE_TYPE(UPPERCASE, CAMELCASE, VALUE)         \
   case WireFormatLite::TYPE_##UPPERCASE:                 \
-    stream->EnsureSpace(&target);                        \
+    target = stream->EnsureSpace(target);                \
     target = stream->WriteString(number, VALUE, target); \
     break
       HANDLE_TYPE(STRING, String, *string_value);
       HANDLE_TYPE(BYTES, Bytes, *string_value);
 #undef HANDLE_TYPE
       case WireFormatLite::TYPE_GROUP:
-        stream->EnsureSpace(&target);
+        target = stream->EnsureSpace(target);
         target = WireFormatLite::InternalWriteGroupToArray(
             number, *message_value, target, stream);
         break;
@@ -2061,7 +2061,7 @@ uint8* ExtensionSet::Extension::InternalSerializeFieldWithCachedSizesToArray(
           target =
               lazymessage_value->WriteMessageToArray(number, target, stream);
         } else {
-          stream->EnsureSpace(&target);
+          target = stream->EnsureSpace(target);
           target = WireFormatLite::InternalWriteMessageToArray(
               number, *message_value, target, stream);
         }
@@ -2082,7 +2082,7 @@ ExtensionSet::Extension::InternalSerializeMessageSetItemWithCachedSizesToArray(
 
   if (is_cleared) return target;
 
-  stream->EnsureSpace(&target);
+  target = stream->EnsureSpace(target);
   // Start group.
   target = io::CodedOutputStream::WriteTagToArray(
       WireFormatLite::kMessageSetItemStartTag, target);
@@ -2099,7 +2099,7 @@ ExtensionSet::Extension::InternalSerializeMessageSetItemWithCachedSizesToArray(
         stream);
   }
   // End group.
-  stream->EnsureSpace(&target);
+  target = stream->EnsureSpace(target);
   target = io::CodedOutputStream::WriteTagToArray(
       WireFormatLite::kMessageSetItemEndTag, target);
   return target;

src/google/protobuf/generated_message_reflection.cc

@@ -1194,8 +1194,8 @@ void Reflection::SetString(Message* message, const FieldDescriptor* field,
                            std::string value) const {
   USAGE_CHECK_ALL(SetString, SINGULAR, STRING);
   if (field->is_extension()) {
-    return MutableExtensionSet(message)->SetString(field->number(),
-                                                   field->type(), std::move(value), field);
+    return MutableExtensionSet(message)->SetString(
+        field->number(), field->type(), std::move(value), field);
   } else {
     switch (field->options().ctype()) {
       default:  // TODO(kenton):  Support other string reps.
@@ -1214,7 +1214,8 @@ void Reflection::SetString(Message* message, const FieldDescriptor* field,
               ->UnsafeSetDefault(default_ptr);
         }
         MutableField<ArenaStringPtr>(message, field)
-            ->Mutable(default_ptr, GetArena(message))->assign(std::move(value));
+            ->Mutable(default_ptr, GetArena(message))
+            ->assign(std::move(value));
         break;
       }
     }
@@ -1264,7 +1265,8 @@ void Reflection::SetRepeatedString(Message* message,
     switch (field->options().ctype()) {
       default:  // TODO(kenton):  Support other string reps.
       case FieldOptions::STRING:
-        MutableRepeatedField<std::string>(message, field, index)->assign(std::move(value));
+        MutableRepeatedField<std::string>(message, field, index)
+            ->assign(std::move(value));
         break;
     }
   }

src/google/protobuf/io/coded_stream.cc

@@ -866,7 +866,7 @@ uint8* EpsCopyOutputStream::WriteRawLittleEndian32(const void* data, int size,
   auto p = static_cast<const uint8*>(data);
   auto end = p + size;
   while (end - p >= kSlopBytes) {
-    EnsureSpace(&ptr);
+    ptr = EnsureSpace(ptr);
     uint32 buffer[4];
     static_assert(sizeof(buffer) == kSlopBytes, "Buffer must be kSlopBytes");
     std::memcpy(buffer, p, kSlopBytes);
@@ -875,7 +875,7 @@ uint8* EpsCopyOutputStream::WriteRawLittleEndian32(const void* data, int size,
       ptr = CodedOutputStream::WriteLittleEndian32ToArray(x, ptr);
   }
   while (p < end) {
-    EnsureSpace(&ptr);
+    ptr = EnsureSpace(ptr);
     uint32 buffer;
     std::memcpy(&buffer, p, 4);
     p += 4;
@@ -889,7 +889,7 @@ uint8* EpsCopyOutputStream::WriteRawLittleEndian64(const void* data, int size,
   auto p = static_cast<const uint8*>(data);
   auto end = p + size;
   while (end - p >= kSlopBytes) {
-    EnsureSpace(&ptr);
+    ptr = EnsureSpace(ptr);
     uint64 buffer[2];
     static_assert(sizeof(buffer) == kSlopBytes, "Buffer must be kSlopBytes");
     std::memcpy(buffer, p, kSlopBytes);
@@ -898,7 +898,7 @@ uint8* EpsCopyOutputStream::WriteRawLittleEndian64(const void* data, int size,
       ptr = CodedOutputStream::WriteLittleEndian64ToArray(x, ptr);
   }
   while (p < end) {
-    EnsureSpace(&ptr);
+    ptr = EnsureSpace(ptr);
     uint64 buffer;
     std::memcpy(&buffer, p, 8);
     p += 8;
@@ -912,7 +912,7 @@ uint8* EpsCopyOutputStream::WriteRawLittleEndian64(const void* data, int size,
 uint8* EpsCopyOutputStream::WriteStringMaybeAliasedOutline(uint32 num,
                                                            const std::string& s,
                                                            uint8* ptr) {
-  EnsureSpace(&ptr);
+  ptr = EnsureSpace(ptr);
   uint32 size = s.size();
   ptr = WriteLengthDelim(num, size, ptr);
   return WriteRawMaybeAliased(s.data(), size, ptr);
@@ -920,7 +920,7 @@ uint8* EpsCopyOutputStream::WriteStringMaybeAliasedOutline(uint32 num,
 
 uint8* EpsCopyOutputStream::WriteStringOutline(uint32 num, const std::string& s,
                                                uint8* ptr) {
-  EnsureSpace(&ptr);
+  ptr = EnsureSpace(ptr);
   uint32 size = s.size();
   ptr = WriteLengthDelim(num, size, ptr);
   return WriteRaw(s.data(), size, ptr);

src/google/protobuf/io/coded_stream.h

@@ -109,6 +109,7 @@
 #ifndef GOOGLE_PROTOBUF_IO_CODED_STREAM_H__
 #define GOOGLE_PROTOBUF_IO_CODED_STREAM_H__
 
+
 #include <assert.h>
 
 #include <atomic>
@@ -684,10 +685,11 @@ class PROTOBUF_EXPORT EpsCopyOutputStream {
   // After this it's guaranteed you can safely write kSlopBytes to ptr. This
   // will never fail! The underlying stream can produce an error. Use HadError
   // to check for errors.
-  void EnsureSpace(uint8** ptr) {
-    if (PROTOBUF_PREDICT_FALSE(*ptr >= end_)) {
-      *ptr = EnsureSpaceFallback(*ptr);
+  PROTOBUF_MUST_USE_RESULT uint8* EnsureSpace(uint8* ptr) {
+    if (PROTOBUF_PREDICT_FALSE(ptr >= end_)) {
+      return EnsureSpaceFallback(ptr);
     }
+    return ptr;
   }
 
   uint8* WriteRaw(const void* data, int size, uint8* ptr) {
@@ -782,7 +784,7 @@ class PROTOBUF_EXPORT EpsCopyOutputStream {
   template <typename T>
   PROTOBUF_ALWAYS_INLINE uint8* WriteFixedPacked(int num, const T& r,
                                                  uint8* ptr) {
-    EnsureSpace(&ptr);
+    ptr = EnsureSpace(ptr);
     constexpr auto element_size = sizeof(typename T::value_type);
     auto size = r.size() * element_size;
     ptr = WriteLengthDelim(num, size, ptr);
@@ -874,12 +876,12 @@ class PROTOBUF_EXPORT EpsCopyOutputStream {
   template <typename T, typename E>
   PROTOBUF_ALWAYS_INLINE uint8* WriteVarintPacked(int num, const T& r, int size,
                                                   uint8* ptr, const E& encode) {
-    EnsureSpace(&ptr);
+    ptr = EnsureSpace(ptr);
     ptr = WriteLengthDelim(num, size, ptr);
     auto it = r.data();
     auto end = it + r.size();
     do {
-      EnsureSpace(&ptr);
+      ptr = EnsureSpace(ptr);
       ptr = UnsafeVarint(encode(*it++), ptr);
     } while (it < end);
     return ptr;
@@ -1124,14 +1126,14 @@ class PROTOBUF_EXPORT CodedOutputStream {
 
   // Write a 32-bit little-endian integer.
   void WriteLittleEndian32(uint32 value) {
-    impl_.EnsureSpace(&cur_);
+    cur_ = impl_.EnsureSpace(cur_);
     SetCur(WriteLittleEndian32ToArray(value, Cur()));
   }
   // Like WriteLittleEndian32()  but writing directly to the target array.
   static uint8* WriteLittleEndian32ToArray(uint32 value, uint8* target);
   // Write a 64-bit little-endian integer.
   void WriteLittleEndian64(uint64 value) {
-    impl_.EnsureSpace(&cur_);
+    cur_ = impl_.EnsureSpace(cur_);
     SetCur(WriteLittleEndian64ToArray(value, Cur()));
   }
   // Like WriteLittleEndian64()  but writing directly to the target array.
@@ -1636,12 +1638,12 @@ inline uint8* CodedOutputStream::WriteLittleEndian64ToArray(uint64 value,
 }
 
 inline void CodedOutputStream::WriteVarint32(uint32 value) {
-  impl_.EnsureSpace(&cur_);
+  cur_ = impl_.EnsureSpace(cur_);
   SetCur(WriteVarint32ToArray(value, Cur()));
 }
 
 inline void CodedOutputStream::WriteVarint64(uint64 value) {
-  impl_.EnsureSpace(&cur_);
+  cur_ = impl_.EnsureSpace(cur_);
   SetCur(WriteVarint64ToArray(value, Cur()));
 }
 

src/google/protobuf/io/coded_stream_unittest.cc

@@ -228,17 +228,17 @@ TEST_F(CodedStreamTest, EmptyInputBeforeEos) {
     In() : count_(0) {}
 
    private:
-    virtual bool Next(const void** data, int* size) {
+    bool Next(const void** data, int* size) override {
       *data = NULL;
       *size = 0;
       return count_++ < 2;
     }
-    virtual void BackUp(int count) { GOOGLE_LOG(FATAL) << "Tests never call this."; }
-    virtual bool Skip(int count) {
+    void BackUp(int count) override { GOOGLE_LOG(FATAL) << "Tests never call this."; }
+    bool Skip(int count) override {
       GOOGLE_LOG(FATAL) << "Tests never call this.";
       return false;
     }
-    virtual int64 ByteCount() const { return 0; }
+    int64_t ByteCount() const override { return 0; }
     int count_;
   } in;
   CodedInputStream input(&in);
@@ -1280,7 +1280,7 @@ class ReallyBigInputStream : public ZeroCopyInputStream {
   ~ReallyBigInputStream() {}
 
   // implements ZeroCopyInputStream ----------------------------------
-  bool Next(const void** data, int* size) {
+  bool Next(const void** data, int* size) override {
     // We only expect BackUp() to be called at the end.
     EXPECT_EQ(0, backup_amount_);
 
@@ -1302,13 +1302,13 @@ class ReallyBigInputStream : public ZeroCopyInputStream {
     }
   }
 
-  void BackUp(int count) { backup_amount_ = count; }
+  void BackUp(int count) override { backup_amount_ = count; }
 
-  bool Skip(int count) {
+  bool Skip(int count) override {
     GOOGLE_LOG(FATAL) << "Not implemented.";
     return false;
   }
-  int64 ByteCount() const {
+  int64_t ByteCount() const override {
     GOOGLE_LOG(FATAL) << "Not implemented.";
     return 0;
   }

src/google/protobuf/io/gzip_stream.cc

@@ -185,7 +185,7 @@ bool GzipInputStream::Skip(int count) {
   }
   return ok;
 }
-int64 GzipInputStream::ByteCount() const {
+int64_t GzipInputStream::ByteCount() const {
   int64 ret = byte_count_ + zcontext_.total_out;
   if (zcontext_.next_out != NULL && output_position_ != NULL) {
     ret += reinterpret_cast<uintptr_t>(zcontext_.next_out) -
@@ -301,7 +301,7 @@ void GzipOutputStream::BackUp(int count) {
   GOOGLE_CHECK_GE(zcontext_.avail_in, count);
   zcontext_.avail_in -= count;
 }
-int64 GzipOutputStream::ByteCount() const {
+int64_t GzipOutputStream::ByteCount() const {
   return zcontext_.total_in + zcontext_.avail_in;
 }
 

src/google/protobuf/io/gzip_stream.h

@@ -43,6 +43,7 @@
 #ifndef GOOGLE_PROTOBUF_IO_GZIP_STREAM_H__
 #define GOOGLE_PROTOBUF_IO_GZIP_STREAM_H__
 
+
 #include <google/protobuf/stubs/common.h>
 #include <google/protobuf/io/zero_copy_stream.h>
 #include <google/protobuf/port.h>
@@ -82,7 +83,7 @@ class PROTOBUF_EXPORT GzipInputStream : public ZeroCopyInputStream {
   bool Next(const void** data, int* size);
   void BackUp(int count);
   bool Skip(int count);
-  int64 ByteCount() const;
+  int64_t ByteCount() const;
 
  private:
   Format format_;
@@ -168,7 +169,7 @@ class PROTOBUF_EXPORT GzipOutputStream : public ZeroCopyOutputStream {
   // implements ZeroCopyOutputStream ---------------------------------
   bool Next(void** data, int* size);
   void BackUp(int count);
-  int64 ByteCount() const;
+  int64_t ByteCount() const;
 
  private:
   ZeroCopyOutputStream* sub_stream_;

src/google/protobuf/io/printer.h

@@ -37,6 +37,7 @@
 #ifndef GOOGLE_PROTOBUF_IO_PRINTER_H__
 #define GOOGLE_PROTOBUF_IO_PRINTER_H__
 
+
 #include <map>
 #include <string>
 #include <vector>

src/google/protobuf/io/tokenizer.h

@@ -37,6 +37,7 @@
 #ifndef GOOGLE_PROTOBUF_IO_TOKENIZER_H__
 #define GOOGLE_PROTOBUF_IO_TOKENIZER_H__
 
+
 #include <string>
 #include <vector>
 

src/google/protobuf/io/tokenizer_unittest.cc

@@ -127,7 +127,7 @@ class TestInputStream : public ZeroCopyInputStream {
   ~TestInputStream() {}
 
   // implements ZeroCopyInputStream ----------------------------------
-  bool Next(const void** data, int* size) {
+  bool Next(const void** data, int* size) override {
     // We'll return empty buffers starting with the first buffer, and every
     // 3 and 5 buffers after that.
     if (counter_ % 3 == 0 || counter_ % 5 == 0) {
@@ -141,9 +141,9 @@ class TestInputStream : public ZeroCopyInputStream {
     }
   }
 
-  void BackUp(int count) { return array_stream_.BackUp(count); }
-  bool Skip(int count) { return array_stream_.Skip(count); }
-  int64 ByteCount() const { return array_stream_.ByteCount(); }
+  void BackUp(int count) override { return array_stream_.BackUp(count); }
+  bool Skip(int count) override { return array_stream_.Skip(count); }
+  int64_t ByteCount() const override { return array_stream_.ByteCount(); }
 
  private:
   ArrayInputStream array_stream_;

src/google/protobuf/io/zero_copy_stream.h

@@ -107,6 +107,7 @@
 #ifndef GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_H__
 #define GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_H__
 
+
 #include <string>
 
 #include <google/protobuf/stubs/common.h>

src/google/protobuf/io/zero_copy_stream_impl.cc

@@ -95,7 +95,7 @@ void FileInputStream::BackUp(int count) { impl_.BackUp(count); }
 
 bool FileInputStream::Skip(int count) { return impl_.Skip(count); }
 
-int64 FileInputStream::ByteCount() const { return impl_.ByteCount(); }
+int64_t FileInputStream::ByteCount() const { return impl_.ByteCount(); }
 
 FileInputStream::CopyingFileInputStream::CopyingFileInputStream(
     int file_descriptor)
@@ -182,7 +182,7 @@ bool FileOutputStream::Next(void** data, int* size) {
 
 void FileOutputStream::BackUp(int count) { impl_.BackUp(count); }
 
-int64 FileOutputStream::ByteCount() const { return impl_.ByteCount(); }
+int64_t FileOutputStream::ByteCount() const { return impl_.ByteCount(); }
 
 FileOutputStream::CopyingFileOutputStream::CopyingFileOutputStream(
     int file_descriptor)
@@ -262,7 +262,7 @@ void IstreamInputStream::BackUp(int count) { impl_.BackUp(count); }
 
 bool IstreamInputStream::Skip(int count) { return impl_.Skip(count); }
 
-int64 IstreamInputStream::ByteCount() const { return impl_.ByteCount(); }
+int64_t IstreamInputStream::ByteCount() const { return impl_.ByteCount(); }
 
 IstreamInputStream::CopyingIstreamInputStream::CopyingIstreamInputStream(
     std::istream* input)
@@ -293,7 +293,7 @@ bool OstreamOutputStream::Next(void** data, int* size) {
 
 void OstreamOutputStream::BackUp(int count) { impl_.BackUp(count); }
 
-int64 OstreamOutputStream::ByteCount() const { return impl_.ByteCount(); }
+int64_t OstreamOutputStream::ByteCount() const { return impl_.ByteCount(); }
 
 OstreamOutputStream::CopyingOstreamOutputStream::CopyingOstreamOutputStream(
     std::ostream* output)
@@ -359,7 +359,7 @@ bool ConcatenatingInputStream::Skip(int count) {
   return false;
 }
 
-int64 ConcatenatingInputStream::ByteCount() const {
+int64_t ConcatenatingInputStream::ByteCount() const {
   if (stream_count_ == 0) {
     return bytes_retired_;
   } else {

src/google/protobuf/io/zero_copy_stream_impl.h

@@ -40,6 +40,7 @@
 #ifndef GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_IMPL_H__
 #define GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_IMPL_H__
 
+
 #include <iosfwd>
 #include <string>
 
@@ -92,7 +93,7 @@ class PROTOBUF_EXPORT FileInputStream : public ZeroCopyInputStream {
   bool Next(const void** data, int* size) override;
   void BackUp(int count) override;
   bool Skip(int count) override;
-  int64 ByteCount() const override;
+  int64_t ByteCount() const override;
 
  private:
   class PROTOBUF_EXPORT CopyingFileInputStream : public CopyingInputStream {
@@ -174,7 +175,7 @@ class PROTOBUF_EXPORT FileOutputStream : public ZeroCopyOutputStream {
   // implements ZeroCopyOutputStream ---------------------------------
   bool Next(void** data, int* size) override;
   void BackUp(int count) override;
-  int64 ByteCount() const override;
+  int64_t ByteCount() const override;
 
  private:
   class PROTOBUF_EXPORT CopyingFileOutputStream : public CopyingOutputStream {
@@ -225,7 +226,7 @@ class PROTOBUF_EXPORT IstreamInputStream : public ZeroCopyInputStream {
   bool Next(const void** data, int* size) override;
   void BackUp(int count) override;
   bool Skip(int count) override;
-  int64 ByteCount() const override;
+  int64_t ByteCount() const override;
 
  private:
   class PROTOBUF_EXPORT CopyingIstreamInputStream : public CopyingInputStream {
@@ -268,7 +269,7 @@ class PROTOBUF_EXPORT OstreamOutputStream : public ZeroCopyOutputStream {
   // implements ZeroCopyOutputStream ---------------------------------
   bool Next(void** data, int* size) override;
   void BackUp(int count) override;
-  int64 ByteCount() const override;
+  int64_t ByteCount() const override;
 
  private:
   class PROTOBUF_EXPORT CopyingOstreamOutputStream
@@ -313,7 +314,7 @@ class PROTOBUF_EXPORT ConcatenatingInputStream : public ZeroCopyInputStream {
   bool Next(const void** data, int* size) override;
   void BackUp(int count) override;
   bool Skip(int count) override;
-  int64 ByteCount() const override;
+  int64_t ByteCount() const override;
 
 
  private:

src/google/protobuf/io/zero_copy_stream_impl_lite.cc

@@ -97,7 +97,7 @@ bool ArrayInputStream::Skip(int count) {
   }
 }
 
-int64 ArrayInputStream::ByteCount() const { return position_; }
+int64_t ArrayInputStream::ByteCount() const { return position_; }
 
 
 // ===================================================================
@@ -132,7 +132,7 @@ void ArrayOutputStream::BackUp(int count) {
   last_returned_size_ = 0;  // Don't let caller back up further.
 }
 
-int64 ArrayOutputStream::ByteCount() const { return position_; }
+int64_t ArrayOutputStream::ByteCount() const { return position_; }
 
 // ===================================================================
 
@@ -176,7 +176,7 @@ void StringOutputStream::BackUp(int count) {
   target_->resize(target_->size() - count);
 }
 
-int64 StringOutputStream::ByteCount() const {
+int64_t StringOutputStream::ByteCount() const {
   GOOGLE_CHECK(target_ != NULL);
   return target_->size();
 }
@@ -282,7 +282,7 @@ bool CopyingInputStreamAdaptor::Skip(int count) {
   return skipped == count;
 }
 
-int64 CopyingInputStreamAdaptor::ByteCount() const {
+int64_t CopyingInputStreamAdaptor::ByteCount() const {
   return position_ - backup_bytes_;
 }
 
@@ -342,7 +342,7 @@ void CopyingOutputStreamAdaptor::BackUp(int count) {
   buffer_used_ -= count;
 }
 
-int64 CopyingOutputStreamAdaptor::ByteCount() const {
+int64_t CopyingOutputStreamAdaptor::ByteCount() const {
   return position_ + buffer_used_;
 }
 
@@ -424,7 +424,7 @@ bool LimitingInputStream::Skip(int count) {
   }
 }
 
-int64 LimitingInputStream::ByteCount() const {
+int64_t LimitingInputStream::ByteCount() const {
   if (limit_ < 0) {
     return input_->ByteCount() + limit_ - prior_bytes_read_;
   } else {

src/google/protobuf/io/zero_copy_stream_impl_lite.h

@@ -44,6 +44,7 @@
 #ifndef GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_IMPL_LITE_H__
 #define GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_IMPL_LITE_H__
 
+
 #include <iosfwd>
 #include <memory>
 #include <string>
@@ -79,7 +80,7 @@ class PROTOBUF_EXPORT ArrayInputStream : public ZeroCopyInputStream {
   bool Next(const void** data, int* size) override;
   void BackUp(int count) override;
   bool Skip(int count) override;
-  int64 ByteCount() const override;
+  int64_t ByteCount() const override;
 
 
  private:
@@ -112,7 +113,7 @@ class PROTOBUF_EXPORT ArrayOutputStream : public ZeroCopyOutputStream {
   // implements ZeroCopyOutputStream ---------------------------------
   bool Next(void** data, int* size) override;
   void BackUp(int count) override;
-  int64 ByteCount() const override;
+  int64_t ByteCount() const override;
 
  private:
   uint8* const data_;     // The byte array.
@@ -146,7 +147,7 @@ class PROTOBUF_EXPORT StringOutputStream : public ZeroCopyOutputStream {
   // implements ZeroCopyOutputStream ---------------------------------
   bool Next(void** data, int* size) override;
   void BackUp(int count) override;
-  int64 ByteCount() const override;
+  int64_t ByteCount() const override;
 
  private:
   static const int kMinimumSize = 16;
@@ -218,7 +219,7 @@ class PROTOBUF_EXPORT CopyingInputStreamAdaptor : public ZeroCopyInputStream {
   bool Next(const void** data, int* size) override;
   void BackUp(int count) override;
   bool Skip(int count) override;
-  int64 ByteCount() const override;
+  int64_t ByteCount() const override;
 
  private:
   // Insures that buffer_ is not NULL.
@@ -305,7 +306,7 @@ class PROTOBUF_EXPORT CopyingOutputStreamAdaptor : public ZeroCopyOutputStream {
   // implements ZeroCopyOutputStream ---------------------------------
   bool Next(void** data, int* size) override;
   void BackUp(int count) override;
-  int64 ByteCount() const override;
+  int64_t ByteCount() const override;
 
  private:
   // Write the current buffer, if it is present.
@@ -352,7 +353,7 @@ class PROTOBUF_EXPORT LimitingInputStream : public ZeroCopyInputStream {
   bool Next(const void** data, int* size) override;
   void BackUp(int count) override;
   bool Skip(int count) override;
-  int64 ByteCount() const override;
+  int64_t ByteCount() const override;
 
 
  private:

src/google/protobuf/map_entry_lite.h

@@ -114,7 +114,7 @@ struct MapEntryFuncs {
   static uint8* InternalSerialize(int field_number, const Key& key,
                                   const Value& value, uint8* ptr,
                                   io::EpsCopyOutputStream* stream) {
-    stream->EnsureSpace(&ptr);
+    ptr = stream->EnsureSpace(ptr);
     ptr = WireFormatLite::WriteTagToArray(
         field_number, WireFormatLite::WIRETYPE_LENGTH_DELIMITED, ptr);
     ptr = io::CodedOutputStream::WriteVarint32ToArray(GetCachedSize(key, value),

src/google/protobuf/map_field.cc

@@ -172,10 +172,13 @@ int DynamicMapField::size() const { return GetMap().size(); }
 
 void DynamicMapField::Clear() {
   Map<MapKey, MapValueRef>* map = &const_cast<DynamicMapField*>(this)->map_;
-  for (Map<MapKey, MapValueRef>::iterator iter = map->begin();
-       iter != map->end(); ++iter) {
-    iter->second.DeleteData();
+  if (MapFieldBase::arena_ == nullptr) {
+    for (Map<MapKey, MapValueRef>::iterator iter = map->begin();
+         iter != map->end(); ++iter) {
+      iter->second.DeleteData();
+    }
   }
+
   map->clear();
 
   if (MapFieldBase::repeated_field_ != nullptr) {
@@ -199,11 +202,11 @@ void DynamicMapField::AllocateMapValue(MapValueRef* map_val) {
   // Allocate memory for the MapValueRef, and initialize to
   // default value.
   switch (val_des->cpp_type()) {
-#define HANDLE_TYPE(CPPTYPE, TYPE)           \
-  case FieldDescriptor::CPPTYPE_##CPPTYPE: { \
-    TYPE* value = new TYPE();                \
-    map_val->SetValue(value);                \
-    break;                                   \
+#define HANDLE_TYPE(CPPTYPE, TYPE)                           \
+  case FieldDescriptor::CPPTYPE_##CPPTYPE: {                 \
+    TYPE* value = Arena::Create<TYPE>(MapFieldBase::arena_); \
+    map_val->SetValue(value);                                \
+    break;                                                   \
   }
     HANDLE_TYPE(INT32, int32);
     HANDLE_TYPE(INT64, int64);
@@ -218,7 +221,7 @@ void DynamicMapField::AllocateMapValue(MapValueRef* map_val) {
     case FieldDescriptor::CPPTYPE_MESSAGE: {
       const Message& message =
           default_entry_->GetReflection()->GetMessage(*default_entry_, val_des);
-      Message* value = message.New();
+      Message* value = message.New(MapFieldBase::arena_);
       map_val->SetValue(value);
       break;
     }
@@ -251,7 +254,9 @@ bool DynamicMapField::DeleteMapValue(const MapKey& map_key) {
   }
   // Set map dirty only if the delete is successful.
   MapFieldBase::SetMapDirty();
-  iter->second.DeleteData();
+  if (MapFieldBase::arena_ == nullptr) {
+    iter->second.DeleteData();
+  }
   map_.erase(iter);
   return true;
 }
@@ -373,7 +378,7 @@ void DynamicMapField::SyncRepeatedFieldWithMapNoLock() const {
 
   for (Map<MapKey, MapValueRef>::const_iterator it = map_.begin();
        it != map_.end(); ++it) {
-    Message* new_entry = default_entry_->New();
+    Message* new_entry = default_entry_->New(MapFieldBase::arena_);
     MapFieldBase::repeated_field_->AddAllocated(new_entry);
     const MapKey& map_key = it->first;
     switch (key_des->cpp_type()) {
@@ -449,14 +454,17 @@ void DynamicMapField::SyncMapWithRepeatedFieldNoLock() const {
       default_entry_->GetDescriptor()->FindFieldByName("value");
   // DynamicMapField owns map values. Need to delete them before clearing
   // the map.
-  for (Map<MapKey, MapValueRef>::iterator iter = map->begin();
-       iter != map->end(); ++iter) {
-    iter->second.DeleteData();
+  if (MapFieldBase::arena_ == nullptr) {
+    for (Map<MapKey, MapValueRef>::iterator iter = map->begin();
+         iter != map->end(); ++iter) {
+      iter->second.DeleteData();
+    }
   }
   map->clear();
   for (RepeatedPtrField<Message>::iterator it =
            MapFieldBase::repeated_field_->begin();
        it != MapFieldBase::repeated_field_->end(); ++it) {
+    // MapKey type will be set later.
     MapKey map_key;
     switch (key_des->cpp_type()) {
       case FieldDescriptor::CPPTYPE_STRING:
@@ -485,21 +493,23 @@ void DynamicMapField::SyncMapWithRepeatedFieldNoLock() const {
         break;
     }
 
-    // Remove existing map value with same key.
-    Map<MapKey, MapValueRef>::iterator iter = map->find(map_key);
-    if (iter != map->end()) {
-      iter->second.DeleteData();
+    if (MapFieldBase::arena_ == nullptr) {
+      // Remove existing map value with same key.
+      Map<MapKey, MapValueRef>::iterator iter = map->find(map_key);
+      if (iter != map->end()) {
+        iter->second.DeleteData();
+      }
     }
 
     MapValueRef& map_val = (*map)[map_key];
     map_val.SetType(val_des->cpp_type());
     switch (val_des->cpp_type()) {
-#define HANDLE_TYPE(CPPTYPE, TYPE, METHOD)          \
-  case FieldDescriptor::CPPTYPE_##CPPTYPE: {        \
-    TYPE* value = new TYPE;                         \
-    *value = reflection->Get##METHOD(*it, val_des); \
-    map_val.SetValue(value);                        \
-    break;                                          \
+#define HANDLE_TYPE(CPPTYPE, TYPE, METHOD)                   \
+  case FieldDescriptor::CPPTYPE_##CPPTYPE: {                 \
+    TYPE* value = Arena::Create<TYPE>(MapFieldBase::arena_); \
+    *value = reflection->Get##METHOD(*it, val_des);          \
+    map_val.SetValue(value);                                 \
+    break;                                                   \
   }
       HANDLE_TYPE(INT32, int32, Int32);
       HANDLE_TYPE(INT64, int64, Int64);
@@ -513,7 +523,7 @@ void DynamicMapField::SyncMapWithRepeatedFieldNoLock() const {
 #undef HANDLE_TYPE
       case FieldDescriptor::CPPTYPE_MESSAGE: {
         const Message& message = reflection->GetMessage(*it, val_des);
-        Message* value = message.New();
+        Message* value = message.New(MapFieldBase::arena_);
         value->CopyFrom(message);
         map_val.SetValue(value);
         break;

src/google/protobuf/map_field.h

@@ -372,6 +372,7 @@ class PROTOBUF_EXPORT MapKey {
  public:
   MapKey() : type_(0) {}
   MapKey(const MapKey& other) : type_(0) { CopyFrom(other); }
+
   MapKey& operator=(const MapKey& other) {
     CopyFrom(other);
     return *this;
@@ -379,7 +380,7 @@ class PROTOBUF_EXPORT MapKey {
 
   ~MapKey() {
     if (type_ == FieldDescriptor::CPPTYPE_STRING) {
-      delete val_.string_value_;
+      val_.string_value_.Destruct();
     }
   }
 
@@ -412,9 +413,9 @@ class PROTOBUF_EXPORT MapKey {
     SetType(FieldDescriptor::CPPTYPE_BOOL);
     val_.bool_value_ = value;
   }
-  void SetStringValue(const std::string& val) {
+  void SetStringValue(std::string val) {
     SetType(FieldDescriptor::CPPTYPE_STRING);
-    *val_.string_value_ = val;
+    *val_.string_value_.get_mutable() = std::move(val);
   }
 
   int64 GetInt64Value() const {
@@ -439,7 +440,7 @@ class PROTOBUF_EXPORT MapKey {
   }
   const std::string& GetStringValue() const {
     TYPE_CHECK(FieldDescriptor::CPPTYPE_STRING, "MapKey::GetStringValue");
-    return *val_.string_value_;
+    return val_.string_value_.get();
   }
 
   bool operator<(const MapKey& other) const {
@@ -456,7 +457,7 @@ class PROTOBUF_EXPORT MapKey {
         GOOGLE_LOG(FATAL) << "Unsupported";
         return false;
       case FieldDescriptor::CPPTYPE_STRING:
-        return *val_.string_value_ < *other.val_.string_value_;
+        return val_.string_value_.get() < other.val_.string_value_.get();
       case FieldDescriptor::CPPTYPE_INT64:
         return val_.int64_value_ < other.val_.int64_value_;
       case FieldDescriptor::CPPTYPE_INT32:
@@ -484,7 +485,7 @@ class PROTOBUF_EXPORT MapKey {
         GOOGLE_LOG(FATAL) << "Unsupported";
         break;
       case FieldDescriptor::CPPTYPE_STRING:
-        return *val_.string_value_ == *other.val_.string_value_;
+        return val_.string_value_.get() == other.val_.string_value_.get();
       case FieldDescriptor::CPPTYPE_INT64:
         return val_.int64_value_ == other.val_.int64_value_;
       case FieldDescriptor::CPPTYPE_INT32:
@@ -510,7 +511,7 @@ class PROTOBUF_EXPORT MapKey {
         GOOGLE_LOG(FATAL) << "Unsupported";
         break;
       case FieldDescriptor::CPPTYPE_STRING:
-        *val_.string_value_ = *other.val_.string_value_;
+        *val_.string_value_.get_mutable() = other.val_.string_value_.get();
         break;
       case FieldDescriptor::CPPTYPE_INT64:
         val_.int64_value_ = other.val_.int64_value_;
@@ -538,7 +539,7 @@ class PROTOBUF_EXPORT MapKey {
 
   union KeyValue {
     KeyValue() {}
-    std::string* string_value_;
+    internal::ExplicitlyConstructed<std::string> string_value_;
     int64 int64_value_;
     int32 int32_value_;
     uint64 uint64_value_;
@@ -549,11 +550,11 @@ class PROTOBUF_EXPORT MapKey {
   void SetType(FieldDescriptor::CppType type) {
     if (type_ == type) return;
     if (type_ == FieldDescriptor::CPPTYPE_STRING) {
-      delete val_.string_value_;
+      val_.string_value_.Destruct();
     }
     type_ = type;
     if (type_ == FieldDescriptor::CPPTYPE_STRING) {
-      val_.string_value_ = new std::string;
+      val_.string_value_.DefaultConstruct();
     }
   }
 

src/google/protobuf/map_test.cc

@@ -3402,6 +3402,21 @@ TEST(ArenaTest, IsInitialized) {
   EXPECT_EQ(0, (*message->mutable_map_int32_int32())[0]);
 }
 
+TEST(ArenaTest, DynamicMapFieldOnArena) {
+  Arena arena;
+  unittest::TestMap message2;
+
+  DynamicMessageFactory factory;
+  Message* message1 =
+      factory.GetPrototype(unittest::TestMap::descriptor())->New(&arena);
+  MapReflectionTester reflection_tester(unittest::TestMap::descriptor());
+  reflection_tester.SetMapFieldsViaReflection(message1);
+  reflection_tester.ExpectMapFieldsSetViaReflection(*message1);
+  reflection_tester.ExpectMapFieldsSetViaReflectionIterator(message1);
+  message2.CopyFrom(*message1);
+  MapTestUtil::ExpectMapFieldsSet(message2);
+}
+
 TEST(MoveTest, MoveConstructorWorks) {
   Map<int32, TestAllTypes> original_map;
   original_map[42].mutable_optional_nested_message()->set_bb(42);

src/google/protobuf/map_type_handler.h

@@ -362,7 +362,7 @@ template <typename Type>
 inline uint8* MapTypeHandler<WireFormatLite::TYPE_MESSAGE, Type>::Write(
     int field, const MapEntryAccessorType& value, uint8* ptr,
     io::EpsCopyOutputStream* stream) {
-  stream->EnsureSpace(&ptr);
+  ptr = stream->EnsureSpace(ptr);
   return WireFormatLite::InternalWriteMessageToArray(field, value, ptr, stream);
 }
 
@@ -371,7 +371,7 @@ inline uint8* MapTypeHandler<WireFormatLite::TYPE_MESSAGE, Type>::Write(
   inline uint8* MapTypeHandler<WireFormatLite::TYPE_##FieldType, Type>::Write( \
       int field, const MapEntryAccessorType& value, uint8* ptr,                \
       io::EpsCopyOutputStream* stream) {                                       \
-    stream->EnsureSpace(&ptr);                                                 \
+    ptr = stream->EnsureSpace(ptr);                                            \
     return stream->Write##DeclaredType(field, value, ptr);                     \
   }
 
@@ -384,7 +384,7 @@ WRITE_METHOD(BYTES, Bytes)
   inline uint8* MapTypeHandler<WireFormatLite::TYPE_##FieldType, Type>::Write( \
       int field, const MapEntryAccessorType& value, uint8* ptr,                \
       io::EpsCopyOutputStream* stream) {                                       \
-    stream->EnsureSpace(&ptr);                                                 \
+    ptr = stream->EnsureSpace(ptr);                                            \
     return WireFormatLite::Write##DeclaredType##ToArray(field, value, ptr);    \
   }
 

src/google/protobuf/message.h

@@ -294,7 +294,11 @@ class PROTOBUF_EXPORT Message : public MessageLite {
 
   std::string GetTypeName() const override;
   void Clear() override;
+
+  // Returns whether all required fields have been set. Note that required
+  // fields no longer exist starting in proto3.
   bool IsInitialized() const override;
+
   void CheckTypeAndMergeFrom(const MessageLite& other) override;
   // Reflective parser
   const char* _InternalParse(const char* ptr,
@@ -326,15 +330,11 @@ class PROTOBUF_EXPORT Message : public MessageLite {
   // which can be used to read and modify the fields of the Message dynamically
   // (in other words, without knowing the message type at compile time).  This
   // object remains property of the Message.
-  //
-  // This method remains virtual in case a subclass does not implement
-  // reflection and wants to override the default behavior.
   const Reflection* GetReflection() const { return GetMetadata().reflection; }
 
  protected:
-  // Get a struct containing the metadata for the Message. Most subclasses only
-  // need to implement this method, rather than the GetDescriptor() and
-  // GetReflection() wrappers.
+  // Get a struct containing the metadata for the Message, which is used in turn
+  // to implement GetDescriptor() and GetReflection() above.
   virtual Metadata GetMetadata() const = 0;
 
 

src/google/protobuf/message_lite.cc

@@ -186,7 +186,7 @@ class ZeroCopyCodedInputStream : public io::ZeroCopyInputStream {
   }
   void BackUp(int count) final { cis_->Advance(-count); }
   bool Skip(int count) final { return cis_->Skip(count); }
-  int64 ByteCount() const final { return 0; }
+  int64_t ByteCount() const final { return 0; }
 
   bool aliasing_enabled() { return cis_->aliasing_enabled_; }
 

src/google/protobuf/message_unittest.inc

@@ -269,7 +269,7 @@ class RepeatedInputStream : public io::ZeroCopyInputStream {
   RepeatedInputStream(const std::string& data, size_t count)
       : data_(data), count_(count), position_(0), total_byte_count_(0) {}
 
-  virtual bool Next(const void** data, int* size) {
+  bool Next(const void** data, int* size) override {
     if (position_ == data_.size()) {
       if (--count_ == 0) {
         return false;
@@ -283,12 +283,12 @@ class RepeatedInputStream : public io::ZeroCopyInputStream {
     return true;
   }
 
-  virtual void BackUp(int count) {
+  void BackUp(int count) override {
     position_ -= static_cast<size_t>(count);
     total_byte_count_ -= count;
   }
 
-  virtual bool Skip(int count) {
+  bool Skip(int count) override {
     while (count > 0) {
       const void* data;
       int size;
@@ -305,9 +305,7 @@ class RepeatedInputStream : public io::ZeroCopyInputStream {
     return false;
   }
 
-  virtual int64 ByteCount() const {
-    return total_byte_count_;
-  }
+  int64_t ByteCount() const override { return total_byte_count_; }
 
  private:
   std::string data_;

src/google/protobuf/no_field_presence_test.cc

@@ -68,7 +68,7 @@ void CheckDefaultValues(
   // default instance.
   EXPECT_EQ(41, m.optional_proto2_message().default_int32());
   EXPECT_EQ(false, m.has_optional_foreign_message());
-  EXPECT_EQ(proto2_nofieldpresence_unittest::TestAllTypes_NestedEnum_FOO,
+  EXPECT_EQ(proto2_nofieldpresence_unittest::TestAllTypes::FOO,
             m.optional_nested_enum());
   EXPECT_EQ(proto2_nofieldpresence_unittest::FOREIGN_FOO,
             m.optional_foreign_enum());
@@ -119,7 +119,7 @@ void FillValues(proto2_nofieldpresence_unittest::TestAllTypes* m) {
   m->mutable_optional_foreign_message()->set_c(43);
   m->mutable_optional_proto2_message()->set_optional_int32(44);
   m->set_optional_nested_enum(
-      proto2_nofieldpresence_unittest::TestAllTypes_NestedEnum_BAZ);
+      proto2_nofieldpresence_unittest::TestAllTypes::BAZ);
   m->set_optional_foreign_enum(proto2_nofieldpresence_unittest::FOREIGN_BAZ);
   m->mutable_optional_lazy_message()->set_bb(45);
   m->add_repeated_int32(100);
@@ -141,7 +141,7 @@ void FillValues(proto2_nofieldpresence_unittest::TestAllTypes* m) {
   m->add_repeated_foreign_message()->set_c(47);
   m->add_repeated_proto2_message()->set_optional_int32(48);
   m->add_repeated_nested_enum(
-      proto2_nofieldpresence_unittest::TestAllTypes_NestedEnum_BAZ);
+      proto2_nofieldpresence_unittest::TestAllTypes::BAZ);
   m->add_repeated_foreign_enum(proto2_nofieldpresence_unittest::FOREIGN_BAZ);
   m->add_repeated_lazy_message()->set_bb(49);
 
@@ -173,7 +173,7 @@ void CheckNonDefaultValues(
   EXPECT_EQ(43, m.optional_foreign_message().c());
   EXPECT_EQ(true, m.has_optional_proto2_message());
   EXPECT_EQ(44, m.optional_proto2_message().optional_int32());
-  EXPECT_EQ(proto2_nofieldpresence_unittest::TestAllTypes_NestedEnum_BAZ,
+  EXPECT_EQ(proto2_nofieldpresence_unittest::TestAllTypes::BAZ,
             m.optional_nested_enum());
   EXPECT_EQ(proto2_nofieldpresence_unittest::FOREIGN_BAZ,
             m.optional_foreign_enum());
@@ -217,7 +217,7 @@ void CheckNonDefaultValues(
   EXPECT_EQ(1, m.repeated_proto2_message_size());
   EXPECT_EQ(48, m.repeated_proto2_message(0).optional_int32());
   EXPECT_EQ(1, m.repeated_nested_enum_size());
-  EXPECT_EQ(proto2_nofieldpresence_unittest::TestAllTypes_NestedEnum_BAZ,
+  EXPECT_EQ(proto2_nofieldpresence_unittest::TestAllTypes::BAZ,
             m.repeated_nested_enum(0));
   EXPECT_EQ(1, m.repeated_foreign_enum_size());
   EXPECT_EQ(proto2_nofieldpresence_unittest::FOREIGN_BAZ,
@@ -442,8 +442,7 @@ TEST(NoFieldPresenceTest, DontSerializeDefaultValuesTest) {
   message.set_optional_string("");
   message.set_optional_bytes("");
   message.set_optional_nested_enum(
-      proto2_nofieldpresence_unittest::
-          TestAllTypes_NestedEnum_FOO);  // first enum entry
+      proto2_nofieldpresence_unittest::TestAllTypes::FOO);  // first enum entry
   message.set_optional_foreign_enum(
       proto2_nofieldpresence_unittest::FOREIGN_FOO);  // first enum entry
 
@@ -557,8 +556,8 @@ TEST(NoFieldPresenceTest, OneofPresence) {
 
   message.Clear();
   message.set_oneof_enum(
-      proto2_nofieldpresence_unittest::TestAllTypes_NestedEnum_FOO);  // default
-                                                                      // value.
+      proto2_nofieldpresence_unittest::TestAllTypes::FOO);  // default
+                                                            // value.
   message.SerializeToString(&serialized);
   EXPECT_EQ(3, serialized.size());
   EXPECT_TRUE(message.ParseFromString(serialized));

src/google/protobuf/proto3_arena_unittest.cc

@@ -52,8 +52,7 @@ void SetAllFields(TestAllTypes* m) {
   m->set_optional_bytes("jkl;");
   m->mutable_optional_nested_message()->set_bb(42);
   m->mutable_optional_foreign_message()->set_c(43);
-  m->set_optional_nested_enum(
-      proto3_arena_unittest::TestAllTypes_NestedEnum_BAZ);
+  m->set_optional_nested_enum(proto3_arena_unittest::TestAllTypes::BAZ);
   m->set_optional_foreign_enum(proto3_arena_unittest::FOREIGN_BAZ);
   m->mutable_optional_lazy_message()->set_bb(45);
   m->add_repeated_int32(100);
@@ -61,8 +60,7 @@ void SetAllFields(TestAllTypes* m) {
   m->add_repeated_bytes("jkl;");
   m->add_repeated_nested_message()->set_bb(46);
   m->add_repeated_foreign_message()->set_c(47);
-  m->add_repeated_nested_enum(
-      proto3_arena_unittest::TestAllTypes_NestedEnum_BAZ);
+  m->add_repeated_nested_enum(proto3_arena_unittest::TestAllTypes::BAZ);
   m->add_repeated_foreign_enum(proto3_arena_unittest::FOREIGN_BAZ);
   m->add_repeated_lazy_message()->set_bb(49);
 
@@ -79,8 +77,7 @@ void ExpectAllFieldsSet(const TestAllTypes& m) {
   EXPECT_EQ(42, m.optional_nested_message().bb());
   EXPECT_EQ(true, m.has_optional_foreign_message());
   EXPECT_EQ(43, m.optional_foreign_message().c());
-  EXPECT_EQ(proto3_arena_unittest::TestAllTypes_NestedEnum_BAZ,
-            m.optional_nested_enum());
+  EXPECT_EQ(proto3_arena_unittest::TestAllTypes::BAZ, m.optional_nested_enum());
   EXPECT_EQ(proto3_arena_unittest::FOREIGN_BAZ, m.optional_foreign_enum());
   EXPECT_EQ(true, m.has_optional_lazy_message());
   EXPECT_EQ(45, m.optional_lazy_message().bb());
@@ -96,7 +93,7 @@ void ExpectAllFieldsSet(const TestAllTypes& m) {
   EXPECT_EQ(1, m.repeated_foreign_message_size());
   EXPECT_EQ(47, m.repeated_foreign_message(0).c());
   EXPECT_EQ(1, m.repeated_nested_enum_size());
-  EXPECT_EQ(proto3_arena_unittest::TestAllTypes_NestedEnum_BAZ,
+  EXPECT_EQ(proto3_arena_unittest::TestAllTypes::BAZ,
             m.repeated_nested_enum(0));
   EXPECT_EQ(1, m.repeated_foreign_enum_size());
   EXPECT_EQ(proto3_arena_unittest::FOREIGN_BAZ, m.repeated_foreign_enum(0));

src/google/protobuf/repeated_field.h

@@ -612,8 +612,6 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase {
     void* elements[1];
   };
   static const size_t kRepHeaderSize = sizeof(Rep) - sizeof(void*);
-  // Contains arena ptr and the elements array. We also keep the invariant that
-  // if rep_ is NULL, then arena is NULL.
   Rep* rep_;
 
   template <typename TypeHandler>

src/google/protobuf/struct.pb.cc

@@ -699,14 +699,14 @@ failure:
 
   // .google.protobuf.NullValue null_value = 1;
   if (_internal_has_null_value()) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteEnumToArray(
       1, this->_internal_null_value(), target);
   }
 
   // double number_value = 2;
   if (_internal_has_number_value()) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteDoubleToArray(2, this->_internal_number_value(), target);
   }
 
@@ -722,13 +722,13 @@ failure:
 
   // bool bool_value = 4;
   if (_internal_has_bool_value()) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(4, this->_internal_bool_value(), target);
   }
 
   // .google.protobuf.Struct struct_value = 5;
   if (_internal_has_struct_value()) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(
         5, _Internal::struct_value(this), target, stream);
@@ -736,7 +736,7 @@ failure:
 
   // .google.protobuf.ListValue list_value = 6;
   if (_internal_has_list_value()) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(
         6, _Internal::list_value(this), target, stream);
@@ -1008,7 +1008,7 @@ failure:
   // repeated .google.protobuf.Value values = 1;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_values_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(1, this->_internal_values(i), target, stream);
   }

src/google/protobuf/text_format.h

@@ -157,8 +157,7 @@ class PROTOBUF_EXPORT TextFormat {
     GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(FastFieldValuePrinter);
   };
 
-  class PROTOBUF_EXPORT PROTOBUF_DEPRECATED_MSG(
-      "Please use FastFieldValuePrinter") FieldValuePrinter {
+  class PROTOBUF_EXPORT FieldValuePrinter {
    public:
     FieldValuePrinter();
     virtual ~FieldValuePrinter();
@@ -286,10 +285,12 @@ class PROTOBUF_EXPORT TextFormat {
     // sequences. This will change the default FastFieldValuePrinter.
     void SetUseUtf8StringEscaping(bool as_utf8);
 
-    // Set the default (Fast)FieldValuePrinter that is used for all fields that
+    // Set the default FastFieldValuePrinter that is used for all fields that
     // don't have a field-specific printer registered.
     // Takes ownership of the printer.
     void SetDefaultFieldValuePrinter(const FastFieldValuePrinter* printer);
+
+    PROTOBUF_DEPRECATED_MSG("Please use FastFieldValuePrinter")
     void SetDefaultFieldValuePrinter(const FieldValuePrinter* printer);
 
     // Sets whether we want to hide unknown fields or not.
@@ -335,16 +336,18 @@ class PROTOBUF_EXPORT TextFormat {
       truncate_string_field_longer_than_ = truncate_string_field_longer_than;
     }
 
-    // Register a custom field-specific (Fast)FieldValuePrinter for fields
+    // Register a custom field-specific FastFieldValuePrinter for fields
     // with a particular FieldDescriptor.
     // Returns "true" if the registration succeeded, or "false", if there is
     // already a printer for that FieldDescriptor.
     // Takes ownership of the printer on successful registration.
-    bool RegisterFieldValuePrinter(const FieldDescriptor* field,
-                                   const FieldValuePrinter* printer);
     bool RegisterFieldValuePrinter(const FieldDescriptor* field,
                                    const FastFieldValuePrinter* printer);
 
+    PROTOBUF_DEPRECATED_MSG("Please use FastFieldValuePrinter")
+    bool RegisterFieldValuePrinter(const FieldDescriptor* field,
+                                   const FieldValuePrinter* printer);
+
     // Register a custom message-specific MessagePrinter for messages with a
     // particular Descriptor.
     // Returns "true" if the registration succeeded, or "false" if there is

src/google/protobuf/timestamp.pb.cc

@@ -203,13 +203,13 @@ failure:
 
   // int64 seconds = 1;
   if (this->seconds() != 0) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(1, this->_internal_seconds(), target);
   }
 
   // int32 nanos = 2;
   if (this->nanos() != 0) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(2, this->_internal_nanos(), target);
   }
 

src/google/protobuf/type.pb.cc

@@ -580,7 +580,7 @@ failure:
   // repeated .google.protobuf.Field fields = 2;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_fields_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(2, this->_internal_fields(i), target, stream);
   }
@@ -598,14 +598,14 @@ failure:
   // repeated .google.protobuf.Option options = 4;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_options_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(4, this->_internal_options(i), target, stream);
   }
 
   // .google.protobuf.SourceContext source_context = 5;
   if (this->has_source_context()) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(
         5, _Internal::source_context(this), target, stream);
@@ -613,7 +613,7 @@ failure:
 
   // .google.protobuf.Syntax syntax = 6;
   if (this->syntax() != 0) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteEnumToArray(
       6, this->_internal_syntax(), target);
   }
@@ -977,21 +977,21 @@ failure:
 
   // .google.protobuf.Field.Kind kind = 1;
   if (this->kind() != 0) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteEnumToArray(
       1, this->_internal_kind(), target);
   }
 
   // .google.protobuf.Field.Cardinality cardinality = 2;
   if (this->cardinality() != 0) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteEnumToArray(
       2, this->_internal_cardinality(), target);
   }
 
   // int32 number = 3;
   if (this->number() != 0) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(3, this->_internal_number(), target);
   }
 
@@ -1017,20 +1017,20 @@ failure:
 
   // int32 oneof_index = 7;
   if (this->oneof_index() != 0) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(7, this->_internal_oneof_index(), target);
   }
 
   // bool packed = 8;
   if (this->packed() != 0) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(8, this->_internal_packed(), target);
   }
 
   // repeated .google.protobuf.Option options = 9;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_options_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(9, this->_internal_options(i), target, stream);
   }
@@ -1452,7 +1452,7 @@ failure:
   // repeated .google.protobuf.EnumValue enumvalue = 2;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_enumvalue_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(2, this->_internal_enumvalue(i), target, stream);
   }
@@ -1460,14 +1460,14 @@ failure:
   // repeated .google.protobuf.Option options = 3;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_options_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(3, this->_internal_options(i), target, stream);
   }
 
   // .google.protobuf.SourceContext source_context = 4;
   if (this->has_source_context()) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(
         4, _Internal::source_context(this), target, stream);
@@ -1475,7 +1475,7 @@ failure:
 
   // .google.protobuf.Syntax syntax = 5;
   if (this->syntax() != 0) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteEnumToArray(
       5, this->_internal_syntax(), target);
   }
@@ -1758,14 +1758,14 @@ failure:
 
   // int32 number = 2;
   if (this->number() != 0) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(2, this->_internal_number(), target);
   }
 
   // repeated .google.protobuf.Option options = 3;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->_internal_options_size()); i < n; i++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(3, this->_internal_options(i), target, stream);
   }
@@ -2048,7 +2048,7 @@ failure:
 
   // .google.protobuf.Any value = 2;
   if (this->has_value()) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
       InternalWriteMessageToArray(
         2, _Internal::value(this), target, stream);

src/google/protobuf/unittest.proto

@@ -441,6 +441,16 @@ message TestEmptyMessageWithExtensions {
   extensions 1 to max;
 }
 
+// Needed for a Python test.
+message TestPickleNestedMessage {
+  message NestedMessage {
+    optional int32 bb = 1;
+    message NestedNestedMessage {
+      optional int32 cc = 1;
+    }
+  }
+}
+
 message TestMultipleExtensionRanges {
   extensions 42;
   extensions 4143 to 4243;

src/google/protobuf/unittest_proto3_arena.proto

@@ -206,3 +206,13 @@ enum ForeignEnum {
 // TestEmptyMessage is used to test behavior of unknown fields.
 message TestEmptyMessage {
 }
+
+// Needed for a Python test.
+message TestPickleNestedMessage {
+  message NestedMessage {
+    int32 bb = 1;
+    message NestedNestedMessage {
+      int32 cc = 1;
+    }
+  }
+}

src/google/protobuf/util/delimited_message_util.h

@@ -34,6 +34,7 @@
 #ifndef GOOGLE_PROTOBUF_UTIL_DELIMITED_MESSAGE_UTIL_H__
 #define GOOGLE_PROTOBUF_UTIL_DELIMITED_MESSAGE_UTIL_H__
 
+
 #include <ostream>
 
 #include <google/protobuf/message_lite.h>

src/google/protobuf/util/internal/datapiece.cc

@@ -43,9 +43,6 @@ namespace protobuf {
 namespace util {
 namespace converter {
 
-;
-;
-;
 using util::Status;
 using util::StatusOr;
 using util::error::Code;

src/google/protobuf/util/internal/default_value_objectwriter.cc

@@ -280,7 +280,7 @@ const google::protobuf::Type* DefaultValueObjectWriter::Node::GetMapValueType(
     if (sub_field.number() != 2) {
       continue;
     }
-    if (sub_field.kind() != google::protobuf::Field_Kind_TYPE_MESSAGE) {
+    if (sub_field.kind() != google::protobuf::Field::TYPE_MESSAGE) {
       // This map's value type is not a message type. We don't need to
       // get the field_type in this case.
       break;
@@ -346,7 +346,7 @@ void DefaultValueObjectWriter::Node::PopulateChildren(
     bool is_map = false;
     NodeKind kind = PRIMITIVE;
 
-    if (field.kind() == google::protobuf::Field_Kind_TYPE_MESSAGE) {
+    if (field.kind() == google::protobuf::Field::TYPE_MESSAGE) {
       kind = OBJECT;
       util::StatusOr<const google::protobuf::Type*> found_result =
           typeinfo->ResolveTypeUrl(field.type_url());
@@ -369,8 +369,7 @@ void DefaultValueObjectWriter::Node::PopulateChildren(
     }
 
     if (!is_map &&
-        field.cardinality() ==
-            google::protobuf::Field_Cardinality_CARDINALITY_REPEATED) {
+        field.cardinality() == google::protobuf::Field::CARDINALITY_REPEATED) {
       kind = LIST;
     }
 
@@ -436,47 +435,47 @@ DataPiece DefaultValueObjectWriter::CreateDefaultDataPieceForField(
     const google::protobuf::Field& field, const TypeInfo* typeinfo,
     bool use_ints_for_enums) {
   switch (field.kind()) {
-    case google::protobuf::Field_Kind_TYPE_DOUBLE: {
+    case google::protobuf::Field::TYPE_DOUBLE: {
       return DataPiece(ConvertTo<double>(
           field.default_value(), &DataPiece::ToDouble, static_cast<double>(0)));
     }
-    case google::protobuf::Field_Kind_TYPE_FLOAT: {
+    case google::protobuf::Field::TYPE_FLOAT: {
       return DataPiece(ConvertTo<float>(
           field.default_value(), &DataPiece::ToFloat, static_cast<float>(0)));
     }
-    case google::protobuf::Field_Kind_TYPE_INT64:
-    case google::protobuf::Field_Kind_TYPE_SINT64:
-    case google::protobuf::Field_Kind_TYPE_SFIXED64: {
+    case google::protobuf::Field::TYPE_INT64:
+    case google::protobuf::Field::TYPE_SINT64:
+    case google::protobuf::Field::TYPE_SFIXED64: {
       return DataPiece(ConvertTo<int64>(
           field.default_value(), &DataPiece::ToInt64, static_cast<int64>(0)));
     }
-    case google::protobuf::Field_Kind_TYPE_UINT64:
-    case google::protobuf::Field_Kind_TYPE_FIXED64: {
+    case google::protobuf::Field::TYPE_UINT64:
+    case google::protobuf::Field::TYPE_FIXED64: {
       return DataPiece(ConvertTo<uint64>(
           field.default_value(), &DataPiece::ToUint64, static_cast<uint64>(0)));
     }
-    case google::protobuf::Field_Kind_TYPE_INT32:
-    case google::protobuf::Field_Kind_TYPE_SINT32:
-    case google::protobuf::Field_Kind_TYPE_SFIXED32: {
+    case google::protobuf::Field::TYPE_INT32:
+    case google::protobuf::Field::TYPE_SINT32:
+    case google::protobuf::Field::TYPE_SFIXED32: {
       return DataPiece(ConvertTo<int32>(
           field.default_value(), &DataPiece::ToInt32, static_cast<int32>(0)));
     }
-    case google::protobuf::Field_Kind_TYPE_BOOL: {
+    case google::protobuf::Field::TYPE_BOOL: {
       return DataPiece(
           ConvertTo<bool>(field.default_value(), &DataPiece::ToBool, false));
     }
-    case google::protobuf::Field_Kind_TYPE_STRING: {
+    case google::protobuf::Field::TYPE_STRING: {
       return DataPiece(field.default_value(), true);
     }
-    case google::protobuf::Field_Kind_TYPE_BYTES: {
+    case google::protobuf::Field::TYPE_BYTES: {
       return DataPiece(field.default_value(), false, true);
     }
-    case google::protobuf::Field_Kind_TYPE_UINT32:
-    case google::protobuf::Field_Kind_TYPE_FIXED32: {
+    case google::protobuf::Field::TYPE_UINT32:
+    case google::protobuf::Field::TYPE_FIXED32: {
       return DataPiece(ConvertTo<uint32>(
           field.default_value(), &DataPiece::ToUint32, static_cast<uint32>(0)));
     }
-    case google::protobuf::Field_Kind_TYPE_ENUM: {
+    case google::protobuf::Field::TYPE_ENUM: {
       return FindEnumDefault(field, typeinfo, use_ints_for_enums);
     }
     default: {

src/google/protobuf/util/internal/json_escaping.cc

@@ -341,6 +341,28 @@ void JsonEscaping::Escape(strings::ByteSource* input,
   }
 }
 
+void JsonEscaping::Escape(StringPiece input, strings::ByteSink* output) {
+  const size_t len = input.length();
+  const char* p = input.data();
+
+  bool can_skip_escaping = true;
+  for (int i = 0; i < len; i++) {
+    char c = p[i];
+    if (c < 0x20 || c >= 0x7F || c == '"' || c == '<' || c == '>' ||
+        c == '\\') {
+      can_skip_escaping = false;
+      break;
+    }
+  }
+
+  if (can_skip_escaping) {
+    output->Append(input.data(), input.length());
+  } else {
+    strings::ArrayByteSource source(input);
+    Escape(&source, output);
+  }
+}
+
 }  // namespace converter
 }  // namespace util
 }  // namespace protobuf

src/google/protobuf/util/internal/json_escaping.h

@@ -79,6 +79,11 @@ class JsonEscaping {
   // Escape the given ByteSource to the given ByteSink.
   static void Escape(strings::ByteSource* input, strings::ByteSink* output);
 
+  // Escape the given ByteSource to the given ByteSink.
+  // This is optimized for the case where the string is all printable 7-bit
+  // ASCII and does not contain a few other characters (such as quotes).
+  static void Escape(StringPiece input, strings::ByteSink* output);
+
  private:
   GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(JsonEscaping);
 };

src/google/protobuf/util/internal/json_objectwriter.cc

@@ -45,8 +45,6 @@ namespace protobuf {
 namespace util {
 namespace converter {
 
-using strings::ArrayByteSource;
-;
 
 JsonObjectWriter::~JsonObjectWriter() {
   if (element_ && !element_->is_root()) {
@@ -101,7 +99,7 @@ JsonObjectWriter* JsonObjectWriter::RenderInt64(StringPiece name,
                                                 int64 value) {
   WritePrefix(name);
   WriteChar('"');
-  stream_->WriteString(StrCat(value));
+  WriteRawString(StrCat(value));
   WriteChar('"');
   return this;
 }
@@ -110,7 +108,7 @@ JsonObjectWriter* JsonObjectWriter::RenderUint64(StringPiece name,
                                                  uint64 value) {
   WritePrefix(name);
   WriteChar('"');
-  stream_->WriteString(StrCat(value));
+  WriteRawString(StrCat(value));
   WriteChar('"');
   return this;
 }
@@ -139,8 +137,7 @@ JsonObjectWriter* JsonObjectWriter::RenderString(StringPiece name,
                                                  StringPiece value) {
   WritePrefix(name);
   WriteChar('"');
-  ArrayByteSource source(value);
-  JsonEscaping::Escape(&source, &sink_);
+  JsonEscaping::Escape(value, &sink_);
   WriteChar('"');
   return this;
 }
@@ -179,10 +176,9 @@ void JsonObjectWriter::WritePrefix(StringPiece name) {
   if (!name.empty() || element()->is_json_object()) {
     WriteChar('"');
     if (!name.empty()) {
-      ArrayByteSource source(name);
-      JsonEscaping::Escape(&source, &sink_);
+      JsonEscaping::Escape(name, &sink_);
     }
-    stream_->WriteString("\":");
+    WriteRawString("\":");
     if (!indent_string_.empty()) WriteChar(' ');
   }
 }

src/google/protobuf/util/internal/json_objectwriter.h

@@ -38,7 +38,9 @@
 #include <google/protobuf/util/internal/structured_objectwriter.h>
 #include <google/protobuf/stubs/bytestream.h>
 
+// clang-format off
 #include <google/protobuf/port_def.inc>
+// clang-format on
 
 namespace google {
 namespace protobuf {
@@ -92,7 +94,22 @@ class PROTOBUF_EXPORT JsonObjectWriter : public StructuredObjectWriter {
         stream_(out),
         sink_(out),
         indent_string_(indent_string),
-        use_websafe_base64_for_bytes_(false) {}
+        indent_char_('\0'),
+        indent_count_(0),
+        use_websafe_base64_for_bytes_(false) {
+    // See if we have a trivial sequence of indent characters.
+    if (!indent_string.empty()) {
+      indent_char_ = indent_string[0];
+      indent_count_ = indent_string.length();
+      for (int i = 1; i < indent_string.length(); i++) {
+        if (indent_char_ != indent_string_[i]) {
+          indent_char_ = '\0';
+          indent_count_ = 0;
+          break;
+        }
+      }
+    }
+  }
   virtual ~JsonObjectWriter();
 
   // ObjectWriter methods.
@@ -169,9 +186,9 @@ class PROTOBUF_EXPORT JsonObjectWriter : public StructuredObjectWriter {
   // methods convert their argument to a string and call this method. This
   // method can then be used to render the simple value without escaping it.
   JsonObjectWriter* RenderSimple(StringPiece name,
-                                 const std::string& value) {
+                                 StringPiece value) {
     WritePrefix(name);
-    stream_->WriteString(value);
+    WriteRawString(value);
     return this;
   }
 
@@ -196,9 +213,25 @@ class PROTOBUF_EXPORT JsonObjectWriter : public StructuredObjectWriter {
   // followed by optional indentation. Otherwise this method is a noop.
   void NewLine() {
     if (!indent_string_.empty()) {
-      WriteChar('\n');
-      for (int i = 0; i < element()->level(); i++) {
-        stream_->WriteString(indent_string_);
+      size_t len = sizeof('\n') + (indent_string_.size() * element()->level());
+
+      // Take the slow-path if we don't have sufficient characters remaining in
+      // our buffer or we have a non-trivial indent string which would prevent
+      // us from using memset.
+      uint8* out = nullptr;
+      if (indent_count_ > 0) {
+        out = stream_->GetDirectBufferForNBytesAndAdvance(len);
+      }
+
+      if (out != nullptr) {
+        out[0] = '\n';
+        memset(&out[1], indent_char_, len - 1);
+      } else {
+        // Slow path, no contiguous output buffer available.
+        WriteChar('\n');
+        for (int i = 0; i < element()->level(); i++) {
+          stream_->WriteRaw(indent_string_.c_str(), indent_string_.length());
+        }
       }
     }
   }
@@ -211,11 +244,20 @@ class PROTOBUF_EXPORT JsonObjectWriter : public StructuredObjectWriter {
   // Writes an individual character to the output.
   void WriteChar(const char c) { stream_->WriteRaw(&c, sizeof(c)); }
 
+  // Writes a string to the output.
+  void WriteRawString(StringPiece s) {
+    stream_->WriteRaw(s.data(), s.length());
+  }
+
   std::unique_ptr<Element> element_;
   io::CodedOutputStream* stream_;
   ByteSinkWrapper sink_;
   const std::string indent_string_;
 
+  // For the common case of indent being a single character repeated.
+  char indent_char_;
+  int indent_count_;
+
   // Whether to use regular or websafe base64 encoding for byte fields. Defaults
   // to regular base64 encoding.
   bool use_websafe_base64_for_bytes_;

src/google/protobuf/util/internal/json_stream_parser.cc

@@ -65,10 +65,10 @@ static const int kUnicodeEscapedLength = 6;
 
 static const int kDefaultMaxRecursionDepth = 100;
 
-// Length of the true, false, and null literals.
-static const int true_len = strlen("true");
-static const int false_len = strlen("false");
-static const int null_len = strlen("null");
+// These cannot be constexpr for portability with VS2015.
+static const StringPiece kKeywordTrue = "true";
+static const StringPiece kKeywordFalse = "false";
+static const StringPiece kKeywordNull = "null";
 
 inline bool IsLetter(char c) {
   return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || (c == '_') ||
@@ -79,6 +79,12 @@ inline bool IsAlphanumeric(char c) {
   return IsLetter(c) || ('0' <= c && c <= '9');
 }
 
+// Indicates a character may not be part of an unquoted key.
+inline bool IsKeySeparator(char c) {
+  return (ascii_isspace(c) || c == '"' || c == '\'' || c == '{' || c == '}' ||
+          c == '[' || c == ']' || c == ':' || c == ',');
+}
+
 static bool ConsumeKey(StringPiece* input, StringPiece* key) {
   if (input->empty() || !IsLetter((*input)[0])) return false;
   int len = 1;
@@ -92,6 +98,21 @@ static bool ConsumeKey(StringPiece* input, StringPiece* key) {
   return true;
 }
 
+// Same as 'ConsumeKey', but allows a widened set of key characters.
+static bool ConsumeKeyPermissive(StringPiece* input,
+                                 StringPiece* key) {
+  if (input->empty() || !IsLetter((*input)[0])) return false;
+  int len = 1;
+  for (; len < input->size(); ++len) {
+    if (IsKeySeparator((*input)[len])) {
+      break;
+    }
+  }
+  *key = StringPiece(input->data(), len);
+  *input = StringPiece(input->data() + len, input->size() - len);
+  return true;
+}
+
 static bool MatchKey(StringPiece input) {
   return !input.empty() && IsLetter(input[0]);
 }
@@ -111,6 +132,7 @@ JsonStreamParser::JsonStreamParser(ObjectWriter* ow)
       chunk_storage_(),
       coerce_to_utf8_(false),
       allow_empty_null_(false),
+      allow_permissive_key_naming_(false),
       loose_float_number_conversion_(false),
       recursion_depth_(0),
       max_recursion_depth_(kDefaultMaxRecursionDepth) {
@@ -292,7 +314,7 @@ util::Status JsonStreamParser::ParseValue(TokenType type) {
       // This handles things like 'fals' being at the end of the string, we
       // don't know if the next char would be e, completing it, or something
       // else, making it invalid.
-      if (!finishing_ && p_.length() < false_len) {
+      if (!finishing_ && p_.length() < kKeywordFalse.length()) {
         return util::Status(util::error::CANCELLED, "");
       }
       return ReportFailure("Unexpected token.");
@@ -659,6 +681,13 @@ util::Status JsonStreamParser::ParseEntry(TokenType type) {
   } else if (type == BEGIN_KEY) {
     // Key is a bare key (back compat), create a StringPiece pointing to it.
     result = ParseKey();
+  } else if (type == BEGIN_NULL || type == BEGIN_TRUE || type == BEGIN_FALSE) {
+    // Key may be a bare key that begins with a reserved word.
+    result = ParseKey();
+    if (result.ok() && (key_ == kKeywordNull || key_ == kKeywordTrue ||
+                        key_ == kKeywordFalse)) {
+      result = ReportFailure("Expected an object key or }.");
+    }
   } else {
     // Unknown key type, report an error.
     result = ReportFailure("Expected an object key or }.");
@@ -740,21 +769,21 @@ util::Status JsonStreamParser::ParseArrayMid(TokenType type) {
 util::Status JsonStreamParser::ParseTrue() {
   ow_->RenderBool(key_, true);
   key_ = StringPiece();
-  p_.remove_prefix(true_len);
+  p_.remove_prefix(kKeywordTrue.length());
   return util::Status();
 }
 
 util::Status JsonStreamParser::ParseFalse() {
   ow_->RenderBool(key_, false);
   key_ = StringPiece();
-  p_.remove_prefix(false_len);
+  p_.remove_prefix(kKeywordFalse.length());
   return util::Status();
 }
 
 util::Status JsonStreamParser::ParseNull() {
   ow_->RenderNull(key_);
   key_ = StringPiece();
-  p_.remove_prefix(null_len);
+  p_.remove_prefix(kKeywordNull.length());
   return util::Status();
 }
 
@@ -821,9 +850,17 @@ void JsonStreamParser::Advance() {
 
 util::Status JsonStreamParser::ParseKey() {
   StringPiece original = p_;
-  if (!ConsumeKey(&p_, &key_)) {
-    return ReportFailure("Invalid key or variable name.");
+
+  if (allow_permissive_key_naming_) {
+    if (!ConsumeKeyPermissive(&p_, &key_)) {
+      return ReportFailure("Invalid key or variable name.");
+    }
+  } else {
+    if (!ConsumeKey(&p_, &key_)) {
+      return ReportFailure("Invalid key or variable name.");
+    }
   }
+
   // If we consumed everything but expect more data, reset p_ and cancel since
   // we can't know if the key was complete or not.
   if (!finishing_ && p_.empty()) {
@@ -847,17 +884,21 @@ JsonStreamParser::TokenType JsonStreamParser::GetNextTokenType() {
   // TODO(sven): Split this method based on context since different contexts
   // support different tokens. Would slightly speed up processing?
   const char* data = p_.data();
+  StringPiece data_view = StringPiece(data, size);
   if (*data == '\"' || *data == '\'') return BEGIN_STRING;
   if (*data == '-' || ('0' <= *data && *data <= '9')) {
     return BEGIN_NUMBER;
   }
-  if (size >= true_len && !strncmp(data, "true", true_len)) {
+  if (size >= kKeywordTrue.length() &&
+      HasPrefixString(data_view, kKeywordTrue)) {
     return BEGIN_TRUE;
   }
-  if (size >= false_len && !strncmp(data, "false", false_len)) {
+  if (size >= kKeywordFalse.length() &&
+      HasPrefixString(data_view, kKeywordFalse)) {
     return BEGIN_FALSE;
   }
-  if (size >= null_len && !strncmp(data, "null", null_len)) {
+  if (size >= kKeywordNull.length() &&
+      HasPrefixString(data_view, kKeywordNull)) {
     return BEGIN_NULL;
   }
   if (*data == '{') return BEGIN_OBJECT;

src/google/protobuf/util/internal/json_stream_parser.h

@@ -273,6 +273,10 @@ class PROTOBUF_EXPORT JsonStreamParser {
   // value.
   bool allow_empty_null_;
 
+  // Whether unquoted object keys can contain embedded non-alphanumeric
+  // characters when this is unambiguous for parsing.
+  bool allow_permissive_key_naming_;
+
   // Whether allows out-of-range floating point numbers or reject them.
   bool loose_float_number_conversion_;
 

src/google/protobuf/util/internal/json_stream_parser_test.cc

@@ -88,9 +88,9 @@ class JsonStreamParserTest : public ::testing::Test {
   virtual ~JsonStreamParserTest() {}
 
   util::Status RunTest(StringPiece json, int split,
-                       bool coerce_utf8 = false, bool allow_empty_null = false,
-                       bool loose_float_number_conversion = false) {
+                       std::function<void(JsonStreamParser*)> setup) {
     JsonStreamParser parser(&mock_);
+    setup(&parser);
 
     // Special case for split == length, test parsing one character at a time.
     if (split == json.length()) {
@@ -122,21 +122,22 @@ class JsonStreamParserTest : public ::testing::Test {
     return result;
   }
 
-  void DoTest(StringPiece json, int split, bool coerce_utf8 = false,
-              bool allow_empty_null = false,
-              bool loose_float_number_conversion = false) {
-    util::Status result = RunTest(json, split, coerce_utf8, allow_empty_null,
-                                  loose_float_number_conversion);
+  void DoTest(
+      StringPiece json, int split,
+      std::function<void(JsonStreamParser*)> setup = [](JsonStreamParser* p) {
+      }) {
+    util::Status result = RunTest(json, split, setup);
     if (!result.ok()) {
       GOOGLE_LOG(WARNING) << result;
     }
     EXPECT_OK(result);
   }
 
-  void DoErrorTest(StringPiece json, int split,
-                   StringPiece error_prefix, bool coerce_utf8 = false,
-                   bool allow_empty_null = false) {
-    util::Status result = RunTest(json, split, coerce_utf8, allow_empty_null);
+  void DoErrorTest(
+      StringPiece json, int split, StringPiece error_prefix,
+      std::function<void(JsonStreamParser*)> setup = [](JsonStreamParser* p) {
+      }) {
+    util::Status result = RunTest(json, split, setup);
     EXPECT_EQ(util::error::INVALID_ARGUMENT, result.code());
     StringPiece error_message(result.error_message());
     EXPECT_EQ(error_prefix, error_message.substr(0, error_prefix.size()));
@@ -324,6 +325,33 @@ TEST_F(JsonStreamParserTest, ObjectKeyTypes) {
   }
 }
 
+TEST_F(JsonStreamParserTest, UnquotedObjectKeyWithReservedPrefxes) {
+  StringPiece str = "{ nullkey: \"a\", truekey: \"b\", falsekey: \"c\"}";
+  for (int i = 0; i <= str.length(); ++i) {
+    ow_.StartObject("")
+        ->RenderString("nullkey", "a")
+        ->RenderString("truekey", "b")
+        ->RenderString("falsekey", "c")
+        ->EndObject();
+    DoTest(str, i);
+  }
+}
+
+TEST_F(JsonStreamParserTest, UnquotedObjectKeyWithReservedKeyword) {
+  StringPiece str = "{ null: \"a\", true: \"b\", false: \"c\"}";
+  for (int i = 0; i <= str.length(); ++i) {
+    DoErrorTest(str, i, "Expected an object key or }.");
+  }
+}
+
+TEST_F(JsonStreamParserTest, UnquotedObjectKeyWithEmbeddedNonAlphanumeric) {
+  StringPiece str = "{ foo-bar-baz: \"a\"}";
+  for (int i = 0; i <= str.length(); ++i) {
+    DoErrorTest(str, i, "Expected : between key:value pair.");
+  }
+}
+
+
 // - array containing primitive values (true, false, null, num, string)
 TEST_F(JsonStreamParserTest, ArrayPrimitiveValues) {
   StringPiece str = "[true, false, null, 'one', \"two\"]";

src/google/protobuf/util/internal/proto_writer.cc

@@ -275,8 +275,7 @@ std::set<const google::protobuf::Field*> GetRequiredFields(
   std::set<const google::protobuf::Field*> required;
   for (int i = 0; i < type.fields_size(); i++) {
     const google::protobuf::Field& field = type.fields(i);
-    if (field.cardinality() ==
-        google::protobuf::Field_Cardinality_CARDINALITY_REQUIRED) {
+    if (field.cardinality() == google::protobuf::Field::CARDINALITY_REQUIRED) {
       required.insert(&field);
     }
   }
@@ -313,8 +312,8 @@ ProtoWriter::ProtoElement::ProtoElement(ProtoWriter::ProtoElement* parent,
       typeinfo_(this->parent()->typeinfo_),
       proto3_(type.syntax() == google::protobuf::SYNTAX_PROTO3),
       type_(type),
-      size_index_(!is_list && field->kind() ==
-                                  google::protobuf::Field_Kind_TYPE_MESSAGE
+      size_index_(!is_list &&
+                          field->kind() == google::protobuf::Field::TYPE_MESSAGE
                       ? ow_->size_insert_.size()
                       : -1),
       array_index_(is_list ? 0 : -1),
@@ -329,7 +328,7 @@ ProtoWriter::ProtoElement::ProtoElement(ProtoWriter::ProtoElement* parent,
       this->parent()->RegisterField(field);
     }
 
-    if (field->kind() == google::protobuf::Field_Kind_TYPE_MESSAGE) {
+    if (field->kind() == google::protobuf::Field::TYPE_MESSAGE) {
       if (!proto3_) {
         required_fields_ = GetRequiredFields(type_);
       }
@@ -381,8 +380,7 @@ ProtoWriter::ProtoElement* ProtoWriter::ProtoElement::pop() {
 void ProtoWriter::ProtoElement::RegisterField(
     const google::protobuf::Field* field) {
   if (!required_fields_.empty() &&
-      field->cardinality() ==
-          google::protobuf::Field_Cardinality_CARDINALITY_REQUIRED) {
+      field->cardinality() == google::protobuf::Field::CARDINALITY_REQUIRED) {
     required_fields_.erase(field);
   }
 }
@@ -570,8 +568,7 @@ bool ProtoWriter::ValidOneof(const google::protobuf::Field& field,
 }
 
 bool ProtoWriter::IsRepeated(const google::protobuf::Field& field) {
-  return field.cardinality() ==
-         google::protobuf::Field_Cardinality_CARDINALITY_REPEATED;
+  return field.cardinality() == google::protobuf::Field::CARDINALITY_REPEATED;
 }
 
 ProtoWriter* ProtoWriter::StartObjectField(const google::protobuf::Field& field,
@@ -617,8 +614,8 @@ ProtoWriter* ProtoWriter::RenderPrimitiveField(
     element_.reset(new ProtoElement(element_.release(), &field, type, false));
   }
 
-  if (field.kind() == google::protobuf::Field_Kind_TYPE_UNKNOWN ||
-      field.kind() == google::protobuf::Field_Kind_TYPE_MESSAGE) {
+  if (field.kind() == google::protobuf::Field::TYPE_UNKNOWN ||
+      field.kind() == google::protobuf::Field::TYPE_MESSAGE) {
     // Push a ProtoElement for location reporting purposes.
     if (element_->proto3()) {
       element_.reset(new ProtoElement(element_.release(), &field, type, false));
@@ -632,67 +629,67 @@ ProtoWriter* ProtoWriter::RenderPrimitiveField(
   }
 
   switch (field.kind()) {
-    case google::protobuf::Field_Kind_TYPE_INT32: {
+    case google::protobuf::Field::TYPE_INT32: {
       status = WriteInt32(field.number(), data, stream_.get());
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_SFIXED32: {
+    case google::protobuf::Field::TYPE_SFIXED32: {
       status = WriteSFixed32(field.number(), data, stream_.get());
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_SINT32: {
+    case google::protobuf::Field::TYPE_SINT32: {
       status = WriteSInt32(field.number(), data, stream_.get());
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_FIXED32: {
+    case google::protobuf::Field::TYPE_FIXED32: {
       status = WriteFixed32(field.number(), data, stream_.get());
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_UINT32: {
+    case google::protobuf::Field::TYPE_UINT32: {
       status = WriteUInt32(field.number(), data, stream_.get());
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_INT64: {
+    case google::protobuf::Field::TYPE_INT64: {
       status = WriteInt64(field.number(), data, stream_.get());
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_SFIXED64: {
+    case google::protobuf::Field::TYPE_SFIXED64: {
       status = WriteSFixed64(field.number(), data, stream_.get());
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_SINT64: {
+    case google::protobuf::Field::TYPE_SINT64: {
       status = WriteSInt64(field.number(), data, stream_.get());
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_FIXED64: {
+    case google::protobuf::Field::TYPE_FIXED64: {
       status = WriteFixed64(field.number(), data, stream_.get());
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_UINT64: {
+    case google::protobuf::Field::TYPE_UINT64: {
       status = WriteUInt64(field.number(), data, stream_.get());
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_DOUBLE: {
+    case google::protobuf::Field::TYPE_DOUBLE: {
       status = WriteDouble(field.number(), data, stream_.get());
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_FLOAT: {
+    case google::protobuf::Field::TYPE_FLOAT: {
       status = WriteFloat(field.number(), data, stream_.get());
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_BOOL: {
+    case google::protobuf::Field::TYPE_BOOL: {
       status = WriteBool(field.number(), data, stream_.get());
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_BYTES: {
+    case google::protobuf::Field::TYPE_BYTES: {
       status = WriteBytes(field.number(), data, stream_.get());
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_STRING: {
+    case google::protobuf::Field::TYPE_STRING: {
       status = WriteString(field.number(), data, stream_.get());
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_ENUM: {
+    case google::protobuf::Field::TYPE_ENUM: {
       status = WriteEnum(
           field.number(), data, typeinfo_->GetEnumByTypeUrl(field.type_url()),
           stream_.get(), use_lower_camel_for_enums_,
@@ -767,8 +764,8 @@ const google::protobuf::Field* ProtoWriter::Lookup(
 
 const google::protobuf::Type* ProtoWriter::LookupType(
     const google::protobuf::Field* field) {
-  return ((field->kind() == google::protobuf::Field_Kind_TYPE_MESSAGE ||
-           field->kind() == google::protobuf::Field_Kind_TYPE_GROUP)
+  return ((field->kind() == google::protobuf::Field::TYPE_MESSAGE ||
+           field->kind() == google::protobuf::Field::TYPE_GROUP)
               ? typeinfo_->GetTypeByTypeUrl(field->type_url())
               : &element_->type());
 }

src/google/protobuf/util/internal/protostream_objectsource.cc

@@ -91,20 +91,20 @@ const std::string FormatNanos(uint32 nanos, bool with_trailing_zeros);
 StatusOr<std::string> MapKeyDefaultValueAsString(
     const google::protobuf::Field& field) {
   switch (field.kind()) {
-    case google::protobuf::Field_Kind_TYPE_BOOL:
+    case google::protobuf::Field::TYPE_BOOL:
       return std::string("false");
-    case google::protobuf::Field_Kind_TYPE_INT32:
-    case google::protobuf::Field_Kind_TYPE_INT64:
-    case google::protobuf::Field_Kind_TYPE_UINT32:
-    case google::protobuf::Field_Kind_TYPE_UINT64:
-    case google::protobuf::Field_Kind_TYPE_SINT32:
-    case google::protobuf::Field_Kind_TYPE_SINT64:
-    case google::protobuf::Field_Kind_TYPE_SFIXED32:
-    case google::protobuf::Field_Kind_TYPE_SFIXED64:
-    case google::protobuf::Field_Kind_TYPE_FIXED32:
-    case google::protobuf::Field_Kind_TYPE_FIXED64:
+    case google::protobuf::Field::TYPE_INT32:
+    case google::protobuf::Field::TYPE_INT64:
+    case google::protobuf::Field::TYPE_UINT32:
+    case google::protobuf::Field::TYPE_UINT64:
+    case google::protobuf::Field::TYPE_SINT32:
+    case google::protobuf::Field::TYPE_SINT64:
+    case google::protobuf::Field::TYPE_SFIXED32:
+    case google::protobuf::Field::TYPE_SFIXED64:
+    case google::protobuf::Field::TYPE_FIXED32:
+    case google::protobuf::Field::TYPE_FIXED64:
       return std::string("0");
-    case google::protobuf::Field_Kind_TYPE_STRING:
+    case google::protobuf::Field::TYPE_STRING:
       return std::string();
     default:
       return Status(util::error::INTERNAL, "Invalid map key type.");
@@ -227,8 +227,7 @@ Status ProtoStreamObjectSource::WriteMessage(const google::protobuf::Type& type,
       continue;
     }
 
-    if (field->cardinality() ==
-        google::protobuf::Field_Cardinality_CARDINALITY_REPEATED) {
+    if (field->cardinality() == google::protobuf::Field::CARDINALITY_REPEATED) {
       if (IsMap(*field)) {
         ow->StartObject(field_name);
         ASSIGN_OR_RETURN(tag, RenderMap(field, field_name, tag, ow));
@@ -763,7 +762,7 @@ Status ProtoStreamObjectSource::RenderField(
   // Short-circuit message types as it tends to call WriteMessage recursively
   // and ends up using a lot of stack space. Keep the stack usage of this
   // message small in order to preserve stack space and not crash.
-  if (field->kind() == google::protobuf::Field_Kind_TYPE_MESSAGE) {
+  if (field->kind() == google::protobuf::Field::TYPE_MESSAGE) {
     uint32 buffer32;
     stream_->ReadVarint32(&buffer32);  // message length
     int old_limit = stream_->PushLimit(buffer32);
@@ -809,72 +808,72 @@ Status ProtoStreamObjectSource::RenderNonMessageField(
   uint64 buffer64;
   std::string strbuffer;
   switch (field->kind()) {
-    case google::protobuf::Field_Kind_TYPE_BOOL: {
+    case google::protobuf::Field::TYPE_BOOL: {
       stream_->ReadVarint64(&buffer64);
       ow->RenderBool(field_name, buffer64 != 0);
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_INT32: {
+    case google::protobuf::Field::TYPE_INT32: {
       stream_->ReadVarint32(&buffer32);
       ow->RenderInt32(field_name, bit_cast<int32>(buffer32));
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_INT64: {
+    case google::protobuf::Field::TYPE_INT64: {
       stream_->ReadVarint64(&buffer64);
       ow->RenderInt64(field_name, bit_cast<int64>(buffer64));
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_UINT32: {
+    case google::protobuf::Field::TYPE_UINT32: {
       stream_->ReadVarint32(&buffer32);
       ow->RenderUint32(field_name, bit_cast<uint32>(buffer32));
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_UINT64: {
+    case google::protobuf::Field::TYPE_UINT64: {
       stream_->ReadVarint64(&buffer64);
       ow->RenderUint64(field_name, bit_cast<uint64>(buffer64));
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_SINT32: {
+    case google::protobuf::Field::TYPE_SINT32: {
       stream_->ReadVarint32(&buffer32);
       ow->RenderInt32(field_name, WireFormatLite::ZigZagDecode32(buffer32));
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_SINT64: {
+    case google::protobuf::Field::TYPE_SINT64: {
       stream_->ReadVarint64(&buffer64);
       ow->RenderInt64(field_name, WireFormatLite::ZigZagDecode64(buffer64));
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_SFIXED32: {
+    case google::protobuf::Field::TYPE_SFIXED32: {
       stream_->ReadLittleEndian32(&buffer32);
       ow->RenderInt32(field_name, bit_cast<int32>(buffer32));
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_SFIXED64: {
+    case google::protobuf::Field::TYPE_SFIXED64: {
       stream_->ReadLittleEndian64(&buffer64);
       ow->RenderInt64(field_name, bit_cast<int64>(buffer64));
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_FIXED32: {
+    case google::protobuf::Field::TYPE_FIXED32: {
       stream_->ReadLittleEndian32(&buffer32);
       ow->RenderUint32(field_name, bit_cast<uint32>(buffer32));
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_FIXED64: {
+    case google::protobuf::Field::TYPE_FIXED64: {
       stream_->ReadLittleEndian64(&buffer64);
       ow->RenderUint64(field_name, bit_cast<uint64>(buffer64));
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_FLOAT: {
+    case google::protobuf::Field::TYPE_FLOAT: {
       stream_->ReadLittleEndian32(&buffer32);
       ow->RenderFloat(field_name, bit_cast<float>(buffer32));
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_DOUBLE: {
+    case google::protobuf::Field::TYPE_DOUBLE: {
       stream_->ReadLittleEndian64(&buffer64);
       ow->RenderDouble(field_name, bit_cast<double>(buffer64));
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_ENUM: {
+    case google::protobuf::Field::TYPE_ENUM: {
       stream_->ReadVarint32(&buffer32);
 
       // If the field represents an explicit NULL value, render null.
@@ -910,13 +909,13 @@ Status ProtoStreamObjectSource::RenderNonMessageField(
       }
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_STRING: {
+    case google::protobuf::Field::TYPE_STRING: {
       stream_->ReadVarint32(&buffer32);  // string size.
       stream_->ReadString(&strbuffer, buffer32);
       ow->RenderString(field_name, strbuffer);
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_BYTES: {
+    case google::protobuf::Field::TYPE_BYTES: {
       stream_->ReadVarint32(&buffer32);  // bytes size.
       stream_->ReadString(&strbuffer, buffer32);
       ow->RenderBytes(field_name, strbuffer);
@@ -933,85 +932,85 @@ const std::string ProtoStreamObjectSource::ReadFieldValueAsString(
     const google::protobuf::Field& field) const {
   std::string result;
   switch (field.kind()) {
-    case google::protobuf::Field_Kind_TYPE_BOOL: {
+    case google::protobuf::Field::TYPE_BOOL: {
       uint64 buffer64;
       stream_->ReadVarint64(&buffer64);
       result = buffer64 != 0 ? "true" : "false";
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_INT32: {
+    case google::protobuf::Field::TYPE_INT32: {
       uint32 buffer32;
       stream_->ReadVarint32(&buffer32);
       result = StrCat(bit_cast<int32>(buffer32));
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_INT64: {
+    case google::protobuf::Field::TYPE_INT64: {
       uint64 buffer64;
       stream_->ReadVarint64(&buffer64);
       result = StrCat(bit_cast<int64>(buffer64));
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_UINT32: {
+    case google::protobuf::Field::TYPE_UINT32: {
       uint32 buffer32;
       stream_->ReadVarint32(&buffer32);
       result = StrCat(bit_cast<uint32>(buffer32));
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_UINT64: {
+    case google::protobuf::Field::TYPE_UINT64: {
       uint64 buffer64;
       stream_->ReadVarint64(&buffer64);
       result = StrCat(bit_cast<uint64>(buffer64));
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_SINT32: {
+    case google::protobuf::Field::TYPE_SINT32: {
       uint32 buffer32;
       stream_->ReadVarint32(&buffer32);
       result = StrCat(WireFormatLite::ZigZagDecode32(buffer32));
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_SINT64: {
+    case google::protobuf::Field::TYPE_SINT64: {
       uint64 buffer64;
       stream_->ReadVarint64(&buffer64);
       result = StrCat(WireFormatLite::ZigZagDecode64(buffer64));
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_SFIXED32: {
+    case google::protobuf::Field::TYPE_SFIXED32: {
       uint32 buffer32;
       stream_->ReadLittleEndian32(&buffer32);
       result = StrCat(bit_cast<int32>(buffer32));
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_SFIXED64: {
+    case google::protobuf::Field::TYPE_SFIXED64: {
       uint64 buffer64;
       stream_->ReadLittleEndian64(&buffer64);
       result = StrCat(bit_cast<int64>(buffer64));
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_FIXED32: {
+    case google::protobuf::Field::TYPE_FIXED32: {
       uint32 buffer32;
       stream_->ReadLittleEndian32(&buffer32);
       result = StrCat(bit_cast<uint32>(buffer32));
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_FIXED64: {
+    case google::protobuf::Field::TYPE_FIXED64: {
       uint64 buffer64;
       stream_->ReadLittleEndian64(&buffer64);
       result = StrCat(bit_cast<uint64>(buffer64));
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_FLOAT: {
+    case google::protobuf::Field::TYPE_FLOAT: {
       uint32 buffer32;
       stream_->ReadLittleEndian32(&buffer32);
       result = SimpleFtoa(bit_cast<float>(buffer32));
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_DOUBLE: {
+    case google::protobuf::Field::TYPE_DOUBLE: {
       uint64 buffer64;
       stream_->ReadLittleEndian64(&buffer64);
       result = SimpleDtoa(bit_cast<double>(buffer64));
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_ENUM: {
+    case google::protobuf::Field::TYPE_ENUM: {
       uint32 buffer32;
       stream_->ReadVarint32(&buffer32);
       // Get the nested enum type for this field.
@@ -1029,13 +1028,13 @@ const std::string ProtoStreamObjectSource::ReadFieldValueAsString(
       }
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_STRING: {
+    case google::protobuf::Field::TYPE_STRING: {
       uint32 buffer32;
       stream_->ReadVarint32(&buffer32);  // string size.
       stream_->ReadString(&result, buffer32);
       break;
     }
-    case google::protobuf::Field_Kind_TYPE_BYTES: {
+    case google::protobuf::Field::TYPE_BYTES: {
       uint32 buffer32;
       stream_->ReadVarint32(&buffer32);  // bytes size.
       stream_->ReadString(&result, buffer32);
@@ -1053,7 +1052,7 @@ bool ProtoStreamObjectSource::IsMap(
     const google::protobuf::Field& field) const {
   const google::protobuf::Type* field_type =
       typeinfo_->GetTypeByTypeUrl(field.type_url());
-  return field.kind() == google::protobuf::Field_Kind_TYPE_MESSAGE &&
+  return field.kind() == google::protobuf::Field::TYPE_MESSAGE &&
          util::converter::IsMap(field, *field_type);
 }
 
@@ -1112,8 +1111,7 @@ const google::protobuf::Field* FindFieldByNumber(
 // TODO(skarvaje): Replace FieldDescriptor by implementing IsTypePackable()
 // using tech Field.
 bool IsPackable(const google::protobuf::Field& field) {
-  return field.cardinality() ==
-             google::protobuf::Field_Cardinality_CARDINALITY_REPEATED &&
+  return field.cardinality() == google::protobuf::Field::CARDINALITY_REPEATED &&
          FieldDescriptor::IsTypePackable(
              static_cast<FieldDescriptor::Type>(field.kind()));
 }

src/google/protobuf/util/internal/protostream_objectwriter.cc

@@ -1315,9 +1315,8 @@ void ProtoStreamObjectWriter::PopOneElement() {
 
 bool ProtoStreamObjectWriter::IsMap(const google::protobuf::Field& field) {
   if (field.type_url().empty() ||
-      field.kind() != google::protobuf::Field_Kind_TYPE_MESSAGE ||
-      field.cardinality() !=
-          google::protobuf::Field_Cardinality_CARDINALITY_REPEATED) {
+      field.kind() != google::protobuf::Field::TYPE_MESSAGE ||
+      field.cardinality() != google::protobuf::Field::CARDINALITY_REPEATED) {
     return false;
   }
   const google::protobuf::Type* field_type =

src/google/protobuf/util/internal/protostream_objectwriter_test.cc

@@ -281,7 +281,7 @@ TEST_P(ProtoStreamObjectWriterTest, ConflictingJsonName) {
 TEST_P(ProtoStreamObjectWriterTest, IntEnumValuesAreAccepted) {
   Book book;
   book.set_title("Some Book");
-  book.set_type(proto_util_converter::testing::Book_Type_KIDS);
+  book.set_type(proto_util_converter::testing::Book::KIDS);
   Author* robert = book.mutable_author();
   robert->set_name("robert");
 
@@ -317,7 +317,7 @@ TEST_P(ProtoStreamObjectWriterTest, EnumValuesWithDifferentCaseIsRejected) {
 TEST_P(ProtoStreamObjectWriterTest, EnumValuesWithSameCaseIsAccepted) {
   Book book;
   book.set_title("Some Book");
-  book.set_type(proto_util_converter::testing::Book_Type_ACTION_AND_ADVENTURE);
+  book.set_type(proto_util_converter::testing::Book::ACTION_AND_ADVENTURE);
   Author* robert = book.mutable_author();
   robert->set_name("robert");
 
@@ -337,7 +337,7 @@ TEST_P(ProtoStreamObjectWriterTest, EnumValuesWithSameCaseIsAccepted) {
 TEST_P(ProtoStreamObjectWriterTest, EnumValuesWithDifferentCaseIsAccepted) {
   Book book;
   book.set_title("Some Book");
-  book.set_type(proto_util_converter::testing::Book_Type_ACTION_AND_ADVENTURE);
+  book.set_type(proto_util_converter::testing::Book::ACTION_AND_ADVENTURE);
   Author* robert = book.mutable_author();
   robert->set_name("robert");
 
@@ -357,7 +357,7 @@ TEST_P(ProtoStreamObjectWriterTest, EnumValuesWithDifferentCaseIsAccepted) {
 TEST_P(ProtoStreamObjectWriterTest, EnumValuesWithoutUnderscoreAreAccepted) {
   Book book;
   book.set_title("Some Book");
-  book.set_type(proto_util_converter::testing::Book_Type_ACTION_AND_ADVENTURE);
+  book.set_type(proto_util_converter::testing::Book::ACTION_AND_ADVENTURE);
   Author* robert = book.mutable_author();
   robert->set_name("robert");
 
@@ -377,7 +377,7 @@ TEST_P(ProtoStreamObjectWriterTest, EnumValuesWithoutUnderscoreAreAccepted) {
 TEST_P(ProtoStreamObjectWriterTest, EnumValuesInCamelCaseAreAccepted) {
   Book book;
   book.set_title("Some Book");
-  book.set_type(proto_util_converter::testing::Book_Type_ACTION_AND_ADVENTURE);
+  book.set_type(proto_util_converter::testing::Book::ACTION_AND_ADVENTURE);
   Author* robert = book.mutable_author();
   robert->set_name("robert");
 
@@ -398,7 +398,7 @@ TEST_P(ProtoStreamObjectWriterTest,
        EnumValuesInCamelCaseRemoveDashAndUnderscoreAreAccepted) {
   Book book;
   book.set_title("Some Book");
-  book.set_type(proto_util_converter::testing::Book_Type_ACTION_AND_ADVENTURE);
+  book.set_type(proto_util_converter::testing::Book::ACTION_AND_ADVENTURE);
   Author* robert = book.mutable_author();
   robert->set_name("robert");
 
@@ -420,7 +420,7 @@ TEST_P(ProtoStreamObjectWriterTest,
        EnumValuesInCamelCaseWithNameNotUppercaseAreAccepted) {
   Book book;
   book.set_title("Some Book");
-  book.set_type(proto_util_converter::testing::Book_Type_arts_and_photography);
+  book.set_type(proto_util_converter::testing::Book::arts_and_photography);
   Author* robert = book.mutable_author();
   robert->set_name("robert");
 

src/google/protobuf/util/internal/utility.cc

@@ -43,7 +43,9 @@
 #include <google/protobuf/stubs/map_util.h>
 #include <google/protobuf/stubs/mathlimits.h>
 
+// clang-format off
 #include <google/protobuf/port_def.inc>
+// clang-format on
 
 namespace google {
 namespace protobuf {
@@ -52,7 +54,7 @@ namespace converter {
 
 bool GetBoolOptionOrDefault(
     const RepeatedPtrField<google::protobuf::Option>& options,
-    const std::string& option_name, bool default_value) {
+    StringPiece option_name, bool default_value) {
   const google::protobuf::Option* opt = FindOptionOrNull(options, option_name);
   if (opt == nullptr) {
     return default_value;
@@ -62,7 +64,7 @@ bool GetBoolOptionOrDefault(
 
 int64 GetInt64OptionOrDefault(
     const RepeatedPtrField<google::protobuf::Option>& options,
-    const std::string& option_name, int64 default_value) {
+    StringPiece option_name, int64 default_value) {
   const google::protobuf::Option* opt = FindOptionOrNull(options, option_name);
   if (opt == nullptr) {
     return default_value;
@@ -72,7 +74,7 @@ int64 GetInt64OptionOrDefault(
 
 double GetDoubleOptionOrDefault(
     const RepeatedPtrField<google::protobuf::Option>& options,
-    const std::string& option_name, double default_value) {
+    StringPiece option_name, double default_value) {
   const google::protobuf::Option* opt = FindOptionOrNull(options, option_name);
   if (opt == nullptr) {
     return default_value;
@@ -82,10 +84,10 @@ double GetDoubleOptionOrDefault(
 
 std::string GetStringOptionOrDefault(
     const RepeatedPtrField<google::protobuf::Option>& options,
-    const std::string& option_name, const std::string& default_value) {
+    StringPiece option_name, StringPiece default_value) {
   const google::protobuf::Option* opt = FindOptionOrNull(options, option_name);
   if (opt == nullptr) {
-    return default_value;
+    return std::string(default_value);
   }
   return GetStringFromAny(opt->value());
 }
@@ -139,7 +141,7 @@ const std::string GetFullTypeWithUrl(StringPiece simple_type) {
 
 const google::protobuf::Option* FindOptionOrNull(
     const RepeatedPtrField<google::protobuf::Option>& options,
-    const std::string& option_name) {
+    StringPiece option_name) {
   for (int i = 0; i < options.size(); ++i) {
     const google::protobuf::Option& opt = options.Get(i);
     if (opt.name() == option_name) {
@@ -240,14 +242,14 @@ const google::protobuf::EnumValue* FindEnumValueByNameWithoutUnderscoreOrNull(
   return nullptr;
 }
 
-std::string EnumValueNameToLowerCamelCase(const StringPiece input) {
+std::string EnumValueNameToLowerCamelCase(StringPiece input) {
   std::string input_string(input);
   std::transform(input_string.begin(), input_string.end(), input_string.begin(),
                  ::tolower);
   return ToCamelCase(input_string);
 }
 
-std::string ToCamelCase(const StringPiece input) {
+std::string ToCamelCase(StringPiece input) {
   bool capitalize_next = false;
   bool was_cap = true;
   bool is_cap = false;
@@ -352,15 +354,14 @@ bool IsWellKnownType(const std::string& type_name) {
   return ContainsKey(*well_known_types_, type_name);
 }
 
-bool IsValidBoolString(const std::string& bool_string) {
+bool IsValidBoolString(StringPiece bool_string) {
   return bool_string == "true" || bool_string == "false" ||
          bool_string == "1" || bool_string == "0";
 }
 
 bool IsMap(const google::protobuf::Field& field,
            const google::protobuf::Type& type) {
-  return field.cardinality() ==
-             google::protobuf::Field_Cardinality_CARDINALITY_REPEATED &&
+  return field.cardinality() == google::protobuf::Field::CARDINALITY_REPEATED &&
          (GetBoolOptionOrDefault(type.options(), "map_entry", false) ||
           GetBoolOptionOrDefault(type.options(),
                                  "google.protobuf.MessageOptions.map_entry",

src/google/protobuf/util/internal/utility.h

@@ -60,25 +60,25 @@ static const int64 kTypeUrlSize = 19;
 // When the option with the given name is not found, default_value is returned.
 PROTOBUF_EXPORT bool GetBoolOptionOrDefault(
     const RepeatedPtrField<google::protobuf::Option>& options,
-    const std::string& option_name, bool default_value);
+    StringPiece option_name, bool default_value);
 
 // Returns int64 option value. If the option isn't found, returns the
 // default_value.
 PROTOBUF_EXPORT int64 GetInt64OptionOrDefault(
     const RepeatedPtrField<google::protobuf::Option>& options,
-    const std::string& option_name, int64 default_value);
+    StringPiece option_name, int64 default_value);
 
 // Returns double option value. If the option isn't found, returns the
 // default_value.
 PROTOBUF_EXPORT double GetDoubleOptionOrDefault(
     const RepeatedPtrField<google::protobuf::Option>& options,
-    const std::string& option_name, double default_value);
+    StringPiece option_name, double default_value);
 
 // Returns string option value. If the option isn't found, returns the
 // default_value.
 PROTOBUF_EXPORT std::string GetStringOptionOrDefault(
     const RepeatedPtrField<google::protobuf::Option>& options,
-    const std::string& option_name, const std::string& default_value);
+    StringPiece option_name, StringPiece default_value);
 
 // Returns a boolean value contained in Any type.
 // TODO(skarvaje): Make these utilities dealing with Any types more generic,
@@ -113,7 +113,7 @@ PROTOBUF_EXPORT const std::string GetFullTypeWithUrl(
 // provided map. Returns nullptr if none found.
 const google::protobuf::Option* FindOptionOrNull(
     const RepeatedPtrField<google::protobuf::Option>& options,
-    const std::string& option_name);
+    StringPiece option_name);
 
 // Finds and returns the field identified by field_name in the passed tech Type
 // object. Returns nullptr if none found.
@@ -160,7 +160,7 @@ PROTOBUF_EXPORT bool IsWellKnownType(const std::string& type_name);
 
 // Returns true if 'bool_string' represents a valid boolean value. Only "true",
 // "false", "0" and "1" are allowed.
-PROTOBUF_EXPORT bool IsValidBoolString(const std::string& bool_string);
+PROTOBUF_EXPORT bool IsValidBoolString(StringPiece bool_string);
 
 // Returns true if "field" is a protobuf map field based on its type.
 PROTOBUF_EXPORT bool IsMap(const google::protobuf::Field& field,

src/google/protobuf/util/json_util_test.cc

@@ -499,7 +499,7 @@ class SegmentedZeroCopyOutputStream : public io::ZeroCopyOutputStream {
         last_segment_(static_cast<char*>(NULL), 0),
         byte_count_(0) {}
 
-  virtual bool Next(void** buffer, int* length) {
+  bool Next(void** buffer, int* length) override {
     if (segments_.empty()) {
       return false;
     }
@@ -511,7 +511,7 @@ class SegmentedZeroCopyOutputStream : public io::ZeroCopyOutputStream {
     return true;
   }
 
-  virtual void BackUp(int length) {
+  void BackUp(int length) override {
     GOOGLE_CHECK(length <= last_segment_.second);
     segments_.push_front(
         Segment(last_segment_.first + last_segment_.second - length, length));
@@ -519,7 +519,7 @@ class SegmentedZeroCopyOutputStream : public io::ZeroCopyOutputStream {
     byte_count_ -= length;
   }
 
-  virtual int64 ByteCount() const { return byte_count_; }
+  int64_t ByteCount() const override { return byte_count_; }
 
  private:
   std::list<Segment> segments_;

src/google/protobuf/util/message_differencer_unittest.cc

@@ -659,32 +659,32 @@ TEST(MessageDifferencerTest, WithinDefaultFractionOrMarginDoubleTest) {
   // Compare
   EXPECT_FALSE(differencer.Compare(msg1, msg2));
 
-  // Set up a custom field comparitor, with a default fraction and margin for
+  // Set up a custom field comparator, with a default fraction and margin for
   // float and double comparison.
-  util::DefaultFieldComparator field_comparitor;
-  field_comparitor.SetDefaultFractionAndMargin(0.0, 10.0);
-  differencer.set_field_comparator(&field_comparitor);
+  util::DefaultFieldComparator field_comparator;
+  field_comparator.SetDefaultFractionAndMargin(0.0, 10.0);
+  differencer.set_field_comparator(&field_comparator);
 
   // Set comparison to exact, margin and fraction value should not matter.
-  field_comparitor.set_float_comparison(util::DefaultFieldComparator::EXACT);
+  field_comparator.set_float_comparison(util::DefaultFieldComparator::EXACT);
   EXPECT_FALSE(differencer.Compare(msg1, msg2));
 
   // Margin and fraction comparison is activated when float comparison is
   // set to approximate.
-  field_comparitor.set_float_comparison(
+  field_comparator.set_float_comparison(
       util::DefaultFieldComparator::APPROXIMATE);
   EXPECT_TRUE(differencer.Compare(msg1, msg2));
 
   // Test out comparison with fraction.
-  field_comparitor.SetDefaultFractionAndMargin(0.2, 0.0);
+  field_comparator.SetDefaultFractionAndMargin(0.2, 0.0);
   EXPECT_TRUE(differencer.Compare(msg1, msg2));
 
   // Should fail since the fraction is smaller than error.
-  field_comparitor.SetDefaultFractionAndMargin(0.01, 0.0);
+  field_comparator.SetDefaultFractionAndMargin(0.01, 0.0);
   EXPECT_FALSE(differencer.Compare(msg1, msg2));
 
   // Should pass if either fraction or margin are satisfied.
-  field_comparitor.SetDefaultFractionAndMargin(0.01, 10.0);
+  field_comparator.SetDefaultFractionAndMargin(0.01, 10.0);
   EXPECT_TRUE(differencer.Compare(msg1, msg2));
 
   // Make sure that the default margin and fraction affects all fields

src/google/protobuf/wire_format.cc

@@ -195,7 +195,7 @@ uint8* WireFormat::InternalSerializeUnknownFieldsToArray(
   for (int i = 0; i < unknown_fields.field_count(); i++) {
     const UnknownField& field = unknown_fields.field(i);
 
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     switch (field.type()) {
       case UnknownField::TYPE_VARINT:
         target = WireFormatLite::WriteUInt64ToArray(field.number(),
@@ -218,7 +218,7 @@ uint8* WireFormat::InternalSerializeUnknownFieldsToArray(
             field.number(), WireFormatLite::WIRETYPE_START_GROUP, target);
         target = InternalSerializeUnknownFieldsToArray(field.group(), target,
                                                        stream);
-        stream->EnsureSpace(&target);
+        target = stream->EnsureSpace(target);
         target = WireFormatLite::WriteTagToArray(
             field.number(), WireFormatLite::WIRETYPE_END_GROUP, target);
         break;
@@ -236,7 +236,7 @@ uint8* WireFormat::InternalSerializeUnknownMessageSetItemsToArray(
     // The only unknown fields that are allowed to exist in a MessageSet are
     // messages, which are length-delimited.
     if (field.type() == UnknownField::TYPE_LENGTH_DELIMITED) {
-      stream->EnsureSpace(&target);
+      target = stream->EnsureSpace(target);
       // Start group.
       target = io::CodedOutputStream::WriteTagToArray(
           WireFormatLite::kMessageSetItemStartTag, target);
@@ -253,7 +253,7 @@ uint8* WireFormat::InternalSerializeUnknownMessageSetItemsToArray(
 
       target = field.InternalSerializeLengthDelimitedNoTag(target, stream);
 
-      stream->EnsureSpace(&target);
+      target = stream->EnsureSpace(target);
       // End group.
       target = io::CodedOutputStream::WriteTagToArray(
           WireFormatLite::kMessageSetItemEndTag, target);
@@ -687,7 +687,7 @@ uint8* WireFormat::InternalSerializeWithCachedSizesToArray(
 static uint8* SerializeMapKeyWithCachedSizes(const FieldDescriptor* field,
                                              const MapKey& value, uint8* target,
                                              io::EpsCopyOutputStream* stream) {
-  stream->EnsureSpace(&target);
+  target = stream->EnsureSpace(target);
   switch (field->type()) {
     case FieldDescriptor::TYPE_DOUBLE:
     case FieldDescriptor::TYPE_FLOAT:
@@ -724,7 +724,7 @@ static uint8* SerializeMapKeyWithCachedSizes(const FieldDescriptor* field,
 static uint8* SerializeMapValueRefWithCachedSizes(
     const FieldDescriptor* field, const MapValueRef& value, uint8* target,
     io::EpsCopyOutputStream* stream) {
-  stream->EnsureSpace(&target);
+  target = stream->EnsureSpace(target);
   switch (field->type()) {
 #define CASE_TYPE(FieldType, CamelFieldType, CamelCppType)   \
   case FieldDescriptor::TYPE_##FieldType:                    \
@@ -815,7 +815,7 @@ static uint8* InternalSerializeMapEntry(const FieldDescriptor* field,
   size_t size = kMapEntryTagByteSize;
   size += MapKeyDataOnlyByteSize(key_field, key);
   size += MapValueRefDataOnlyByteSize(value_field, value);
-  stream->EnsureSpace(&target);
+  target = stream->EnsureSpace(target);
   target = WireFormatLite::WriteTagToArray(
       field->number(), WireFormatLite::WIRETYPE_LENGTH_DELIMITED, target);
   target = io::CodedOutputStream::WriteVarint32ToArray(size, target);
@@ -900,7 +900,7 @@ uint8* WireFormat::InternalSerializeField(const FieldDescriptor* field,
 
   if (field->is_packed()) {
     if (count == 0) return target;
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     switch (field->type()) {
 #define HANDLE_PRIMITIVE_TYPE(TYPE, CPPTYPE, TYPE_METHOD, CPPTYPE_METHOD)   \
   case FieldDescriptor::TYPE_##TYPE: {                                      \
@@ -943,7 +943,7 @@ uint8* WireFormat::InternalSerializeField(const FieldDescriptor* field,
   }
 
   for (int j = 0; j < count; j++) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     switch (field->type()) {
 #define HANDLE_PRIMITIVE_TYPE(TYPE, CPPTYPE, TYPE_METHOD, CPPTYPE_METHOD)     \
   case FieldDescriptor::TYPE_##TYPE: {                                        \
@@ -1046,7 +1046,7 @@ uint8* WireFormat::InternalSerializeMessageSetItem(
     io::EpsCopyOutputStream* stream) {
   const Reflection* message_reflection = message.GetReflection();
 
-  stream->EnsureSpace(&target);
+  target = stream->EnsureSpace(target);
   // Start group.
   target = io::CodedOutputStream::WriteTagToArray(
       WireFormatLite::kMessageSetItemStartTag, target);
@@ -1058,7 +1058,7 @@ uint8* WireFormat::InternalSerializeMessageSetItem(
       WireFormatLite::kMessageSetMessageNumber,
       message_reflection->GetMessage(message, field), target, stream);
   // End group.
-  stream->EnsureSpace(&target);
+  target = stream->EnsureSpace(target);
   target = io::CodedOutputStream::WriteTagToArray(
       WireFormatLite::kMessageSetItemEndTag, target);
   return target;

src/google/protobuf/wire_format_lite.h

@@ -1703,7 +1703,7 @@ inline uint8* WireFormatLite::InternalWriteGroupToArray(
     io::EpsCopyOutputStream* stream) {
   target = WriteTagToArray(field_number, WIRETYPE_START_GROUP, target);
   target = value.InternalSerializeWithCachedSizesToArray(target, stream);
-  stream->EnsureSpace(&target);
+  target = stream->EnsureSpace(target);
   return WriteTagToArray(field_number, WIRETYPE_END_GROUP, target);
 }
 template <typename MessageType>

src/google/protobuf/wire_format_unittest.cc

@@ -961,8 +961,7 @@ class Proto3PrimitiveRepeatedWireFormatTest : public ::testing::Test {
     message->add_repeated_float(1.0);
     message->add_repeated_double(1.0);
     message->add_repeated_bool(true);
-    message->add_repeated_nested_enum(
-        proto3_arena_unittest::TestAllTypes_NestedEnum_FOO);
+    message->add_repeated_nested_enum(proto3_arena_unittest::TestAllTypes::FOO);
   }
 
   template <class Proto>
@@ -980,7 +979,7 @@ class Proto3PrimitiveRepeatedWireFormatTest : public ::testing::Test {
     EXPECT_EQ(1.0, message.repeated_float(0));
     EXPECT_EQ(1.0, message.repeated_double(0));
     EXPECT_EQ(true, message.repeated_bool(0));
-    EXPECT_EQ(proto3_arena_unittest::TestAllTypes_NestedEnum_FOO,
+    EXPECT_EQ(proto3_arena_unittest::TestAllTypes::FOO,
               message.repeated_nested_enum(0));
   }
 

src/google/protobuf/wrappers.pb.cc

@@ -411,7 +411,7 @@ failure:
 
   // double value = 1;
   if (!(this->value() <= 0 && this->value() >= 0)) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteDoubleToArray(1, this->_internal_value(), target);
   }
 
@@ -610,7 +610,7 @@ failure:
 
   // float value = 1;
   if (!(this->value() <= 0 && this->value() >= 0)) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteFloatToArray(1, this->_internal_value(), target);
   }
 
@@ -809,7 +809,7 @@ failure:
 
   // int64 value = 1;
   if (this->value() != 0) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(1, this->_internal_value(), target);
   }
 
@@ -1010,7 +1010,7 @@ failure:
 
   // uint64 value = 1;
   if (this->value() != 0) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteUInt64ToArray(1, this->_internal_value(), target);
   }
 
@@ -1211,7 +1211,7 @@ failure:
 
   // int32 value = 1;
   if (this->value() != 0) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(1, this->_internal_value(), target);
   }
 
@@ -1412,7 +1412,7 @@ failure:
 
   // uint32 value = 1;
   if (this->value() != 0) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteUInt32ToArray(1, this->_internal_value(), target);
   }
 
@@ -1613,7 +1613,7 @@ failure:
 
   // bool value = 1;
   if (this->value() != 0) {
-    stream->EnsureSpace(&target);
+    target = stream->EnsureSpace(target);
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(1, this->_internal_value(), target);
   }
 

update_compatibility_version.py

@@ -1,34 +1,34 @@
 #!/usr/bin/env python
-# Usage: ./update_compatibility_version.py <MAJOR>.<MINOR>.<MICRO> [<RC version>]
-#
-# Example:
-# ./update_compatibility_version.py 3.7.1
+"""Compatibility tests between last released and the current version.
 
-import datetime
+Usage: ./update_compatibility_version.py <MAJOR>.<MINOR>.<MICRO> [<RC version>]
+Example: ./update_compatibility_version.py 3.7.1
+"""
+
+from __future__ import print_function
 import re
 import sys
-from xml.dom import minidom
 
 if len(sys.argv) < 2 or len(sys.argv) > 3:
-  print """
+  print("""
 [ERROR] Please specify a version.
 
-./update_version.py <MAJOR>.<MINOR>.<MICRO> [<RC version>]
+./update_compatibility_version.py.py <MAJOR>.<MINOR>.<MICRO> [<RC version>]
 
 Example:
-./update_version.py 3.7.1 2
-"""
+./update_compatibility_version.py.py 3.7.1 2
+""")
   exit(1)
 
 NEW_VERSION = sys.argv[1]
 NEW_VERSION_INFO = NEW_VERSION.split('.')
 if len(NEW_VERSION_INFO) != 3:
-  print """
+  print("""
 [ERROR] Version must be in the format <MAJOR>.<MINOR>.<MICRO>
 
 Example:
-./update_version.py 3.7.3
-"""
+./update_compatibility_version.py.py 3.7.3
+""")
   exit(1)
 
 if len(sys.argv) > 2:
@@ -37,21 +37,20 @@ if len(sys.argv) > 2:
   if RC_VERSION != 0:
     exit(0)
 
+
 def RewriteTextFile(filename, line_rewriter):
   lines = open(filename, 'r').readlines()
   updated_lines = []
   for line in lines:
     updated_lines.append(line_rewriter(line))
   if lines == updated_lines:
-    print '%s was not updated. Please double check.' % filename
+    print('%s was not updated. Please double check.' % filename)
   f = open(filename, 'w')
   f.write(''.join(updated_lines))
   f.close()
 
 
-RewriteTextFile('tests.sh',
-  lambda line : re.sub(
-    r'LAST_RELEASED=.*$',
-    'LAST_RELEASED=%s' % NEW_VERSION,
-    line))
+def ReplaceVersion(line):
+  return re.sub(r'LAST_RELEASED=.*$', 'LAST_RELEASED=%s' % NEW_VERSION, line)
 
+RewriteTextFile('tests.sh', ReplaceVersion)