Merge pull request #3770 from pherl/3.5-integrate

Integrate google internal changes for the up coming 3.5 release.

BUILD

@@ -541,6 +541,7 @@ cc_test(
         "src/google/protobuf/any_test.cc",
         "src/google/protobuf/arena_unittest.cc",
         "src/google/protobuf/arenastring_unittest.cc",
+        "src/google/protobuf/compiler/annotation_test_util.cc",
         "src/google/protobuf/compiler/command_line_interface_unittest.cc",
         "src/google/protobuf/compiler/cpp/cpp_bootstrap_unittest.cc",
         "src/google/protobuf/compiler/cpp/cpp_move_unittest.cc",

cmake/tests.cmake

@@ -121,6 +121,7 @@ set(tests_files
   ${protobuf_source_dir}/src/google/protobuf/any_test.cc
   ${protobuf_source_dir}/src/google/protobuf/arena_unittest.cc
   ${protobuf_source_dir}/src/google/protobuf/arenastring_unittest.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/annotation_test_util.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/command_line_interface_unittest.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/cpp_bootstrap_unittest.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/cpp_move_unittest.cc

conformance/failure_list_js.txt

@@ -1,5 +1,3 @@
-Required.Proto3.ProtobufInput.RepeatedScalarSelectsLast.INT32.ProtobufOutput
-Required.Proto3.ProtobufInput.RepeatedScalarSelectsLast.UINT32.ProtobufOutput
 Required.Proto3.ProtobufInput.ValidDataRepeated.BOOL.ProtobufOutput
 Required.Proto3.ProtobufInput.ValidDataRepeated.DOUBLE.ProtobufOutput
 Required.Proto3.ProtobufInput.ValidDataRepeated.FIXED32.ProtobufOutput
@@ -13,7 +11,3 @@ Required.Proto3.ProtobufInput.ValidDataRepeated.SINT32.ProtobufOutput
 Required.Proto3.ProtobufInput.ValidDataRepeated.SINT64.ProtobufOutput
 Required.Proto3.ProtobufInput.ValidDataRepeated.UINT32.ProtobufOutput
 Required.Proto3.ProtobufInput.ValidDataRepeated.UINT64.ProtobufOutput
-Required.Proto2.ProtobufInput.RepeatedScalarSelectsLast.INT32.ProtobufOutput
-Required.Proto2.ProtobufInput.RepeatedScalarSelectsLast.UINT32.ProtobufOutput
-Required.Proto2.ProtobufInput.ValidDataRepeated.INT32.ProtobufOutput
-Required.Proto2.ProtobufInput.ValidDataRepeated.UINT32.ProtobufOutput

conformance/failure_list_python.txt

@@ -7,7 +7,6 @@ Recommended.Proto3.JsonInput.FloatFieldInfinityNotQuoted
 Recommended.Proto3.JsonInput.FloatFieldNanNotQuoted
 Recommended.Proto3.JsonInput.FloatFieldNegativeInfinityNotQuoted
 Required.Proto3.JsonInput.DoubleFieldTooSmall
-Required.Proto3.JsonInput.EnumFieldUnknownValue.Validator
 Required.Proto3.JsonInput.FloatFieldTooLarge
 Required.Proto3.JsonInput.FloatFieldTooSmall
 Required.Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingIntegersGotBool

conformance/failure_list_python_cpp.txt

@@ -16,7 +16,6 @@ Recommended.Proto3.JsonInput.FloatFieldInfinityNotQuoted
 Recommended.Proto3.JsonInput.FloatFieldNanNotQuoted
 Recommended.Proto3.JsonInput.FloatFieldNegativeInfinityNotQuoted
 Required.Proto3.JsonInput.DoubleFieldTooSmall
-Required.Proto3.JsonInput.EnumFieldUnknownValue.Validator
 Required.Proto3.JsonInput.FloatFieldTooLarge
 Required.Proto3.JsonInput.FloatFieldTooSmall
 Required.Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingIntegersGotBool

csharp/src/Google.Protobuf/Reflection/Descriptor.cs

@@ -3478,7 +3478,8 @@ namespace Google.Protobuf.Reflection {
         = pb::FieldCodec.ForMessage(7994, global::Google.Protobuf.Reflection.UninterpretedOption.Parser);
     private readonly pbc::RepeatedField<global::Google.Protobuf.Reflection.UninterpretedOption> uninterpretedOption_ = new pbc::RepeatedField<global::Google.Protobuf.Reflection.UninterpretedOption>();
     /// <summary>
-    /// The parser stores options it doesn't recognize here. See above.
+    /// The parser stores options it doesn't recognize here.
+    /// See the documentation for the "Options" section above.
     /// </summary>
     [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
     public pbc::RepeatedField<global::Google.Protobuf.Reflection.UninterpretedOption> UninterpretedOption {

csharp/src/Google.Protobuf/WellKnownTypes/FieldMask.cs

@@ -239,6 +239,12 @@ namespace Google.Protobuf.WellKnownTypes {
   ///
   /// Note that oneof type names ("test_oneof" in this case) cannot be used in
   /// paths.
+  ///
+  /// ## Field Mask Verification
+  ///
+  /// The implementation of the all the API methods, which have any FieldMask type
+  /// field in the request, should verify the included field paths, and return
+  /// `INVALID_ARGUMENT` error if any path is duplicated or unmappable.
   /// </summary>
   public sealed partial class FieldMask : pb::IMessage<FieldMask> {
     private static readonly pb::MessageParser<FieldMask> _parser = new pb::MessageParser<FieldMask>(() => new FieldMask());

java/core/src/main/java/com/google/protobuf/AbstractMessage.java

@@ -380,6 +380,10 @@ public abstract class AbstractMessage
 
     @Override
     public BuilderType mergeFrom(final Message other) {
+      return mergeFrom(other, other.getAllFields());
+    }
+    
+    BuilderType mergeFrom(final Message other, Map<FieldDescriptor, Object> allFields) {
       if (other.getDescriptorForType() != getDescriptorForType()) {
         throw new IllegalArgumentException(
           "mergeFrom(Message) can only merge messages of the same type.");
@@ -394,8 +398,7 @@ public abstract class AbstractMessage
       // TODO(kenton):  Provide a function somewhere called makeDeepCopy()
       //   which allows people to make secure deep copies of messages.
 
-      for (final Map.Entry<FieldDescriptor, Object> entry :
-           other.getAllFields().entrySet()) {
+      for (final Map.Entry<FieldDescriptor, Object> entry : allFields.entrySet()) {
         final FieldDescriptor field = entry.getKey();
         if (field.isRepeated()) {
           for (final Object element : (List)entry.getValue()) {

java/core/src/main/java/com/google/protobuf/CodedInputStream.java

@@ -413,8 +413,7 @@ public abstract class CodedInputStream {
 
   private boolean explicitDiscardUnknownFields = false;
 
-  /** TODO(liujisi): flip the default.*/
-  private static volatile boolean proto3DiscardUnknownFieldsDefault = true;
+  private static volatile boolean proto3DiscardUnknownFieldsDefault = false;
 
   static void setProto3DiscardUnknownsByDefaultForTest() {
     proto3DiscardUnknownFieldsDefault = true;

java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java

@@ -31,7 +31,6 @@
 package com.google.protobuf;
 
 import com.google.protobuf.AbstractMessageLite.Builder.LimitedInputStream;
-import com.google.protobuf.GeneratedMessageLite.EqualsVisitor.NotEqualsException;
 import com.google.protobuf.Internal.BooleanList;
 import com.google.protobuf.Internal.DoubleList;
 import com.google.protobuf.Internal.EnumLiteMap;
@@ -52,6 +51,7 @@ import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 
 /**
  * Lite version of {@link GeneratedMessage}.
@@ -62,6 +62,12 @@ public abstract class GeneratedMessageLite<
     MessageType extends GeneratedMessageLite<MessageType, BuilderType>,
     BuilderType extends GeneratedMessageLite.Builder<MessageType, BuilderType>>
         extends AbstractMessageLite<MessageType, BuilderType> {
+  // BEGIN REGULAR
+  static final boolean ENABLE_EXPERIMENTAL_RUNTIME_AT_BUILD_TIME = false;
+  // END REGULAR
+  // BEGIN EXPERIMENTAL
+  // static final boolean ENABLE_EXPERIMENTAL_RUNTIME_AT_BUILD_TIME = true;
+  // END EXPERIMENTAL
 
   /** For use by generated code only. Lazily initialized to reduce allocations. */
   protected UnknownFieldSetLite unknownFields = UnknownFieldSetLite.getDefaultInstance();
@@ -110,12 +116,19 @@ public abstract class GeneratedMessageLite<
     if (memoizedHashCode != 0) {
       return memoizedHashCode;
     }
+    // BEGIN EXPERIMENTAL
+    // memoizedHashCode = Protobuf.getInstance().schemaFor(this).hashCode(this);
+    // return memoizedHashCode;
+    // END EXPERIMENTAL
+    // BEGIN REGULAR
     HashCodeVisitor visitor = new HashCodeVisitor();
     visit(visitor, (MessageType) this);
     memoizedHashCode = visitor.hashCode;
     return memoizedHashCode;
+    // END REGULAR
   }
 
+  // BEGIN REGULAR
   @SuppressWarnings("unchecked") // Guaranteed by runtime
   int hashCode(HashCodeVisitor visitor) {
     if (memoizedHashCode == 0) {
@@ -127,6 +140,7 @@ public abstract class GeneratedMessageLite<
     }
     return memoizedHashCode;
   }
+  // END REGULAR
 
   @SuppressWarnings("unchecked") // Guaranteed by isInstance + runtime
   @Override
@@ -139,18 +153,22 @@ public abstract class GeneratedMessageLite<
       return false;
     }
 
+    // BEGIN EXPERIMENTAL
+    // return Protobuf.getInstance().schemaFor(this).equals(this, (MessageType) other);
+    // END EXPERIMENTAL
+    // BEGIN REGULAR
 
     try {
       visit(EqualsVisitor.INSTANCE, (MessageType) other);
-    } catch (NotEqualsException e) {
+    } catch (EqualsVisitor.NotEqualsException e) {
       return false;
     }
     return true;
+    // END REGULAR
   }
 
-  /**
-   * Same as {@link #equals(Object)} but throws {@code NotEqualsException}.
-   */
+  // BEGIN REGULAR
+  /** Same as {@link #equals(Object)} but throws {@code NotEqualsException}. */
   @SuppressWarnings("unchecked") // Guaranteed by isInstance + runtime
   boolean equals(EqualsVisitor visitor, MessageLite other) {
     if (this == other) {
@@ -164,14 +182,13 @@ public abstract class GeneratedMessageLite<
     visit(visitor, (MessageType) other);
     return true;
   }
+  // END REGULAR
 
   // The general strategy for unknown fields is to use an UnknownFieldSetLite that is treated as
   // mutable during the parsing constructor and immutable after. This allows us to avoid
   // any unnecessary intermediary allocations while reducing the generated code size.
 
-  /**
-   * Lazily initializes unknown fields.
-   */
+  /** Lazily initializes unknown fields. */
   private final void ensureUnknownFieldsInitialized() {
     if (unknownFields == UnknownFieldSetLite.getDefaultInstance()) {
       unknownFields = UnknownFieldSetLite.newInstance();
@@ -218,6 +235,20 @@ public abstract class GeneratedMessageLite<
     unknownFields.makeImmutable();
   }
 
+  protected final <
+    MessageType extends GeneratedMessageLite<MessageType, BuilderType>,
+    BuilderType extends GeneratedMessageLite.Builder<MessageType, BuilderType>>
+        BuilderType createBuilder() {
+    return (BuilderType) dynamicMethod(MethodToInvoke.NEW_BUILDER);
+  }
+
+  protected final <
+    MessageType extends GeneratedMessageLite<MessageType, BuilderType>,
+    BuilderType extends GeneratedMessageLite.Builder<MessageType, BuilderType>>
+        BuilderType createBuilder(MessageType prototype) {
+    return ((BuilderType) createBuilder()).mergeFrom(prototype);
+  }
+
   @Override
   public final boolean isInitialized() {
     return isInitialized((MessageType) this, Boolean.TRUE);
@@ -238,11 +269,13 @@ public abstract class GeneratedMessageLite<
    * For use by generated code only.
    */
   public static enum MethodToInvoke {
-    // Rely on/modify instance state
     IS_INITIALIZED,
+    // BEGIN REGULAR
+    VISIT,
+    // END REGULAR
+    // Rely on/modify instance state
     GET_MEMOIZED_IS_INITIALIZED,
     SET_MEMOIZED_IS_INITIALIZED,
-    VISIT,
     MERGE_FROM_STREAM,
     MAKE_IMMUTABLE,
 
@@ -299,10 +332,13 @@ public abstract class GeneratedMessageLite<
     return dynamicMethod(method, null, null);
   }
 
+  // BEGIN REGULAR
   void visit(Visitor visitor, MessageType other) {
     dynamicMethod(MethodToInvoke.VISIT, visitor, other);
     unknownFields = visitor.visitUnknownFields(unknownFields, other.unknownFields);
   }
+  // END REGULAR
+
 
 
   /**
@@ -399,7 +435,12 @@ public abstract class GeneratedMessageLite<
     }
 
     private void mergeFromInstance(MessageType dest, MessageType src) {
+      // BEGIN EXPERIMENTAL
+      // Protobuf.getInstance().schemaFor(dest).mergeFrom(dest, src);
+      // END EXPERIMENTAL
+      // BEGIN REGULAR
       dest.visit(MergeFromVisitor.INSTANCE, src);
+      // END REGULAR
     }
 
     @Override
@@ -477,11 +518,13 @@ public abstract class GeneratedMessageLite<
       extensions.mergeFrom(((ExtendableMessage) other).extensions);
     }
 
+    // BEGIN REGULAR
     @Override
     final void visit(Visitor visitor, MessageType other) {
       super.visit(visitor, other);
       extensions = visitor.visitExtensions(extensions, other.extensions);
     }
+    // END REGULAR
 
     /**
      * Parse an unknown field or an extension. For use by generated code only.
@@ -494,7 +537,8 @@ public abstract class GeneratedMessageLite<
         MessageType defaultInstance,
         CodedInputStream input,
         ExtensionRegistryLite extensionRegistry,
-        int tag) throws IOException {
+        int tag)
+        throws IOException {
       int fieldNumber = WireFormat.getTagFieldNumber(tag);
 
       // TODO(dweis): How much bytecode would be saved by not requiring the generated code to
@@ -1716,6 +1760,7 @@ public abstract class GeneratedMessageLite<
     return message;
   }
 
+  // BEGIN REGULAR
   /**
    * An abstract visitor that the generated code calls into that we use to implement various
    * features. Fields that are not members of oneofs are always visited. Members of a oneof are only
@@ -2401,4 +2446,5 @@ public abstract class GeneratedMessageLite<
       return mine;
     }
   }
+  // END REGULAR
 }

java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java

@@ -358,6 +358,10 @@ public abstract class GeneratedMessageV3 extends AbstractMessage
       throw e.unwrapIOException();
     }
   }
+  
+  protected static boolean canUseUnsafe() {
+    return UnsafeUtil.hasUnsafeArrayOperations() && UnsafeUtil.hasUnsafeByteBufferOperations();
+  }
 
   @Override
   public void writeTo(final CodedOutputStream output) throws IOException {
@@ -655,6 +659,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage
                        .build());
     }
 
+
     @Override
     public boolean isInitialized() {
       for (final FieldDescriptor field : getDescriptorForType().getFields()) {
@@ -2853,3 +2858,4 @@ public abstract class GeneratedMessageV3 extends AbstractMessage
     }
   }
 }
+

java/core/src/main/java/com/google/protobuf/Message.java

@@ -125,7 +125,7 @@ public interface Message extends MessageLite, MessageOrBuilder {
      *   it is merged into the corresponding sub-message of this message
      *   using the same merging rules.<br>
      * * For repeated fields, the elements in {@code other} are concatenated
-     *   with the elements in this message.
+     *   with the elements in this message.<br>
      * * For oneof groups, if the other message has one of the fields set,
      *   the group of this message is cleared and replaced by the field
      *   of the other message, so that the oneof constraint is preserved.

java/core/src/test/java/com/google/protobuf/FieldPresenceTest.java

@@ -404,34 +404,4 @@ public class FieldPresenceTest extends TestCase {
     assertTrue(builder.buildPartial().isInitialized());
   }
 
-
-  // Test that unknown fields are dropped.
-  public void testUnknownFields() throws Exception {
-    TestAllTypes.Builder builder = TestAllTypes.newBuilder();
-    builder.setOptionalInt32(1234);
-    builder.addRepeatedInt32(5678);
-    TestAllTypes message = builder.build();
-    ByteString data = message.toByteString();
-
-    TestOptionalFieldsOnly optionalOnlyMessage =
-        TestOptionalFieldsOnly.parseFrom(data);
-    // UnknownFieldSet should be empty.
-    assertEquals(
-        0, optionalOnlyMessage.getUnknownFields().toByteString().size());
-    assertEquals(1234, optionalOnlyMessage.getOptionalInt32());
-    message = TestAllTypes.parseFrom(optionalOnlyMessage.toByteString());
-    assertEquals(1234, message.getOptionalInt32());
-    // The repeated field is discarded because it's unknown to the optional-only
-    // message.
-    assertEquals(0, message.getRepeatedInt32Count());
-
-    DynamicMessage dynamicOptionalOnlyMessage =
-        DynamicMessage.getDefaultInstance(
-            TestOptionalFieldsOnly.getDescriptor())
-        .getParserForType().parseFrom(data);
-    assertEquals(
-        0, dynamicOptionalOnlyMessage.getUnknownFields().toByteString().size());
-    assertEquals(optionalOnlyMessage.toByteString(),
-        dynamicOptionalOnlyMessage.toByteString());
-  }
 }

java/core/src/test/java/com/google/protobuf/LiteTest.java

@@ -1453,6 +1453,15 @@ public class LiteTest extends TestCase {
             UnittestLite.optionalFixed32ExtensionLite));
   }
 
+  public void testBuilderMergeFromNull() throws Exception {
+    try {
+      TestAllTypesLite.newBuilder().mergeFrom((TestAllTypesLite) null);
+      fail("Expected exception");
+    } catch (NullPointerException e) {
+      // Pass.
+    }
+  }
+
   // Builder.mergeFrom() should keep existing extensions.
   public void testBuilderMergeFromWithExtensions() throws Exception {
     TestAllExtensionsLite message =

java/util/src/main/java/com/google/protobuf/util/FieldMaskTree.java

@@ -34,7 +34,6 @@ import com.google.protobuf.Descriptors.Descriptor;
 import com.google.protobuf.Descriptors.FieldDescriptor;
 import com.google.protobuf.FieldMask;
 import com.google.protobuf.Message;
-
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map.Entry;
@@ -244,6 +243,11 @@ final class FieldMaskTree {
                   + "singluar message field and cannot have sub-fields.");
           continue;
         }
+        if (!source.hasField(field) && !destination.hasField(field)) {
+          // If the message field is not present in both source and destination, skip recursing
+          // so we don't create unneccessary empty messages.
+          continue;
+        }
         String childPath = path.isEmpty() ? entry.getKey() : path + "." + entry.getKey();
         merge(
             entry.getValue(),

java/util/src/test/java/com/google/protobuf/util/FieldMaskTreeTest.java

@@ -33,7 +33,6 @@ package com.google.protobuf.util;
 import protobuf_unittest.UnittestProto.NestedTestAllTypes;
 import protobuf_unittest.UnittestProto.TestAllTypes;
 import protobuf_unittest.UnittestProto.TestAllTypes.NestedMessage;
-
 import junit.framework.TestCase;
 
 public class FieldMaskTreeTest extends TestCase {
@@ -222,6 +221,13 @@ public class FieldMaskTreeTest extends TestCase {
     new FieldMaskTree().addFieldPath("payload").merge(clearedSource, builder, options);
     assertEquals(false, builder.hasPayload());
 
+    // Skip a message field if they are unset in both source and target.
+    builder = NestedTestAllTypes.newBuilder();
+    new FieldMaskTree()
+        .addFieldPath("payload.optional_int32")
+        .merge(clearedSource, builder, options);
+    assertEquals(false, builder.hasPayload());
+
     // Change to replace message fields.
     options.setReplaceMessageFields(true);
     builder = NestedTestAllTypes.newBuilder();

js/binary/arith_test.js

@@ -36,7 +36,6 @@
  * @author cfallin@google.com (Chris Fallin)
  */
 
-goog.require('goog.testing.asserts');
 goog.require('jspb.arith.Int64');
 goog.require('jspb.arith.UInt64');
 
@@ -48,30 +47,30 @@ describe('binaryArithTest', function() {
   it('testCompare', function() {
     var a = new jspb.arith.UInt64(1234, 5678);
     var b = new jspb.arith.UInt64(1234, 5678);
-    assertEquals(a.cmp(b), 0);
-    assertEquals(b.cmp(a), 0);
+    expect(a.cmp(b)).toEqual(0);
+    expect(b.cmp(a)).toEqual(0);
     b.lo -= 1;
-    assertEquals(a.cmp(b), 1);
-    assertEquals(b.cmp(a), -1);
+    expect(a.cmp(b)).toEqual(1);
+    expect(b.cmp(a)).toEqual(-1);
     b.lo += 2;
-    assertEquals(a.cmp(b), -1);
-    assertEquals(b.cmp(a), 1);
+    expect(a.cmp(b)).toEqual(-1);
+    expect(b.cmp(a)).toEqual(1);
     b.lo = a.lo;
     b.hi = a.hi - 1;
-    assertEquals(a.cmp(b), 1);
-    assertEquals(b.cmp(a), -1);
+    expect(a.cmp(b)).toEqual(1);
+    expect(b.cmp(a)).toEqual(-1);
 
-    assertEquals(a.zero(), false);
-    assertEquals(a.msb(), false);
-    assertEquals(a.lsb(), false);
+    expect(a.zero()).toEqual(false);
+    expect(a.msb()).toEqual(false);
+    expect(a.lsb()).toEqual(false);
     a.hi = 0;
     a.lo = 0;
-    assertEquals(a.zero(), true);
+    expect(a.zero()).toEqual(true);
     a.hi = 0x80000000;
-    assertEquals(a.zero(), false);
-    assertEquals(a.msb(), true);
+    expect(a.zero()).toEqual(false);
+    expect(a.msb()).toEqual(true);
     a.lo = 0x00000001;
-    assertEquals(a.lsb(), true);
+    expect(a.lsb()).toEqual(true);
   });
 
 
@@ -80,35 +79,35 @@ describe('binaryArithTest', function() {
    */
   it('testShifts', function() {
     var a = new jspb.arith.UInt64(1, 0);
-    assertEquals(a.lo, 1);
-    assertEquals(a.hi, 0);
+    expect(a.lo).toEqual(1);
+    expect(a.hi).toEqual(0);
     var orig = a;
     a = a.leftShift();
-    assertEquals(orig.lo, 1);  // original unmodified.
-    assertEquals(orig.hi, 0);
-    assertEquals(a.lo, 2);
-    assertEquals(a.hi, 0);
+    expect(orig.lo).toEqual(1);  // original unmodified.
+    expect(orig.hi).toEqual(0);
+    expect(a.lo).toEqual(2);
+    expect(a.hi).toEqual(0);
     a = a.leftShift();
-    assertEquals(a.lo, 4);
-    assertEquals(a.hi, 0);
+    expect(a.lo).toEqual(4);
+    expect(a.hi).toEqual(0);
     for (var i = 0; i < 29; i++) {
       a = a.leftShift();
     }
-    assertEquals(a.lo, 0x80000000);
-    assertEquals(a.hi, 0);
+    expect(a.lo).toEqual(0x80000000);
+    expect(a.hi).toEqual(0);
     a = a.leftShift();
-    assertEquals(a.lo, 0);
-    assertEquals(a.hi, 1);
+    expect(a.lo).toEqual(0);
+    expect(a.hi).toEqual(1);
     a = a.leftShift();
-    assertEquals(a.lo, 0);
-    assertEquals(a.hi, 2);
+    expect(a.lo).toEqual(0);
+    expect(a.hi).toEqual(2);
     a = a.rightShift();
     a = a.rightShift();
-    assertEquals(a.lo, 0x80000000);
-    assertEquals(a.hi, 0);
+    expect(a.lo).toEqual(0x80000000);
+    expect(a.hi).toEqual(0);
     a = a.rightShift();
-    assertEquals(a.lo, 0x40000000);
-    assertEquals(a.hi, 0);
+    expect(a.lo).toEqual(0x40000000);
+    expect(a.hi).toEqual(0);
   });
 
 
@@ -122,12 +121,12 @@ describe('binaryArithTest', function() {
                                          /* hi = */ 0x92fa2123);
     // Addition with carry.
     var c = a.add(b);
-    assertEquals(a.lo, 0x89abcdef);  // originals unmodified.
-    assertEquals(a.hi, 0x01234567);
-    assertEquals(b.lo, 0xff52ab91);
-    assertEquals(b.hi, 0x92fa2123);
-    assertEquals(c.lo, 0x88fe7980);
-    assertEquals(c.hi, 0x941d668b);
+    expect(a.lo).toEqual(0x89abcdef);  // originals unmodified.
+    expect(a.hi).toEqual(0x01234567);
+    expect(b.lo).toEqual(0xff52ab91);
+    expect(b.hi).toEqual(0x92fa2123);
+    expect(c.lo).toEqual(0x88fe7980);
+    expect(c.hi).toEqual(0x941d668b);
 
     // Simple addition without carry.
     a.lo = 2;
@@ -135,8 +134,8 @@ describe('binaryArithTest', function() {
     b.lo = 3;
     b.hi = 0;
     c = a.add(b);
-    assertEquals(c.lo, 5);
-    assertEquals(c.hi, 0);
+    expect(c.lo).toEqual(5);
+    expect(c.hi).toEqual(0);
   });
 
 
@@ -170,8 +169,8 @@ describe('binaryArithTest', function() {
         var a = new jspb.arith.UInt64(loValues[i], hiValues[j]);
         var b = new jspb.arith.UInt64(loValues[j], hiValues[i]);
         var c = a.add(b).sub(b);
-        assertEquals(c.hi, a.hi);
-        assertEquals(c.lo, a.lo);
+        expect(c.hi).toEqual(a.hi);
+        expect(c.lo).toEqual(a.lo);
       }
     }
   });
@@ -201,8 +200,8 @@ describe('binaryArithTest', function() {
       var cLow = testData[i][2] >>> 0;
       var cHigh = testData[i][3] >>> 0;
       var c = jspb.arith.UInt64.mul32x32(a, b);
-      assertEquals(c.lo, cLow);
-      assertEquals(c.hi, cHigh);
+      expect(c.lo).toEqual(cLow);
+      expect(c.hi).toEqual(cHigh);
     }
   });
 
@@ -231,8 +230,8 @@ describe('binaryArithTest', function() {
     for (var i = 0; i < testData.length; i++) {
       var a = new jspb.arith.UInt64(testData[i][0], testData[i][1]);
       var prod = a.mul(testData[i][2]);
-      assertEquals(prod.lo, testData[i][3]);
-      assertEquals(prod.hi, testData[i][4]);
+      expect(prod.lo).toEqual(testData[i][3]);
+      expect(prod.hi).toEqual(testData[i][4]);
     }
   });
 
@@ -274,9 +273,9 @@ describe('binaryArithTest', function() {
       var result = a.div(testData[i][2]);
       var quotient = result[0];
       var remainder = result[1];
-      assertEquals(quotient.lo, testData[i][3]);
-      assertEquals(quotient.hi, testData[i][4]);
-      assertEquals(remainder.lo, testData[i][5]);
+      expect(quotient.lo).toEqual(testData[i][3]);
+      expect(quotient.hi).toEqual(testData[i][4]);
+      expect(remainder.lo).toEqual(testData[i][5]);
     }
   });
 
@@ -311,9 +310,9 @@ describe('binaryArithTest', function() {
     for (var i = 0; i < testData.length; i++) {
       var a = new jspb.arith.UInt64(testData[i][0], testData[i][1]);
       var roundtrip = jspb.arith.UInt64.fromString(a.toString());
-      assertEquals(roundtrip.lo, a.lo);
-      assertEquals(roundtrip.hi, a.hi);
-      assertEquals(a.toString(), testData[i][2]);
+      expect(roundtrip.lo).toEqual(a.lo);
+      expect(roundtrip.hi).toEqual(a.hi);
+      expect(a.toString()).toEqual(testData[i][2]);
     }
   });
 
@@ -349,7 +348,7 @@ describe('binaryArithTest', function() {
     for (var i = 0; i < testStrings.length; i++) {
       var roundtrip =
           jspb.arith.Int64.fromString(testStrings[i]).toString();
-      assertEquals(roundtrip, testStrings[i]);
+      expect(roundtrip).toEqual(testStrings[i]);
     }
   });
 });

js/binary/constants.js

@@ -51,6 +51,9 @@ goog.provide('jspb.ScalarFieldType');
 goog.provide('jspb.WriterFunction');
 
 
+goog.forwardDeclare('jspb.BinaryMessage');
+goog.forwardDeclare('jspb.BinaryReader');
+goog.forwardDeclare('jspb.BinaryWriter');
 goog.forwardDeclare('jspb.Message');
 goog.forwardDeclare('jsproto.BinaryExtension');
 

js/binary/decoder.js

@@ -583,27 +583,24 @@ jspb.BinaryDecoder.prototype.readUnsignedVarint32 = function() {
   x |= (temp & 0x0F) << 28;
   if (temp < 128) {
     // We're reading the high bits of an unsigned varint. The byte we just read
-    // also contains bits 33 through 35, which we're going to discard. Those
-    // bits _must_ be zero, or the encoding is invalid.
-    goog.asserts.assert((temp & 0xF0) == 0);
+    // also contains bits 33 through 35, which we're going to discard.
     this.cursor_ += 5;
     goog.asserts.assert(this.cursor_ <= this.end_);
     return x >>> 0;
   }
 
-  // If we get here, we're reading the sign extension of a negative 32-bit int.
-  // We can skip these bytes, as we know in advance that they have to be all
-  // 1's if the varint is correctly encoded. Since we also know the value is
-  // negative, we don't have to coerce it to unsigned before we return it.
-
-  goog.asserts.assert((temp & 0xF0) == 0xF0);
-  goog.asserts.assert(bytes[this.cursor_ + 5] == 0xFF);
-  goog.asserts.assert(bytes[this.cursor_ + 6] == 0xFF);
-  goog.asserts.assert(bytes[this.cursor_ + 7] == 0xFF);
-  goog.asserts.assert(bytes[this.cursor_ + 8] == 0xFF);
-  goog.asserts.assert(bytes[this.cursor_ + 9] == 0x01);
+  // If we get here, we need to truncate coming bytes. However we need to make
+  // sure cursor place is correct.
+  this.cursor_ += 5;
+  if (bytes[this.cursor_++] >= 128 &&
+      bytes[this.cursor_++] >= 128 &&
+      bytes[this.cursor_++] >= 128 &&
+      bytes[this.cursor_++] >= 128 &&
+      bytes[this.cursor_++] >= 128) {
+    // If we get here, the varint is too long.
+    goog.asserts.assert(false);
+  }
 
-  this.cursor_ += 10;
   goog.asserts.assert(this.cursor_ <= this.end_);
   return x;
 };

js/binary/decoder_test.js

@@ -270,24 +270,7 @@ describe('binaryDecoderTest', function() {
     assertThrows(function() {decoder.readSignedVarint64()});
     decoder.reset();
     assertThrows(function() {decoder.readZigzagVarint64()});
-
-    // Positive 32-bit varints encoded with 1 bits in positions 33 through 35
-    // should trigger assertions.
-    decoder.setBlock([255, 255, 255, 255, 0x1F]);
-    assertThrows(function() {decoder.readUnsignedVarint32()});
-
-    decoder.setBlock([255, 255, 255, 255, 0x2F]);
-    assertThrows(function() {decoder.readUnsignedVarint32()});
-
-    decoder.setBlock([255, 255, 255, 255, 0x4F]);
-    assertThrows(function() {decoder.readUnsignedVarint32()});
-
-    // Negative 32-bit varints encoded with non-1 bits in the high dword should
-    // trigger assertions.
-    decoder.setBlock([255, 255, 255, 255, 255, 255, 0, 255, 255, 1]);
-    assertThrows(function() {decoder.readUnsignedVarint32()});
-
-    decoder.setBlock([255, 255, 255, 255, 255, 255, 255, 255, 255, 0]);
+    decoder.reset();
     assertThrows(function() {decoder.readUnsignedVarint32()});
   });
 

js/compatibility_tests/v3.0.0/binary/decoder_test.js

@@ -228,24 +228,7 @@ describe('binaryDecoderTest', function() {
     assertThrows(function() {decoder.readSignedVarint64()});
     decoder.reset();
     assertThrows(function() {decoder.readZigzagVarint64()});
-
-    // Positive 32-bit varints encoded with 1 bits in positions 33 through 35
-    // should trigger assertions.
-    decoder.setBlock([255, 255, 255, 255, 0x1F]);
-    assertThrows(function() {decoder.readUnsignedVarint32()});
-
-    decoder.setBlock([255, 255, 255, 255, 0x2F]);
-    assertThrows(function() {decoder.readUnsignedVarint32()});
-
-    decoder.setBlock([255, 255, 255, 255, 0x4F]);
-    assertThrows(function() {decoder.readUnsignedVarint32()});
-
-    // Negative 32-bit varints encoded with non-1 bits in the high dword should
-    // trigger assertions.
-    decoder.setBlock([255, 255, 255, 255, 255, 255, 0, 255, 255, 1]);
-    assertThrows(function() {decoder.readUnsignedVarint32()});
-
-    decoder.setBlock([255, 255, 255, 255, 255, 255, 255, 255, 255, 0]);
+    decoder.reset();
     assertThrows(function() {decoder.readUnsignedVarint32()});
   });
 

js/compatibility_tests/v3.1.0/binary/decoder_test.js

@@ -228,24 +228,7 @@ describe('binaryDecoderTest', function() {
     assertThrows(function() {decoder.readSignedVarint64()});
     decoder.reset();
     assertThrows(function() {decoder.readZigzagVarint64()});
-
-    // Positive 32-bit varints encoded with 1 bits in positions 33 through 35
-    // should trigger assertions.
-    decoder.setBlock([255, 255, 255, 255, 0x1F]);
-    assertThrows(function() {decoder.readUnsignedVarint32()});
-
-    decoder.setBlock([255, 255, 255, 255, 0x2F]);
-    assertThrows(function() {decoder.readUnsignedVarint32()});
-
-    decoder.setBlock([255, 255, 255, 255, 0x4F]);
-    assertThrows(function() {decoder.readUnsignedVarint32()});
-
-    // Negative 32-bit varints encoded with non-1 bits in the high dword should
-    // trigger assertions.
-    decoder.setBlock([255, 255, 255, 255, 255, 255, 0, 255, 255, 1]);
-    assertThrows(function() {decoder.readUnsignedVarint32()});
-
-    decoder.setBlock([255, 255, 255, 255, 255, 255, 255, 255, 255, 0]);
+    decoder.reset();
     assertThrows(function() {decoder.readUnsignedVarint32()});
   });
 

objectivec/google/protobuf/FieldMask.pbobjc.h

@@ -252,6 +252,12 @@ typedef GPB_ENUM(GPBFieldMask_FieldNumber) {
  *
  * Note that oneof type names ("test_oneof" in this case) cannot be used in
  * paths.
+ *
+ * ## Field Mask Verification
+ *
+ * The implementation of the all the API methods, which have any FieldMask type
+ * field in the request, should verify the included field paths, and return
+ * `INVALID_ARGUMENT` error if any path is duplicated or unmappable.
  **/
 @interface GPBFieldMask : GPBMessage
 

php/src/Google/Protobuf/Internal/FileOptions.php

@@ -174,7 +174,8 @@ class FileOptions extends \Google\Protobuf\Internal\Message
     private $php_namespace = '';
     private $has_php_namespace = false;
     /**
-     * The parser stores options it doesn't recognize here. See above.
+     * The parser stores options it doesn't recognize here.
+     * See the documentation for the "Options" section above.
      *
      * Generated from protobuf field <code>repeated .google.protobuf.UninterpretedOption uninterpreted_option = 999;</code>
      */
@@ -827,7 +828,8 @@ class FileOptions extends \Google\Protobuf\Internal\Message
     }
 
     /**
-     * The parser stores options it doesn't recognize here. See above.
+     * The parser stores options it doesn't recognize here.
+     * See the documentation for the "Options" section above.
      *
      * Generated from protobuf field <code>repeated .google.protobuf.UninterpretedOption uninterpreted_option = 999;</code>
      * @return \Google\Protobuf\Internal\RepeatedField
@@ -838,7 +840,8 @@ class FileOptions extends \Google\Protobuf\Internal\Message
     }
 
     /**
-     * The parser stores options it doesn't recognize here. See above.
+     * The parser stores options it doesn't recognize here.
+     * See the documentation for the "Options" section above.
      *
      * Generated from protobuf field <code>repeated .google.protobuf.UninterpretedOption uninterpreted_option = 999;</code>
      * @param \Google\Protobuf\Internal\UninterpretedOption[]|\Google\Protobuf\Internal\RepeatedField $var

python/google/protobuf/internal/any_test.proto

@@ -39,6 +39,7 @@ import "google/protobuf/any.proto";
 message TestAny {
   optional google.protobuf.Any value = 1;
   optional int32 int_value = 2;
+  map<string,int32> map_value = 3;
   extensions 10 to max;
 }
 

python/google/protobuf/internal/api_implementation.py

@@ -159,7 +159,7 @@ if _implementation_type == 'cpp':
     # Unrecognized cpp implementation. Skipping the unknown fields APIs.
     pass
 else:
-  _python_proto3_preserve_unknowns_default = False
+  _python_proto3_preserve_unknowns_default = True
 
   def GetPythonProto3PreserveUnknownsDefault():
     return _python_proto3_preserve_unknowns_default

python/google/protobuf/internal/json_format_test.py

@@ -50,6 +50,7 @@ from google.protobuf import struct_pb2
 from google.protobuf import timestamp_pb2
 from google.protobuf import wrappers_pb2
 from google.protobuf import unittest_mset_pb2
+from google.protobuf import unittest_pb2
 from google.protobuf.internal import well_known_types
 from google.protobuf import json_format
 from google.protobuf.util import json_format_proto3_pb2
@@ -159,15 +160,15 @@ class JsonFormatTest(JsonFormatBase):
     json_format.Parse(text, parsed_message)
     self.assertEqual(message, parsed_message)
 
-  def testUnknownEnumToJsonError(self):
+  def testUnknownEnumToJsonAndBack(self):
+    text = '{\n  "enumValue": 999\n}'
     message = json_format_proto3_pb2.TestMessage()
     message.enum_value = 999
-    # TODO(jieluo): should accept numeric unknown enum for proto3.
-    with self.assertRaises(json_format.SerializeToJsonError) as e:
-      json_format.MessageToJson(message)
-    self.assertEqual(str(e.exception),
-                     'Enum field contains an integer value which can '
-                     'not mapped to an enum value.')
+    self.assertEqual(json_format.MessageToJson(message),
+                     text)
+    parsed_message = json_format_proto3_pb2.TestMessage()
+    json_format.Parse(text, parsed_message)
+    self.assertEqual(message, parsed_message)
 
   def testExtensionToJsonAndBack(self):
     message = unittest_mset_pb2.TestMessageSetContainer()
@@ -757,11 +758,16 @@ class JsonFormatTest(JsonFormatBase):
         '{"enumValue": "baz"}',
         'Failed to parse enumValue field: Invalid enum value baz '
         'for enum type proto3.EnumType.')
-    # TODO(jieluo): fix json format to accept numeric unknown enum for proto3.
-    self.CheckError(
-        '{"enumValue": 12345}',
-        'Failed to parse enumValue field: Invalid enum value 12345 '
-        'for enum type proto3.EnumType.')
+    # Proto3 accepts numeric unknown enums.
+    text = '{"enumValue": 12345}'
+    json_format.Parse(text, message)
+    # Proto2 does not accept unknown enums.
+    message = unittest_pb2.TestAllTypes()
+    self.assertRaisesRegexp(
+        json_format.ParseError,
+        'Failed to parse optionalNestedEnum field: Invalid enum value 12345 '
+        'for enum type protobuf_unittest.TestAllTypes.NestedEnum.',
+        json_format.Parse, '{"optionalNestedEnum": 12345}', message)
 
   def testParseBadIdentifer(self):
     self.CheckError('{int32Value: 1}',

python/google/protobuf/internal/message_factory_test.py

@@ -107,8 +107,17 @@ class MessageFactoryTest(unittest.TestCase):
   def testGetMessages(self):
     # performed twice because multiple calls with the same input must be allowed
     for _ in range(2):
-      messages = message_factory.GetMessages([self.factory_test1_fd,
-                                              self.factory_test2_fd])
+      # GetMessage should work regardless of the order the FileDescriptorProto
+      # are provided. In particular, the function should succeed when the files
+      # are not in the topological order of dependencies.
+
+      # Assuming factory_test2_fd depends on factory_test1_fd.
+      self.assertIn(self.factory_test1_fd.name,
+                    self.factory_test2_fd.dependency)
+      # Get messages should work when a file comes before its dependencies:
+      # factory_test2_fd comes before factory_test1_fd.
+      messages = message_factory.GetMessages([self.factory_test2_fd,
+                                              self.factory_test1_fd])
       self.assertTrue(
           set(['google.protobuf.python.internal.Factory2Message',
                'google.protobuf.python.internal.Factory1Message'],

python/google/protobuf/internal/message_test.py

@@ -51,6 +51,7 @@ import operator
 import pickle
 import six
 import sys
+import warnings
 
 try:
   import unittest2 as unittest  # PY26
@@ -146,13 +147,22 @@ class MessageTest(BaseTestCase):
     msg = message_module.TestAllTypes()
     self.assertRaises(TypeError, msg.FromString, 0)
     self.assertRaises(Exception, msg.FromString, '0')
-    # TODO(jieluo): Fix cpp extension to check unexpected end-group tag.
+    # TODO(jieluo): Fix cpp extension to raise error instead of warning.
     # b/27494216
+    end_tag = encoder.TagBytes(1, 4)
     if api_implementation.Type() == 'python':
-      end_tag = encoder.TagBytes(1, 4)
       with self.assertRaises(message.DecodeError) as context:
         msg.FromString(end_tag)
       self.assertEqual('Unexpected end-group tag.', str(context.exception))
+    else:
+      with warnings.catch_warnings(record=True) as w:
+        # Cause all warnings to always be triggered.
+        warnings.simplefilter('always')
+        msg.FromString(end_tag)
+        assert len(w) == 1
+        assert issubclass(w[-1].category, RuntimeWarning)
+        self.assertEqual('Unexpected end-group tag: Not all data was converted',
+                         str(w[-1].message))
 
   def testDeterminismParameters(self, message_module):
     # This message is always deterministically serialized, even if determinism

python/google/protobuf/internal/text_format_test.py

@@ -1,4 +1,5 @@
 #! /usr/bin/env python
+# -*- coding: utf-8 -*-
 #
 # Protocol Buffers - Google's data interchange format
 # Copyright 2008 Google Inc.  All rights reserved.
@@ -298,6 +299,33 @@ class TextFormatTest(TextFormatBase):
     if message_module is unittest_pb2:
       test_util.ExpectAllFieldsSet(self, message)
 
+  def testParseAndMergeUtf8(self, message_module):
+    message = message_module.TestAllTypes()
+    test_util.SetAllFields(message)
+    ascii_text = text_format.MessageToString(message)
+    ascii_text = ascii_text.encode('utf-8')
+
+    parsed_message = message_module.TestAllTypes()
+    text_format.Parse(ascii_text, parsed_message)
+    self.assertEqual(message, parsed_message)
+    if message_module is unittest_pb2:
+      test_util.ExpectAllFieldsSet(self, message)
+
+    parsed_message.Clear()
+    text_format.Merge(ascii_text, parsed_message)
+    self.assertEqual(message, parsed_message)
+    if message_module is unittest_pb2:
+      test_util.ExpectAllFieldsSet(self, message)
+
+    if six.PY2:
+      msg2 = message_module.TestAllTypes()
+      text = (u'optional_string: "café"')
+      text_format.Merge(text, msg2)
+      self.assertEqual(msg2.optional_string, u'café')
+      msg2.Clear()
+      text_format.Parse(text, msg2)
+      self.assertEqual(msg2.optional_string, u'café')
+
   def testParseExotic(self, message_module):
     message = message_module.TestAllTypes()
     text = ('repeated_int64: -9223372036854775808\n'
@@ -399,13 +427,6 @@ class TextFormatTest(TextFormatBase):
                            r'has no value named BARR.'), text_format.Parse,
                           text, message)
 
-    message = message_module.TestAllTypes()
-    text = 'optional_nested_enum: 100'
-    six.assertRaisesRegex(self, text_format.ParseError,
-                          (r'1:23 : Enum type "\w+.TestAllTypes.NestedEnum" '
-                           r'has no value with number 100.'), text_format.Parse,
-                          text, message)
-
   def testParseBadIntValue(self, message_module):
     message = message_module.TestAllTypes()
     text = 'optional_int32: bork'
@@ -920,6 +941,14 @@ class Proto2Tests(TextFormatBase):
         '1:2 : Message type "protobuf_unittest.TestAllTypes" does not have '
         'extensions.'), text_format.Parse, text, message)
 
+  def testParseNumericUnknownEnum(self):
+    message = unittest_pb2.TestAllTypes()
+    text = 'optional_nested_enum: 100'
+    six.assertRaisesRegex(self, text_format.ParseError,
+                          (r'1:23 : Enum type "\w+.TestAllTypes.NestedEnum" '
+                           r'has no value with number 100.'), text_format.Parse,
+                          text, message)
+
   def testMergeDuplicateExtensionScalars(self):
     message = unittest_pb2.TestAllExtensions()
     text = ('[protobuf_unittest.optional_int32_extension]: 42 '
@@ -1113,13 +1142,8 @@ class Proto3Tests(unittest.TestCase):
     message2 = unittest_proto3_arena_pb2.TestAllTypes()
     message.optional_nested_enum = 999
     text_string = text_format.MessageToString(message)
-    # TODO(jieluo): proto3 should support numeric unknown enum.
-    with self.assertRaises(text_format.ParseError) as e:
-      text_format.Parse(text_string, message2)
-      self.assertEqual(999, message2.optional_nested_enum)
-    self.assertEqual(str(e.exception),
-                     '1:23 : Enum type "proto3_arena_unittest.TestAllTypes.'
-                     'NestedEnum" has no value with number 999.')
+    text_format.Parse(text_string, message2)
+    self.assertEqual(999, message2.optional_nested_enum)
 
   def testMergeExpandedAny(self):
     message = any_test_pb2.TestAny()

python/google/protobuf/internal/unknown_fields_test.py

@@ -92,7 +92,6 @@ class UnknownFieldsTest(BaseTestCase):
     # Verify that proto3 unknown fields behavior.
     default_preserve = (api_implementation
                         .GetPythonProto3PreserveUnknownsDefault())
-    self.assertEqual(False, default_preserve)
     self.expectSerializeProto3(default_preserve)
     api_implementation.SetPythonProto3PreserveUnknownsDefault(
         not default_preserve)

python/google/protobuf/internal/well_known_types.py

@@ -40,6 +40,7 @@ This files defines well known classes which need extra maintenance including:
 
 __author__ = 'jieluo@google.com (Jie Luo)'
 
+import collections
 from datetime import datetime
 from datetime import timedelta
 import six
@@ -67,13 +68,14 @@ class ParseError(Error):
 class Any(object):
   """Class for Any Message type."""
 
-  def Pack(self, msg, type_url_prefix='type.googleapis.com/'):
+  def Pack(self, msg, type_url_prefix='type.googleapis.com/',
+           deterministic=None):
     """Packs the specified message into current Any message."""
     if len(type_url_prefix) < 1 or type_url_prefix[-1] != '/':
       self.type_url = '%s/%s' % (type_url_prefix, msg.DESCRIPTOR.full_name)
     else:
       self.type_url = '%s%s' % (type_url_prefix, msg.DESCRIPTOR.full_name)
-    self.value = msg.SerializeToString()
+    self.value = msg.SerializeToString(deterministic=deterministic)
 
   def Unpack(self, msg):
     """Unpacks the current Any message into specified message."""
@@ -734,9 +736,30 @@ class Struct(object):
   def __getitem__(self, key):
     return _GetStructValue(self.fields[key])
 
+  def __contains__(self, item):
+    return item in self.fields
+
   def __setitem__(self, key, value):
     _SetStructValue(self.fields[key], value)
 
+  def __delitem__(self, key):
+    del self.fields[key]
+
+  def __len__(self):
+    return len(self.fields)
+
+  def __iter__(self):
+    return iter(self.fields)
+
+  def keys(self):  # pylint: disable=invalid-name
+    return self.fields.keys()
+
+  def values(self):  # pylint: disable=invalid-name
+    return [self[key] for key in self]
+
+  def items(self):  # pylint: disable=invalid-name
+    return [(key, self[key]) for key in self]
+
   def get_or_create_list(self, key):
     """Returns a list for this key, creating if it didn't exist already."""
     if not self.fields[key].HasField('list_value'):
@@ -755,6 +778,8 @@ class Struct(object):
     for key, value in dictionary.items():
       _SetStructValue(self.fields[key], value)
 
+collections.MutableMapping.register(Struct)
+
 
 class ListValue(object):
   """Class for ListValue message type."""
@@ -776,6 +801,9 @@ class ListValue(object):
   def __setitem__(self, index, value):
     _SetStructValue(self.values.__getitem__(index), value)
 
+  def __delitem__(self, key):
+    del self.values[key]
+
   def items(self):
     for i in range(len(self)):
       yield self[i]
@@ -794,6 +822,8 @@ class ListValue(object):
     list_value.Clear()
     return list_value
 
+collections.MutableSequence.register(ListValue)
+
 
 WKTBASES = {
     'google.protobuf.Any': Any,

python/google/protobuf/internal/well_known_types_test.py

@@ -34,6 +34,7 @@
 
 __author__ = 'jieluo@google.com (Jie Luo)'
 
+import collections
 from datetime import datetime
 
 try:
@@ -667,6 +668,8 @@ class StructTest(unittest.TestCase):
 
   def testStruct(self):
     struct = struct_pb2.Struct()
+    self.assertIsInstance(struct, collections.Mapping)
+    self.assertEqual(0, len(struct))
     struct_class = struct.__class__
 
     struct['key1'] = 5
@@ -674,11 +677,13 @@ class StructTest(unittest.TestCase):
     struct['key3'] = True
     struct.get_or_create_struct('key4')['subkey'] = 11.0
     struct_list = struct.get_or_create_list('key5')
+    self.assertIsInstance(struct_list, collections.Sequence)
     struct_list.extend([6, 'seven', True, False, None])
     struct_list.add_struct()['subkey2'] = 9
     struct['key6'] = {'subkey': {}}
     struct['key7'] = [2, False]
 
+    self.assertEqual(7, len(struct))
     self.assertTrue(isinstance(struct, well_known_types.Struct))
     self.assertEqual(5, struct['key1'])
     self.assertEqual('abc', struct['key2'])
@@ -696,6 +701,20 @@ class StructTest(unittest.TestCase):
     struct2.ParseFromString(serialized)
 
     self.assertEqual(struct, struct2)
+    for key, value in struct.items():
+      self.assertIn(key, struct)
+      self.assertIn(key, struct2)
+      self.assertEqual(value, struct2[key])
+
+    self.assertEqual(7, len(struct.keys()))
+    self.assertEqual(7, len(struct.values()))
+    for key in struct.keys():
+      self.assertIn(key, struct)
+      self.assertIn(key, struct2)
+      self.assertEqual(struct[key], struct2[key])
+
+    item = (next(iter(struct.keys())), next(iter(struct.values())))
+    self.assertEqual(item, next(iter(struct.items())))
 
     self.assertTrue(isinstance(struct2, well_known_types.Struct))
     self.assertEqual(5, struct2['key1'])
@@ -756,6 +775,16 @@ class StructTest(unittest.TestCase):
     empty_struct = list2[1]
     self.assertEqual({}, dict(empty_struct.fields))
 
+    self.assertEqual(9, len(struct))
+    del struct['key3']
+    del struct['key4']
+    self.assertEqual(7, len(struct))
+    self.assertEqual(6, len(struct['key5']))
+    del struct['key5'][1]
+    self.assertEqual(5, len(struct['key5']))
+    self.assertEqual([6, True, False, None, inner_struct],
+                     list(struct['key5'].items()))
+
   def testMergeFrom(self):
     struct = struct_pb2.Struct()
     struct_class = struct.__class__
@@ -863,6 +892,20 @@ class AnyTest(unittest.TestCase):
     self.assertTrue(msg.Unpack(unpacked_message))
     self.assertEqual(submessage, unpacked_message)
 
+  def testPackDeterministic(self):
+    submessage = any_test_pb2.TestAny()
+    for i in range(10):
+      submessage.map_value[str(i)] = i * 2
+    msg = any_pb2.Any()
+    msg.Pack(submessage, deterministic=True)
+    serialized = msg.SerializeToString(deterministic=True)
+    golden = (b'\n4type.googleapis.com/google.protobuf.internal.TestAny\x12F'
+              b'\x1a\x05\n\x010\x10\x00\x1a\x05\n\x011\x10\x02\x1a\x05\n\x01'
+              b'2\x10\x04\x1a\x05\n\x013\x10\x06\x1a\x05\n\x014\x10\x08\x1a'
+              b'\x05\n\x015\x10\n\x1a\x05\n\x016\x10\x0c\x1a\x05\n\x017\x10'
+              b'\x0e\x1a\x05\n\x018\x10\x10\x1a\x05\n\x019\x10\x12')
+    self.assertEqual(golden, serialized)
+
 
 if __name__ == '__main__':
   unittest.main()

python/google/protobuf/json_format.py

@@ -251,6 +251,8 @@ class _Printer(object):
       if enum_value is not None:
         return enum_value.name
       else:
+        if field.file.syntax == 'proto3':
+          return value
         raise SerializeToJsonError('Enum field contains an integer value '
                                    'which can not mapped to an enum value.')
     elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING:
@@ -675,6 +677,9 @@ def _ConvertScalarFieldValue(value, field, require_str=False):
         raise ParseError('Invalid enum value {0} for enum type {1}.'.format(
             value, field.enum_type.full_name))
       if enum_value is None:
+        if field.file.syntax == 'proto3':
+          # Proto3 accepts unknown enums.
+          return number
         raise ParseError('Invalid enum value {0} for enum type {1}.'.format(
             value, field.enum_type.full_name))
     return enum_value.number

python/google/protobuf/message_factory.py

@@ -130,13 +130,22 @@ def GetMessages(file_protos):
   """Builds a dictionary of all the messages available in a set of files.
 
   Args:
-    file_protos: A sequence of file protos to build messages out of.
+    file_protos: Iterable of FileDescriptorProto to build messages out of.
 
   Returns:
     A dictionary mapping proto names to the message classes. This will include
     any dependent messages as well as any messages defined in the same file as
     a specified message.
   """
-  for file_proto in file_protos:
+  # The cpp implementation of the protocol buffer library requires to add the
+  # message in topological order of the dependency graph.
+  file_by_name = {file_proto.name: file_proto for file_proto in file_protos}
+  def _AddFile(file_proto):
+    for dependency in file_proto.dependency:
+      if dependency in file_by_name:
+        # Remove from elements to be visited, in order to cut cycles.
+        _AddFile(file_by_name.pop(dependency))
     _FACTORY.pool.Add(file_proto)
+  while file_by_name:
+    _AddFile(file_by_name.popitem()[1])
   return _FACTORY.GetMessages([file_proto.name for file_proto in file_protos])

python/google/protobuf/pyext/extension_dict.cc

@@ -32,6 +32,10 @@
 // Author: tibell@google.com (Johan Tibell)
 
 #include <google/protobuf/pyext/extension_dict.h>
+#include <memory>
+#ifndef _SHARED_PTR_H
+#include <google/protobuf/stubs/shared_ptr.h>
+#endif
 
 #include <google/protobuf/stubs/logging.h>
 #include <google/protobuf/stubs/common.h>
@@ -45,7 +49,6 @@
 #include <google/protobuf/pyext/repeated_composite_container.h>
 #include <google/protobuf/pyext/repeated_scalar_container.h>
 #include <google/protobuf/pyext/scoped_pyobject_ptr.h>
-#include <google/protobuf/stubs/shared_ptr.h>
 
 #if PY_MAJOR_VERSION >= 3
   #if PY_VERSION_HEX < 0x03030000

python/google/protobuf/pyext/message.cc

@@ -2065,6 +2065,11 @@ static PyObject* MergeFromString(CMessage* self, PyObject* arg) {
   input.SetExtensionRegistry(factory->pool->pool, factory->message_factory);
   bool success = self->message->MergePartialFromCodedStream(&input);
   if (success) {
+    if (!input.ConsumedEntireMessage()) {
+      // TODO(jieluo): Raise error and return NULL instead.
+      // b/27494216
+      PyErr_Warn(NULL, "Unexpected end-group tag: Not all data was converted");
+    }
     return PyInt_FromLong(input.CurrentPosition());
   } else {
     PyErr_Format(DecodeError_class, "Error parsing message");

python/google/protobuf/text_format.py

@@ -485,7 +485,10 @@ def Parse(text,
     ParseError: On text parsing problems.
   """
   if not isinstance(text, str):
-    text = text.decode('utf-8')
+    if six.PY3:
+      text = text.decode('utf-8')
+    else:
+      text = text.encode('utf-8')
   return ParseLines(text.split('\n'),
                     message,
                     allow_unknown_extension,
@@ -517,6 +520,11 @@ def Merge(text,
   Raises:
     ParseError: On text parsing problems.
   """
+  if not isinstance(text, str):
+    if six.PY3:
+      text = text.decode('utf-8')
+    else:
+      text = text.encode('utf-8')
   return MergeLines(
       text.split('\n'),
       message,
@@ -1559,6 +1567,11 @@ def ParseEnum(field, value):
                        (enum_descriptor.full_name, value))
   else:
     # Numeric value.
+    if hasattr(field.file, 'syntax'):
+      # Attribute is checked for compatibility.
+      if field.file.syntax == 'proto3':
+        # Proto3 accept numeric unknown enums.
+        return number
     enum_value = enum_descriptor.values_by_number.get(number, None)
     if enum_value is None:
       raise ValueError('Enum type "%s" has no value with number %d.' %

src/Makefile.am

@@ -813,6 +813,8 @@ protobuf_test_SOURCES =                                        \
   google/protobuf/io/printer_unittest.cc                       \
   google/protobuf/io/tokenizer_unittest.cc                     \
   google/protobuf/io/zero_copy_stream_unittest.cc              \
+  google/protobuf/compiler/annotation_test_util.h              \
+  google/protobuf/compiler/annotation_test_util.cc             \
   google/protobuf/compiler/command_line_interface_unittest.cc  \
   google/protobuf/compiler/importer_unittest.cc                \
   google/protobuf/compiler/mock_code_generator.cc              \

src/google/protobuf/any.pb.cc

@@ -14,6 +14,10 @@
 #include <google/protobuf/generated_message_reflection.h>
 #include <google/protobuf/reflection_ops.h>
 #include <google/protobuf/wire_format.h>
+// This is a temporary google only hack
+#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
+#include "third_party/protobuf/version.h"
+#endif
 // @@protoc_insertion_point(includes)
 namespace google {
 namespace protobuf {
@@ -28,7 +32,11 @@ namespace protobuf_google_2fprotobuf_2fany_2eproto {
 void InitDefaultsAnyImpl() {
   GOOGLE_PROTOBUF_VERIFY_VERSION;
 
+#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
+  ::google::protobuf::internal::InitProtobufDefaultsForceUnique();
+#else
   ::google::protobuf::internal::InitProtobufDefaults();
+#endif  // GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
   {
     void* ptr = &::google::protobuf::_Any_default_instance_;
     new (ptr) ::google::protobuf::Any();

src/google/protobuf/api.pb.cc

@@ -14,6 +14,10 @@
 #include <google/protobuf/generated_message_reflection.h>
 #include <google/protobuf/reflection_ops.h>
 #include <google/protobuf/wire_format.h>
+// This is a temporary google only hack
+#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
+#include "third_party/protobuf/version.h"
+#endif
 // @@protoc_insertion_point(includes)
 namespace google {
 namespace protobuf {
@@ -38,7 +42,11 @@ namespace protobuf_google_2fprotobuf_2fapi_2eproto {
 void InitDefaultsApiImpl() {
   GOOGLE_PROTOBUF_VERIFY_VERSION;
 
+#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
+  ::google::protobuf::internal::InitProtobufDefaultsForceUnique();
+#else
   ::google::protobuf::internal::InitProtobufDefaults();
+#endif  // GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
   protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaultsMethod();
   protobuf_google_2fprotobuf_2ftype_2eproto::InitDefaultsOption();
   protobuf_google_2fprotobuf_2fsource_5fcontext_2eproto::InitDefaultsSourceContext();
@@ -59,7 +67,11 @@ void InitDefaultsApi() {
 void InitDefaultsMethodImpl() {
   GOOGLE_PROTOBUF_VERIFY_VERSION;
 
+#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
+  ::google::protobuf::internal::InitProtobufDefaultsForceUnique();
+#else
   ::google::protobuf::internal::InitProtobufDefaults();
+#endif  // GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
   protobuf_google_2fprotobuf_2ftype_2eproto::InitDefaultsOption();
   {
     void* ptr = &::google::protobuf::_Method_default_instance_;
@@ -77,7 +89,11 @@ void InitDefaultsMethod() {
 void InitDefaultsMixinImpl() {
   GOOGLE_PROTOBUF_VERIFY_VERSION;
 
+#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
+  ::google::protobuf::internal::InitProtobufDefaultsForceUnique();
+#else
   ::google::protobuf::internal::InitProtobufDefaults();
+#endif  // GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
   {
     void* ptr = &::google::protobuf::_Mixin_default_instance_;
     new (ptr) ::google::protobuf::Mixin();
@@ -208,6 +224,15 @@ void Api::InitAsDefaultInstance() {
   ::google::protobuf::_Api_default_instance_._instance.get_mutable()->source_context_ = const_cast< ::google::protobuf::SourceContext*>(
       ::google::protobuf::SourceContext::internal_default_instance());
 }
+void Api::clear_options() {
+  options_.Clear();
+}
+void Api::clear_source_context() {
+  if (GetArenaNoVirtual() == NULL && source_context_ != NULL) {
+    delete source_context_;
+  }
+  source_context_ = NULL;
+}
 #if !defined(_MSC_VER) || _MSC_VER >= 1900
 const int Api::kNameFieldNumber;
 const int Api::kMethodsFieldNumber;
@@ -343,8 +368,7 @@ bool Api::MergePartialFromCodedStream(
       case 2: {
         if (static_cast< ::google::protobuf::uint8>(tag) ==
             static_cast< ::google::protobuf::uint8>(18u /* 18 & 0xFF */)) {
-          DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual(
-                input, add_methods()));
+          DO_(::google::protobuf::internal::WireFormatLite::ReadMessage(input, add_methods()));
         } else {
           goto handle_unusual;
         }
@@ -355,8 +379,7 @@ bool Api::MergePartialFromCodedStream(
       case 3: {
         if (static_cast< ::google::protobuf::uint8>(tag) ==
             static_cast< ::google::protobuf::uint8>(26u /* 26 & 0xFF */)) {
-          DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual(
-                input, add_options()));
+          DO_(::google::protobuf::internal::WireFormatLite::ReadMessage(input, add_options()));
         } else {
           goto handle_unusual;
         }
@@ -383,7 +406,7 @@ bool Api::MergePartialFromCodedStream(
       case 5: {
         if (static_cast< ::google::protobuf::uint8>(tag) ==
             static_cast< ::google::protobuf::uint8>(42u /* 42 & 0xFF */)) {
-          DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual(
+          DO_(::google::protobuf::internal::WireFormatLite::ReadMessage(
                input, mutable_source_context()));
         } else {
           goto handle_unusual;
@@ -395,8 +418,7 @@ bool Api::MergePartialFromCodedStream(
       case 6: {
         if (static_cast< ::google::protobuf::uint8>(tag) ==
             static_cast< ::google::protobuf::uint8>(50u /* 50 & 0xFF */)) {
-          DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual(
-                input, add_mixins()));
+          DO_(::google::protobuf::internal::WireFormatLite::ReadMessage(input, add_mixins()));
         } else {
           goto handle_unusual;
         }
@@ -526,7 +548,7 @@ void Api::SerializeWithCachedSizes(
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->methods_size()); i < n; i++) {
     target = ::google::protobuf::internal::WireFormatLite::
-      InternalWriteMessageNoVirtualToArray(
+      InternalWriteMessageToArray(
         2, this->methods(static_cast<int>(i)), deterministic, target);
   }
 
@@ -534,7 +556,7 @@ void Api::SerializeWithCachedSizes(
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->options_size()); i < n; i++) {
     target = ::google::protobuf::internal::WireFormatLite::
-      InternalWriteMessageNoVirtualToArray(
+      InternalWriteMessageToArray(
         3, this->options(static_cast<int>(i)), deterministic, target);
   }
 
@@ -552,7 +574,7 @@ void Api::SerializeWithCachedSizes(
   // .google.protobuf.SourceContext source_context = 5;
   if (this->has_source_context()) {
     target = ::google::protobuf::internal::WireFormatLite::
-      InternalWriteMessageNoVirtualToArray(
+      InternalWriteMessageToArray(
         5, *this->source_context_, deterministic, target);
   }
 
@@ -560,7 +582,7 @@ void Api::SerializeWithCachedSizes(
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->mixins_size()); i < n; i++) {
     target = ::google::protobuf::internal::WireFormatLite::
-      InternalWriteMessageNoVirtualToArray(
+      InternalWriteMessageToArray(
         6, this->mixins(static_cast<int>(i)), deterministic, target);
   }
 
@@ -593,7 +615,7 @@ size_t Api::ByteSizeLong() const {
     total_size += 1UL * count;
     for (unsigned int i = 0; i < count; i++) {
       total_size +=
-        ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual(
+        ::google::protobuf::internal::WireFormatLite::MessageSize(
           this->methods(static_cast<int>(i)));
     }
   }
@@ -604,7 +626,7 @@ size_t Api::ByteSizeLong() const {
     total_size += 1UL * count;
     for (unsigned int i = 0; i < count; i++) {
       total_size +=
-        ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual(
+        ::google::protobuf::internal::WireFormatLite::MessageSize(
           this->options(static_cast<int>(i)));
     }
   }
@@ -615,7 +637,7 @@ size_t Api::ByteSizeLong() const {
     total_size += 1UL * count;
     for (unsigned int i = 0; i < count; i++) {
       total_size +=
-        ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual(
+        ::google::protobuf::internal::WireFormatLite::MessageSize(
           this->mixins(static_cast<int>(i)));
     }
   }
@@ -637,7 +659,7 @@ size_t Api::ByteSizeLong() const {
   // .google.protobuf.SourceContext source_context = 5;
   if (this->has_source_context()) {
     total_size += 1 +
-      ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual(
+      ::google::protobuf::internal::WireFormatLite::MessageSize(
         *this->source_context_);
   }
 
@@ -740,6 +762,9 @@ void Api::InternalSwap(Api* other) {
 
 void Method::InitAsDefaultInstance() {
 }
+void Method::clear_options() {
+  options_.Clear();
+}
 #if !defined(_MSC_VER) || _MSC_VER >= 1900
 const int Method::kNameFieldNumber;
 const int Method::kRequestTypeUrlFieldNumber;
@@ -932,8 +957,7 @@ bool Method::MergePartialFromCodedStream(
       case 6: {
         if (static_cast< ::google::protobuf::uint8>(tag) ==
             static_cast< ::google::protobuf::uint8>(50u /* 50 & 0xFF */)) {
-          DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual(
-                input, add_options()));
+          DO_(::google::protobuf::internal::WireFormatLite::ReadMessage(input, add_options()));
         } else {
           goto handle_unusual;
         }
@@ -1095,7 +1119,7 @@ void Method::SerializeWithCachedSizes(
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->options_size()); i < n; i++) {
     target = ::google::protobuf::internal::WireFormatLite::
-      InternalWriteMessageNoVirtualToArray(
+      InternalWriteMessageToArray(
         6, this->options(static_cast<int>(i)), deterministic, target);
   }
 
@@ -1128,7 +1152,7 @@ size_t Method::ByteSizeLong() const {
     total_size += 1UL * count;
     for (unsigned int i = 0; i < count; i++) {
       total_size +=
-        ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual(
+        ::google::protobuf::internal::WireFormatLite::MessageSize(
           this->options(static_cast<int>(i)));
     }
   }

src/google/protobuf/api.pb.h

@@ -225,8 +225,8 @@ class LIBPROTOBUF_EXPORT Api : public ::google::protobuf::Message /* @@protoc_in
   void clear_source_context();
   static const int kSourceContextFieldNumber = 5;
   const ::google::protobuf::SourceContext& source_context() const;
-  ::google::protobuf::SourceContext* mutable_source_context();
   ::google::protobuf::SourceContext* release_source_context();
+  ::google::protobuf::SourceContext* mutable_source_context();
   void set_allocated_source_context(::google::protobuf::SourceContext* source_context);
 
   // .google.protobuf.Syntax syntax = 7;
@@ -641,9 +641,6 @@ Api::methods() const {
 inline int Api::options_size() const {
   return options_.size();
 }
-inline void Api::clear_options() {
-  options_.Clear();
-}
 inline const ::google::protobuf::Option& Api::options(int index) const {
   // @@protoc_insertion_point(field_get:google.protobuf.Api.options)
   return options_.Get(index);
@@ -724,16 +721,19 @@ inline void Api::set_allocated_version(::std::string* version) {
 inline bool Api::has_source_context() const {
   return this != internal_default_instance() && source_context_ != NULL;
 }
-inline void Api::clear_source_context() {
-  if (GetArenaNoVirtual() == NULL && source_context_ != NULL) delete source_context_;
-  source_context_ = NULL;
-}
 inline const ::google::protobuf::SourceContext& Api::source_context() const {
   const ::google::protobuf::SourceContext* p = source_context_;
   // @@protoc_insertion_point(field_get:google.protobuf.Api.source_context)
   return p != NULL ? *p : *reinterpret_cast<const ::google::protobuf::SourceContext*>(
       &::google::protobuf::_SourceContext_default_instance_);
 }
+inline ::google::protobuf::SourceContext* Api::release_source_context() {
+  // @@protoc_insertion_point(field_release:google.protobuf.Api.source_context)
+  
+  ::google::protobuf::SourceContext* temp = source_context_;
+  source_context_ = NULL;
+  return temp;
+}
 inline ::google::protobuf::SourceContext* Api::mutable_source_context() {
   
   if (source_context_ == NULL) {
@@ -742,21 +742,22 @@ inline ::google::protobuf::SourceContext* Api::mutable_source_context() {
   // @@protoc_insertion_point(field_mutable:google.protobuf.Api.source_context)
   return source_context_;
 }
-inline ::google::protobuf::SourceContext* Api::release_source_context() {
-  // @@protoc_insertion_point(field_release:google.protobuf.Api.source_context)
-  
-  ::google::protobuf::SourceContext* temp = source_context_;
-  source_context_ = NULL;
-  return temp;
-}
 inline void Api::set_allocated_source_context(::google::protobuf::SourceContext* source_context) {
-  delete source_context_;
-  source_context_ = source_context;
+  ::google::protobuf::Arena* message_arena = GetArenaNoVirtual();
+  if (message_arena == NULL) {
+    delete reinterpret_cast< ::google::protobuf::MessageLite*>(source_context_);
+  }
   if (source_context) {
+    ::google::protobuf::Arena* submessage_arena = NULL;
+    if (message_arena != submessage_arena) {
+      source_context = ::google::protobuf::internal::GetOwnedMessage(
+          message_arena, source_context, submessage_arena);
+    }
     
   } else {
     
   }
+  source_context_ = source_context;
   // @@protoc_insertion_point(field_set_allocated:google.protobuf.Api.source_context)
 }
 
@@ -999,9 +1000,6 @@ inline void Method::set_response_streaming(bool value) {
 inline int Method::options_size() const {
   return options_.size();
 }
-inline void Method::clear_options() {
-  options_.Clear();
-}
 inline const ::google::protobuf::Option& Method::options(int index) const {
   // @@protoc_insertion_point(field_get:google.protobuf.Method.options)
   return options_.Get(index);

src/google/protobuf/arena.cc

@@ -66,90 +66,78 @@ GOOGLE_THREAD_LOCAL ArenaImpl::ThreadCache ArenaImpl::thread_cache_ = {-1, NULL}
 
 void ArenaImpl::Init() {
   lifecycle_id_ = lifecycle_id_generator_.GetNext();
-  blocks_ = 0;
-  hint_ = 0;
-  space_allocated_ = 0;
-  owns_first_block_ = true;
-
-  if (options_.initial_block != NULL && options_.initial_block_size > 0) {
-    GOOGLE_CHECK_GE(options_.initial_block_size, sizeof(Block))
-        << ": Initial block size too small for header.";
-
-    // Add first unowned block to list.
-    Block* first_block = reinterpret_cast<Block*>(options_.initial_block);
-    first_block->size = options_.initial_block_size;
-    first_block->pos = kHeaderSize;
-    first_block->next = NULL;
-    first_block->cleanup = NULL;
+  google::protobuf::internal::NoBarrier_Store(&hint_, 0);
+  google::protobuf::internal::NoBarrier_Store(&threads_, 0);
+
+  if (initial_block_) {
     // Thread which calls Init() owns the first block. This allows the
-    // single-threaded case to allocate on the first block without taking any
-    // locks.
-    first_block->owner = &thread_cache();
-    AddBlockInternal(first_block);
-    CacheBlock(first_block);
-    owns_first_block_ = false;
+    // single-threaded case to allocate on the first block without having to
+    // perform atomic operations.
+    InitBlock(initial_block_, &thread_cache(), options_.initial_block_size);
+    ThreadInfo* info = NewThreadInfo(initial_block_);
+    info->next = NULL;
+    google::protobuf::internal::NoBarrier_Store(&threads_,
+                                  reinterpret_cast<google::protobuf::internal::AtomicWord>(info));
+    google::protobuf::internal::NoBarrier_Store(&space_allocated_,
+                                  options_.initial_block_size);
+    CacheBlock(initial_block_);
+  } else {
+    google::protobuf::internal::NoBarrier_Store(&space_allocated_, 0);
   }
 }
 
-ArenaImpl::~ArenaImpl() { ResetInternal(); }
-
-uint64 ArenaImpl::Reset() {
-  // Invalidate any ThreadCaches pointing to any blocks we just destroyed.
-  lifecycle_id_ = lifecycle_id_generator_.GetNext();
-  return ResetInternal();
+ArenaImpl::~ArenaImpl() {
+  // Have to do this in a first pass, because some of the destructors might
+  // refer to memory in other blocks.
+  CleanupList();
+  FreeBlocks();
 }
 
-uint64 ArenaImpl::ResetInternal() {
-  Block* head =
-      reinterpret_cast<Block*>(google::protobuf::internal::NoBarrier_Load(&blocks_));
-  CleanupList(head);
-  uint64 space_allocated = FreeBlocks(head);
+uint64 ArenaImpl::Reset() {
+  // Have to do this in a first pass, because some of the destructors might
+  // refer to memory in other blocks.
+  CleanupList();
+  uint64 space_allocated = FreeBlocks();
+  Init();
 
   return space_allocated;
 }
 
 ArenaImpl::Block* ArenaImpl::NewBlock(void* me, Block* my_last_block,
-                                      size_t min_bytes, size_t start_block_size,
-                                      size_t max_block_size) {
+                                      size_t min_bytes) {
   size_t size;
   if (my_last_block != NULL) {
     // Double the current block size, up to a limit.
-    size = std::min(2 * my_last_block->size, max_block_size);
+    size = std::min(2 * my_last_block->size, options_.max_block_size);
   } else {
-    size = start_block_size;
+    size = options_.start_block_size;
   }
   // Verify that min_bytes + kHeaderSize won't overflow.
   GOOGLE_CHECK_LE(min_bytes, std::numeric_limits<size_t>::max() - kHeaderSize);
   size = std::max(size, kHeaderSize + min_bytes);
 
   Block* b = reinterpret_cast<Block*>(options_.block_alloc(size));
+  InitBlock(b, me, size);
+  google::protobuf::internal::NoBarrier_AtomicIncrement(&space_allocated_, size);
+  return b;
+}
+
+void ArenaImpl::InitBlock(Block* b, void *me, size_t size) {
   b->pos = kHeaderSize;
   b->size = size;
   b->owner = me;
-  b->cleanup = NULL;
+  b->next = NULL;
 #ifdef ADDRESS_SANITIZER
   // Poison the rest of the block for ASAN. It was unpoisoned by the underlying
   // malloc but it's not yet usable until we return it as part of an allocation.
   ASAN_POISON_MEMORY_REGION(
       reinterpret_cast<char*>(b) + b->pos, b->size - b->pos);
 #endif  // ADDRESS_SANITIZER
-  AddBlock(b);
-  return b;
-}
-
-void ArenaImpl::AddBlock(Block* b) {
-  MutexLock l(&blocks_lock_);
-  AddBlockInternal(b);
 }
 
-void ArenaImpl::AddBlockInternal(Block* b) {
-  b->next = reinterpret_cast<Block*>(google::protobuf::internal::NoBarrier_Load(&blocks_));
-  google::protobuf::internal::Release_Store(&blocks_, reinterpret_cast<google::protobuf::internal::AtomicWord>(b));
-  space_allocated_ += b->size;
-}
-
-ArenaImpl::Block* ArenaImpl::ExpandCleanupList(Block* b) {
-  size_t size = b->cleanup ? b->cleanup->size * 2 : kMinCleanupListElements;
+ArenaImpl::CleanupChunk* ArenaImpl::ExpandCleanupList(CleanupChunk* cleanup,
+                                                      Block* b) {
+  size_t size = cleanup ? cleanup->size * 2 : kMinCleanupListElements;
   size = std::min(size, kMaxCleanupListElements);
   size_t bytes = internal::AlignUpTo8(CleanupChunk::SizeOf(size));
   if (b->avail() < bytes) {
@@ -157,24 +145,25 @@ ArenaImpl::Block* ArenaImpl::ExpandCleanupList(Block* b) {
   }
   CleanupChunk* list =
       reinterpret_cast<CleanupChunk*>(AllocFromBlock(b, bytes));
-  list->next = b->cleanup;
+  list->next = b->thread_info->cleanup;
   list->size = size;
   list->len = 0;
-  b->cleanup = list;
-  return b;
+  b->thread_info->cleanup = list;
+  return list;
 }
 
 inline GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
 void ArenaImpl::AddCleanupInBlock(
-    Block* b, void* elem, void (*cleanup)(void*)) {
-  if (b->cleanup == NULL || b->cleanup->len == b->cleanup->size) {
-    b = ExpandCleanupList(b);
+    Block* b, void* elem, void (*func)(void*)) {
+  CleanupChunk* cleanup = b->thread_info->cleanup;
+  if (cleanup == NULL || cleanup->len == cleanup->size) {
+    cleanup = ExpandCleanupList(cleanup, b);
   }
 
-  CleanupNode* node = &b->cleanup->nodes[b->cleanup->len++];
+  CleanupNode* node = &cleanup->nodes[cleanup->len++];
 
   node->elem = elem;
-  node->cleanup = cleanup;
+  node->cleanup = func;
 }
 
 void ArenaImpl::AddCleanup(void* elem, void (*cleanup)(void*)) {
@@ -240,92 +229,82 @@ void* ArenaImpl::AllocFromBlock(Block* b, size_t n) {
 
 ArenaImpl::Block* ArenaImpl::GetBlockSlow(void* me, Block* my_full_block,
                                           size_t n) {
-  Block* b = FindBlock(me);  // Find block owned by me.
-  if (b == NULL || b->avail() < n) {
-    b = NewBlock(me, b, n, options_.start_block_size, options_.max_block_size);
-
-    // Try to steal the cleanup list from my_full_block.  It's too full for this
-    // allocation, but it might have space left in its cleanup list and there's
-    // no reason to waste that memory.
-    if (my_full_block) {
-      GOOGLE_DCHECK_EQ(my_full_block->owner, me);
-      GOOGLE_DCHECK(b->cleanup == NULL);
-      b->cleanup = my_full_block->cleanup;
-      my_full_block->cleanup = NULL;
-    }
+  ThreadInfo* info =
+      my_full_block ? my_full_block->thread_info : GetThreadInfo(me, n);
+  GOOGLE_DCHECK(info != NULL);
+  Block* b = info->head;
+  if (b->avail() < n) {
+    Block* new_b = NewBlock(me, b, n);
+    new_b->thread_info = info;
+    new_b->next = b;
+    info->head = new_b;
+    b = new_b;
   }
   CacheBlock(b);
   return b;
 }
 
 uint64 ArenaImpl::SpaceAllocated() const {
-  MutexLock l(&blocks_lock_);
-  return space_allocated_;
+  return google::protobuf::internal::NoBarrier_Load(&space_allocated_);
 }
 
 uint64 ArenaImpl::SpaceUsed() const {
+  ThreadInfo* info =
+      reinterpret_cast<ThreadInfo*>(google::protobuf::internal::Acquire_Load(&threads_));
   uint64 space_used = 0;
-  Block* b = reinterpret_cast<Block*>(google::protobuf::internal::NoBarrier_Load(&blocks_));
-  while (b != NULL) {
-    space_used += (b->pos - kHeaderSize);
-    b = b->next;
+
+  for ( ; info; info = info->next) {
+    // Remove the overhead of the ThreadInfo itself.
+    space_used -= sizeof(ThreadInfo);
+    for (Block* b = info->head; b; b = b->next) {
+      space_used += (b->pos - kHeaderSize);
+    }
   }
+
   return space_used;
 }
 
-uint64 ArenaImpl::FreeBlocks(Block* head) {
+uint64 ArenaImpl::FreeBlocks() {
   uint64 space_allocated = 0;
-  Block* first_block = NULL;
-  Block* b = head;
+  // By omitting an Acquire barrier we ensure that any user code that doesn't
+  // properly synchronize Reset() or the destructor will throw a TSAN warning.
+  ThreadInfo* info =
+      reinterpret_cast<ThreadInfo*>(google::protobuf::internal::NoBarrier_Load(&threads_));
+
+  while (info) {
+    // This is inside the block we are freeing, so we need to read it now.
+    ThreadInfo* next_info = info->next;
+    for (Block* b = info->head; b; ) {
+      // This is inside the block we are freeing, so we need to read it now.
+      Block* next_block = b->next;
+      space_allocated += (b->size);
 
-  while (b != NULL) {
-    space_allocated += (b->size);
-    Block* next = b->next;
-    if (next != NULL) {
 #ifdef ADDRESS_SANITIZER
       // This memory was provided by the underlying allocator as unpoisoned, so
       // return it in an unpoisoned state.
       ASAN_UNPOISON_MEMORY_REGION(reinterpret_cast<char*>(b), b->size);
 #endif  // ADDRESS_SANITIZER
-      options_.block_dealloc(b, b->size);
-    } else {
-      if (owns_first_block_) {
-#ifdef ADDRESS_SANITIZER
-        // This memory was provided by the underlying allocator as unpoisoned,
-        // so return it in an unpoisoned state.
-        ASAN_UNPOISON_MEMORY_REGION(reinterpret_cast<char*>(b), b->size);
-#endif  // ADDRESS_SANITIZER
+
+      if (b != initial_block_) {
         options_.block_dealloc(b, b->size);
-      } else {
-        // User passed in the first block, skip free'ing the memory.
-        first_block = b;
       }
+
+      b = next_block;
     }
-    b = next;
-  }
-  blocks_ = 0;
-  hint_ = 0;
-  space_allocated_ = 0;
-  if (!owns_first_block_) {
-    // Make the first block that was passed in through ArenaOptions
-    // available for reuse.
-    first_block->pos = kHeaderSize;
-    first_block->cleanup = NULL;
-    // Thread which calls Reset() owns the first block. This allows the
-    // single-threaded case to allocate on the first block without taking any
-    // locks.
-    first_block->owner = &thread_cache();
-    AddBlockInternal(first_block);
-    CacheBlock(first_block);
+    info = next_info;
   }
+
   return space_allocated;
 }
 
-void ArenaImpl::CleanupList(Block* head) {
-  // Have to do this in a first pass, because some of the destructors might
-  // refer to memory in other blocks.
-  for (Block* b = head; b; b = b->next) {
-    CleanupChunk* list = b->cleanup;
+void ArenaImpl::CleanupList() {
+  // By omitting an Acquire barrier we ensure that any user code that doesn't
+  // properly synchronize Reset() or the destructor will throw a TSAN warning.
+  ThreadInfo* info =
+      reinterpret_cast<ThreadInfo*>(google::protobuf::internal::NoBarrier_Load(&threads_));
+
+  for ( ; info; info = info->next) {
+    CleanupChunk* list = info->cleanup;
     while (list) {
       size_t n = list->len;
       CleanupNode* node = &list->nodes[list->len - 1];
@@ -334,24 +313,56 @@ void ArenaImpl::CleanupList(Block* head) {
       }
       list = list->next;
     }
-    b->cleanup = NULL;
   }
 }
 
-ArenaImpl::Block* ArenaImpl::FindBlock(void* me) {
-  // TODO(sanjay): We might want to keep a separate list with one
-  // entry per thread.
-  Block* b = reinterpret_cast<Block*>(google::protobuf::internal::Acquire_Load(&blocks_));
-  while (b != NULL && b->owner != me) {
-    b = b->next;
+ArenaImpl::ThreadInfo* ArenaImpl::NewThreadInfo(Block* b) {
+  GOOGLE_DCHECK(FindThreadInfo(b->owner) == NULL);
+  ThreadInfo* info =
+      reinterpret_cast<ThreadInfo*>(AllocFromBlock(b, sizeof(ThreadInfo)));
+  b->thread_info = info;
+  info->owner = b->owner;
+  info->head = b;
+  info->cleanup = NULL;
+  return info;
+}
+
+ArenaImpl::ThreadInfo* ArenaImpl::FindThreadInfo(void* me) {
+  ThreadInfo* info =
+      reinterpret_cast<ThreadInfo*>(google::protobuf::internal::Acquire_Load(&threads_));
+  for ( ; info; info = info->next) {
+    if (info->owner == me) {
+      return info;
+    }
   }
-  return b;
+
+  return NULL;
+}
+
+ArenaImpl::ThreadInfo* ArenaImpl::GetThreadInfo(void* me, size_t n) {
+  ThreadInfo* info = FindThreadInfo(me);
+
+  if (!info) {
+    // This thread doesn't have any ThreadInfo, which also means it doesn't have
+    // any blocks yet.  So we'll allocate its first block now.
+    Block* b = NewBlock(me, NULL, sizeof(ThreadInfo) + n);
+    info = NewThreadInfo(b);
+
+    google::protobuf::internal::AtomicWord head;
+    do {
+      head = google::protobuf::internal::NoBarrier_Load(&threads_);
+      info->next = reinterpret_cast<ThreadInfo*>(head);
+    } while (google::protobuf::internal::Release_CompareAndSwap(
+                 &threads_, head, reinterpret_cast<google::protobuf::internal::AtomicWord>(info)) != head);
+  }
+
+  return info;
 }
 
 }  // namespace internal
 
 void Arena::CallDestructorHooks() {
-  uint64 space_allocated = SpaceAllocated();
+  uint64 space_allocated = impl_.SpaceAllocated();
   // Call the reset hook
   if (on_arena_reset_ != NULL) {
     on_arena_reset_(this, hooks_cookie_, space_allocated);

src/google/protobuf/arena.h

@@ -61,7 +61,7 @@ class Arena;       // defined below
 class Message;     // message.h
 
 namespace internal {
-class ArenaString; // arenastring.h
+struct ArenaStringPtr;  // arenastring.h
 class LazyField;   // lazy_field.h
 
 template<typename Type>
@@ -223,6 +223,14 @@ class LIBPROTOBUF_EXPORT Arena {
     Init(options);
   }
 
+  // Block overhead.  Use this as a guide for how much to over-allocate the
+  // initial block if you want an allocation of size N to fit inside it.
+  //
+  // WARNING: if you allocate multiple objects, it is difficult to guarantee
+  // that a series of allocations will fit in the initial block, especially if
+  // Arena changes its alignment guarantees in the future!
+  static const size_t kBlockOverhead = internal::ArenaImpl::kHeaderSize;
+
   // Default constructor with sensible default options, tuned for average
   // use-cases.
   Arena() : impl_(ArenaOptions()) { Init(ArenaOptions()); }
@@ -524,10 +532,9 @@ class LIBPROTOBUF_EXPORT Arena {
   // returns the total space used by the arena which is the sums of the sizes
   // of the allocated blocks. This method is not thread-safe.
   GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE uint64 Reset() {
-    uint64 space_allocated = SpaceAllocated();
     // Call the reset hook
     if (on_arena_reset_ != NULL) {
-      on_arena_reset_(this, hooks_cookie_, space_allocated);
+      on_arena_reset_(this, hooks_cookie_, impl_.SpaceAllocated());
     }
     return impl_.Reset();
   }
@@ -912,7 +919,7 @@ class LIBPROTOBUF_EXPORT Arena {
 
   template <typename Type>
   friend class ::google::protobuf::internal::GenericTypeHandler;
-  friend class internal::ArenaString;  // For AllocateAligned.
+  friend struct internal::ArenaStringPtr;  // For AllocateAligned.
   friend class internal::LazyField;    // For CreateMaybeMessage.
   template <typename Key, typename T>
   friend class Map;

src/google/protobuf/arena_impl.h

@@ -82,6 +82,14 @@ class LIBPROTOBUF_EXPORT ArenaImpl {
 
   template <typename O>
   explicit ArenaImpl(const O& options) : options_(options) {
+    if (options_.initial_block != NULL && options_.initial_block_size > 0) {
+      GOOGLE_CHECK_GE(options_.initial_block_size, sizeof(Block))
+          << ": Initial block size too small for header.";
+      initial_block_ = reinterpret_cast<Block*>(options_.initial_block);
+    } else {
+      initial_block_ = NULL;
+    }
+
     Init();
   }
 
@@ -122,13 +130,22 @@ class LIBPROTOBUF_EXPORT ArenaImpl {
     CleanupNode nodes[1];  // True length is |size|.
   };
 
+  struct Block;
+
+  // Tracks per-thread info.  ThreadInfos are kept in a linked list.
+  struct ThreadInfo {
+    void *owner;             // &ThreadCache of this thread;
+    Block* head;             // Head of linked list of blocks.
+    CleanupChunk* cleanup;   // Head of cleanup list.
+    ThreadInfo* next;        // Next ThreadInfo in this linked list.
+  };
+
   // Blocks are variable length malloc-ed objects.  The following structure
   // describes the common header for all blocks.
   struct Block {
-    void* owner;            // &ThreadCache of thread that owns this block.
-    Block* next;            // Next block in arena (may have different owner)
-    CleanupChunk* cleanup;  // Head of cleanup list (may point to another block,
-                            // but it must have the same owner).
+    void* owner;              // &ThreadCache of thread that owns this block.
+    ThreadInfo* thread_info;  // ThreadInfo of thread that owns this block.
+    Block* next;              // Next block in arena (may have different owner)
     // ((char*) &block) + pos is next available byte. It is always
     // aligned at a multiple of 8 bytes.
     size_t pos;
@@ -139,18 +156,18 @@ class LIBPROTOBUF_EXPORT ArenaImpl {
   };
 
   struct ThreadCache {
+#if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL)
+    // If we are using the ThreadLocalStorage class to store the ThreadCache,
+    // then the ThreadCache's default constructor has to be responsible for
+    // initializing it.
+    ThreadCache() : last_lifecycle_id_seen(-1), last_block_used_(NULL) {}
+#endif
+
     // The ThreadCache is considered valid as long as this matches the
     // lifecycle_id of the arena being used.
     int64 last_lifecycle_id_seen;
     Block* last_block_used_;
   };
-
-  // kHeaderSize is sizeof(Block), aligned up to the nearest multiple of 8 to
-  // protect the invariant that pos is always at a multiple of 8.
-  static const size_t kHeaderSize = (sizeof(Block) + 7) & -8;
-#if LANG_CXX11
-  static_assert(kHeaderSize % 8 == 0, "kHeaderSize must be a multiple of 8.");
-#endif
   static google::protobuf::internal::SequenceNumber lifecycle_id_generator_;
 #if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL)
   // Android ndk does not support GOOGLE_THREAD_LOCAL keyword so we use a custom thread
@@ -170,44 +187,52 @@ class LIBPROTOBUF_EXPORT ArenaImpl {
 
   // Free all blocks and return the total space used which is the sums of sizes
   // of the all the allocated blocks.
-  uint64 FreeBlocks(Block* head);
+  uint64 FreeBlocks();
 
-  void AddCleanupInBlock(Block* b, void* elem, void (*cleanup)(void*));
-  Block* ExpandCleanupList(Block* b);
+  void AddCleanupInBlock(Block* b, void* elem, void (*func)(void*));
+  CleanupChunk* ExpandCleanupList(CleanupChunk* cleanup, Block* b);
   // Delete or Destruct all objects owned by the arena.
-  void CleanupList(Block* head);
-  uint64 ResetInternal();
+  void CleanupList();
 
   inline void CacheBlock(Block* block) {
     thread_cache().last_block_used_ = block;
     thread_cache().last_lifecycle_id_seen = lifecycle_id_;
+    // TODO(haberman): evaluate whether we would gain efficiency by getting rid
+    // of hint_.  It's the only write we do to ArenaImpl in the allocation path,
+    // which will dirty the cache line.
     google::protobuf::internal::Release_Store(&hint_, reinterpret_cast<google::protobuf::internal::AtomicWord>(block));
   }
 
-  google::protobuf::internal::AtomicWord blocks_;       // Head of linked list of all allocated blocks
-  google::protobuf::internal::AtomicWord hint_;         // Fast thread-local block access
-  uint64 space_allocated_;  // Sum of sizes of all allocated blocks.
+  google::protobuf::internal::AtomicWord threads_;          // Pointer to a linked list of ThreadInfo.
+  google::protobuf::internal::AtomicWord hint_;             // Fast thread-local block access
+  google::protobuf::internal::AtomicWord space_allocated_;  // Sum of sizes of all allocated blocks.
 
-  bool owns_first_block_;  // Indicates that arena owns the first block
-  mutable Mutex blocks_lock_;
+  Block *initial_block_;     // If non-NULL, points to the block that came from
+                             // user data.
 
-  void AddBlock(Block* b);
-  // Access must be synchronized, either by blocks_lock_ or by being called from
-  // Init()/Reset().
-  void AddBlockInternal(Block* b);
   // Returns a block owned by this thread.
   Block* GetBlock(size_t n);
   Block* GetBlockSlow(void* me, Block* my_full_block, size_t n);
-  Block* FindBlock(void* me);
-  Block* NewBlock(void* me, Block* my_last_block, size_t min_bytes,
-                  size_t start_block_size, size_t max_block_size);
+  Block* NewBlock(void* me, Block* my_last_block, size_t min_bytes);
+  void InitBlock(Block* b, void *me, size_t size);
   static void* AllocFromBlock(Block* b, size_t n);
+  ThreadInfo* NewThreadInfo(Block* b);
+  ThreadInfo* FindThreadInfo(void* me);
+  ThreadInfo* GetThreadInfo(void* me, size_t n);
 
   int64 lifecycle_id_;  // Unique for each arena. Changes on Reset().
 
   Options options_;
 
   GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ArenaImpl);
+
+ public:
+  // kHeaderSize is sizeof(Block), aligned up to the nearest multiple of 8 to
+  // protect the invariant that pos is always at a multiple of 8.
+  static const size_t kHeaderSize = (sizeof(Block) + 7) & -8;
+#if LANG_CXX11
+  static_assert(kHeaderSize % 8 == 0, "kHeaderSize must be a multiple of 8.");
+#endif
 };
 
 }  // namespace internal

src/google/protobuf/arena_unittest.cc

@@ -270,7 +270,7 @@ TEST(ArenaTest, InitialBlockTooSmall) {
   // Construct a small (64 byte) initial block of memory to be used by the
   // arena allocator; then, allocate an object which will not fit in the
   // initial block.
-  std::vector<char> arena_block(64);
+  std::vector<char> arena_block(72);
   ArenaOptions options;
   options.initial_block = &arena_block[0];
   options.initial_block_size = arena_block.size();
@@ -1299,12 +1299,12 @@ TEST(ArenaTest, SpaceAllocated_and_Used) {
   options.initial_block_size = 0;
   Arena arena_3(options);
   EXPECT_EQ(0, arena_3.SpaceUsed());
-  ::google::protobuf::Arena::CreateArray<char>(&arena_3, 190);
+  ::google::protobuf::Arena::CreateArray<char>(&arena_3, 182);
   EXPECT_EQ(256, arena_3.SpaceAllocated());
-  EXPECT_EQ(Align8(190), arena_3.SpaceUsed());
+  EXPECT_EQ(Align8(182), arena_3.SpaceUsed());
   ::google::protobuf::Arena::CreateArray<char>(&arena_3, 70);
   EXPECT_EQ(256 + 512, arena_3.SpaceAllocated());
-  EXPECT_EQ(Align8(190) + Align8(70), arena_3.SpaceUsed());
+  EXPECT_EQ(Align8(182) + Align8(70), arena_3.SpaceUsed());
   EXPECT_EQ(256 + 512, arena_3.Reset());
 }
 

src/google/protobuf/arenastring_unittest.cc

@@ -49,7 +49,6 @@
 
 
 namespace google {
-using google::protobuf::internal::ArenaString;
 using google::protobuf::internal::ArenaStringPtr;
 
 namespace protobuf {
@@ -110,6 +109,33 @@ TEST(ArenaStringPtrTest, ArenaStringPtrOnArena) {
   field2.Destroy(&default_value, &arena);
 }
 
+TEST(ArenaStringPtrTest, ArenaStringPtrOnArenaNoSSO) {
+  google::protobuf::Arena arena;
+  ArenaStringPtr field;
+  ::std::string default_value = "default";
+  field.UnsafeSetDefault(&default_value);
+  EXPECT_EQ(string("default"), field.Get());
+
+  // Avoid triggering the SSO optimization by setting the string to something
+  // larger than the internal buffer.
+  field.Set(&default_value, WrapString("Test long long long long value"),
+            &arena);
+  EXPECT_EQ(string("Test long long long long value"), field.Get());
+  field.Set(&default_value, string(""), &arena);
+  field.Destroy(&default_value, &arena);
+
+  ArenaStringPtr field2;
+  field2.UnsafeSetDefault(&default_value);
+  ::std::string* mut = field2.Mutable(&default_value, &arena);
+  EXPECT_EQ(mut, field2.Mutable(&default_value, &arena));
+  EXPECT_EQ(mut, &field2.Get());
+  EXPECT_NE(&default_value, mut);
+  EXPECT_EQ(string("default"), *mut);
+  *mut = "Test long long long long value";  // ensure string allocates storage
+  EXPECT_EQ(string("Test long long long long value"), field2.Get());
+  field2.Destroy(&default_value, &arena);
+}
+
 
 }  // namespace protobuf
 }  // namespace google

src/google/protobuf/compiler/annotation_test_util.cc

@@ -0,0 +1,187 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <google/protobuf/compiler/annotation_test_util.h>
+
+#include <memory>
+#ifndef _SHARED_PTR_H
+#include <google/protobuf/stubs/shared_ptr.h>
+#endif
+#include <google/protobuf/compiler/code_generator.h>
+#include <google/protobuf/compiler/command_line_interface.h>
+#include <google/protobuf/io/printer.h>
+#include <google/protobuf/io/zero_copy_stream.h>
+#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
+#include <google/protobuf/descriptor.pb.h>
+
+#include <google/protobuf/testing/file.h>
+#include <google/protobuf/testing/file.h>
+#include <google/protobuf/testing/googletest.h>
+#include <gtest/gtest.h>
+
+namespace google {
+namespace protobuf {
+namespace compiler {
+namespace annotation_test_util {
+namespace {
+
+// A CodeGenerator that captures the FileDescriptor it's passed as a
+// FileDescriptorProto.
+class DescriptorCapturingGenerator : public CodeGenerator {
+ public:
+  // Does not own file; file must outlive the Generator.
+  explicit DescriptorCapturingGenerator(FileDescriptorProto* file)
+      : file_(file) {}
+
+  virtual bool Generate(const FileDescriptor* file, const string& parameter,
+                        GeneratorContext* context, string* error) const {
+    file->CopyTo(file_);
+    return true;
+  }
+
+ private:
+  FileDescriptorProto* file_;
+};
+}  // namespace
+
+void AddFile(const string& filename, const string& data) {
+  GOOGLE_CHECK_OK(File::SetContents(TestTempDir() + "/" + filename, data,
+                             true));
+}
+
+bool CaptureMetadata(const string& filename, const string& plugin_specific_args,
+                     const string& meta_file_suffix, CommandLineInterface* cli,
+                     FileDescriptorProto* file,
+                     std::vector<ExpectedOutput>* outputs) {
+  cli->SetInputsAreProtoPathRelative(true);
+
+  DescriptorCapturingGenerator capturing_generator(file);
+  cli->RegisterGenerator("--capture_out", &capturing_generator, "");
+
+  string proto_path = "-I" + TestTempDir();
+  string capture_out = "--capture_out=" + TestTempDir();
+
+  const char* argv[] = {"protoc", proto_path.c_str(),
+                        plugin_specific_args.c_str(), capture_out.c_str(),
+                        filename.c_str()};
+
+  if (cli->Run(5, argv) != 0) {
+    return false;
+  }
+
+  if (outputs != NULL) {
+    for (std::vector<ExpectedOutput>::iterator i = outputs->begin();
+         i != outputs->end(); ++i) {
+      GOOGLE_CHECK_OK(File::GetContents(TestTempDir() + "/" + i->file_path,
+                                 &i->file_content, true));
+      if (!DecodeMetadata(
+              TestTempDir() + "/" + i->file_path + meta_file_suffix,
+              &i->file_info)) {
+        return false;
+      }
+    }
+  }
+
+  return true;
+}
+
+bool DecodeMetadata(const string& path, GeneratedCodeInfo* info) {
+  string data;
+  GOOGLE_CHECK_OK(File::GetContents(path, &data, true));
+  io::ArrayInputStream input(data.data(), data.size());
+  return info->ParseFromZeroCopyStream(&input);
+}
+
+void FindAnnotationsOnPath(
+    const GeneratedCodeInfo& info, const string& source_file,
+    const std::vector<int>& path,
+    std::vector<const GeneratedCodeInfo::Annotation*>* annotations) {
+  for (int i = 0; i < info.annotation_size(); ++i) {
+    const GeneratedCodeInfo::Annotation* annotation = &info.annotation(i);
+    if (annotation->source_file() != source_file ||
+        annotation->path_size() != path.size()) {
+      continue;
+    }
+    int node = 0;
+    for (; node < path.size(); ++node) {
+      if (annotation->path(node) != path[node]) {
+        break;
+      }
+    }
+    if (node == path.size()) {
+      annotations->push_back(annotation);
+    }
+  }
+}
+
+const GeneratedCodeInfo::Annotation* FindAnnotationOnPath(
+    const GeneratedCodeInfo& info, const string& source_file,
+    const std::vector<int>& path) {
+  std::vector<const GeneratedCodeInfo::Annotation*> annotations;
+  FindAnnotationsOnPath(info, source_file, path, &annotations);
+  if (annotations.empty()) {
+    return NULL;
+  }
+  return annotations[0];
+}
+
+bool AtLeastOneAnnotationMatchesSubstring(
+    const string& file_content,
+    const std::vector<const GeneratedCodeInfo::Annotation*>& annotations,
+    const string& expected_text) {
+  for (std::vector<const GeneratedCodeInfo::Annotation*>::const_iterator
+           i = annotations.begin(),
+           e = annotations.end();
+       i != e; ++i) {
+    const GeneratedCodeInfo::Annotation* annotation = *i;
+    uint32 begin = annotation->begin();
+    uint32 end = annotation->end();
+    if (end < begin || end > file_content.size()) {
+      return false;
+    }
+    if (file_content.substr(begin, end - begin) == expected_text) {
+      return true;
+    }
+  }
+  return false;
+}
+
+bool AnnotationMatchesSubstring(const string& file_content,
+                                const GeneratedCodeInfo::Annotation* annotation,
+                                const string& expected_text) {
+  std::vector<const GeneratedCodeInfo::Annotation*> annotations;
+  annotations.push_back(annotation);
+  return AtLeastOneAnnotationMatchesSubstring(file_content, annotations,
+                                              expected_text);
+}
+}  // namespace annotation_test_util
+}  // namespace compiler
+}  // namespace protobuf
+}  // namespace google

src/google/protobuf/compiler/annotation_test_util.h

@@ -0,0 +1,119 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef GOOGLE_PROTOBUF_COMPILER_ANNOTATION_TEST_UTIL_H__
+#define GOOGLE_PROTOBUF_COMPILER_ANNOTATION_TEST_UTIL_H__
+
+#include <google/protobuf/descriptor.pb.h>
+#include <google/protobuf/testing/googletest.h>
+#include <gtest/gtest.h>
+
+// Utilities that assist in writing tests for generator annotations.
+// See java/internal/annotation_unittest.cc for an example.
+namespace google {
+namespace protobuf {
+namespace compiler {
+namespace annotation_test_util {
+
+// Struct that contains the file generated from a .proto file and its
+// GeneratedCodeInfo. For example, the Java generator will fill this struct
+// (for some 'foo.proto') with:
+//   file_path = "Foo.java"
+//   file_content = content of Foo.java
+//   file_info = parsed content of Foo.java.pb.meta
+struct ExpectedOutput {
+  string file_path;
+  string file_content;
+  GeneratedCodeInfo file_info;
+  explicit ExpectedOutput(const string& file_path) : file_path(file_path) {}
+};
+
+// Creates a file with name `filename` and content `data` in temp test
+// directory.
+void AddFile(const string& filename, const string& data);
+
+// Tries to capture a FileDescriptorProto, GeneratedCodeInfo, and output
+// code from the previously added file with name `filename`.
+//
+// filename: source .proto file used to generate code.
+// plugin_specific_args: command line arguments specific to current generator.
+//     For Java, this value might be "--java_out=annotate_code:test_temp_dir"
+// meta_file_suffix: suffix of meta files that contain annotations. For Java
+//     it is ".pb.meta" because for file Foo.java meta file is Foo.java.pb.meta
+// cli: instance of command line interface to run generator. See Java's
+//     annotation_unittest.cc for an example of how to initialize it.
+// file: output parameter, will be set to the descriptor of the proto file
+//     specified in filename.
+// outputs: output parameter. If not NULL, each ExpectedOutput in the vector
+//     should have its file_path set; CaptureMetadata will fill the rest of
+//     the fields appropriately.
+bool CaptureMetadata(const string& filename, const string& plugin_specific_args,
+                     const string& meta_file_suffix, CommandLineInterface* cli,
+                     FileDescriptorProto* file,
+                     std::vector<ExpectedOutput>* outputs);
+
+bool DecodeMetadata(const string& path, GeneratedCodeInfo* info);
+
+// Finds all of the Annotations for a given source file and path.
+// See Location.path in http://google/protobuf/descriptor.proto for
+// explanation of what path vector is.
+void FindAnnotationsOnPath(
+    const GeneratedCodeInfo& info, const string& source_file,
+    const std::vector<int>& path,
+    std::vector<const GeneratedCodeInfo::Annotation*>* annotations);
+
+// Finds the Annotation for a given source file and path (or returns null if it
+// couldn't). If there are several annotations for given path, returns the first
+// one. See Location.path in
+// http://google/protobuf/descriptor.proto for explanation of what path
+// vector is.
+const GeneratedCodeInfo::Annotation* FindAnnotationOnPath(
+    const GeneratedCodeInfo& info, const string& source_file,
+    const std::vector<int>& path);
+
+// Returns true if at least one of the provided annotations covers a given
+// substring in file_content.
+bool AtLeastOneAnnotationMatchesSubstring(
+    const string& file_content,
+    const std::vector<const GeneratedCodeInfo::Annotation*>& annotations,
+    const string& expected_text);
+
+// Returns true if the provided annotation covers a given substring in
+// file_content.
+bool AnnotationMatchesSubstring(const string& file_content,
+                                const GeneratedCodeInfo::Annotation* annotation,
+                                const string& expected_text);
+
+}  // namespace annotation_test_util
+}  // namespace compiler
+}  // namespace protobuf
+
+}  // namespace google
+#endif  // GOOGLE_PROTOBUF_COMPILER_ANNOTATION_TEST_UTIL_H__

src/google/protobuf/compiler/cpp/cpp_enum_field.cc

@@ -82,21 +82,18 @@ GenerateAccessorDeclarations(io::Printer* printer) const {
 }
 
 void EnumFieldGenerator::
-GenerateInlineAccessorDefinitions(io::Printer* printer,
-                                  bool is_inline) const {
-  std::map<string, string> variables(variables_);
-  variables["inline"] = is_inline ? "inline " : "";
-  printer->Print(variables,
-    "$inline$$type$ $classname$::$name$() const {\n"
+GenerateInlineAccessorDefinitions(io::Printer* printer) const {
+  printer->Print(variables_,
+    "inline $type$ $classname$::$name$() const {\n"
     "  // @@protoc_insertion_point(field_get:$full_name$)\n"
     "  return static_cast< $type$ >($name$_);\n"
     "}\n"
-    "$inline$void $classname$::set_$name$($type$ value) {\n");
+    "inline void $classname$::set_$name$($type$ value) {\n");
   if (!HasPreservingUnknownEnumSemantics(descriptor_->file())) {
-    printer->Print(variables,
+    printer->Print(variables_,
     "  assert($type$_IsValid(value));\n");
   }
-  printer->Print(variables,
+  printer->Print(variables_,
     "  $set_hasbit$\n"
     "  $name$_ = value;\n"
     "  // @@protoc_insertion_point(field_set:$full_name$)\n"
@@ -193,24 +190,21 @@ EnumOneofFieldGenerator(const FieldDescriptor* descriptor,
 EnumOneofFieldGenerator::~EnumOneofFieldGenerator() {}
 
 void EnumOneofFieldGenerator::
-GenerateInlineAccessorDefinitions(io::Printer* printer,
-                                  bool is_inline) const {
-  std::map<string, string> variables(variables_);
-  variables["inline"] = is_inline ? "inline " : "";
-  printer->Print(variables,
-    "$inline$$type$ $classname$::$name$() const {\n"
+GenerateInlineAccessorDefinitions(io::Printer* printer) const {
+  printer->Print(variables_,
+    "inline $type$ $classname$::$name$() const {\n"
     "  // @@protoc_insertion_point(field_get:$full_name$)\n"
     "  if (has_$name$()) {\n"
     "    return static_cast< $type$ >($oneof_prefix$$name$_);\n"
     "  }\n"
     "  return static_cast< $type$ >($default$);\n"
     "}\n"
-    "$inline$void $classname$::set_$name$($type$ value) {\n");
+    "inline void $classname$::set_$name$($type$ value) {\n");
   if (!HasPreservingUnknownEnumSemantics(descriptor_->file())) {
-    printer->Print(variables,
+    printer->Print(variables_,
     "  assert($type$_IsValid(value));\n");
   }
-  printer->Print(variables,
+  printer->Print(variables_,
     "  if (!has_$name$()) {\n"
     "    clear_$oneof_name$();\n"
     "    set_has_$name$();\n"
@@ -280,39 +274,36 @@ GenerateAccessorDeclarations(io::Printer* printer) const {
 }
 
 void RepeatedEnumFieldGenerator::
-GenerateInlineAccessorDefinitions(io::Printer* printer,
-                                  bool is_inline) const {
-  std::map<string, string> variables(variables_);
-  variables["inline"] = is_inline ? "inline " : "";
-  printer->Print(variables,
-    "$inline$$type$ $classname$::$name$(int index) const {\n"
+GenerateInlineAccessorDefinitions(io::Printer* printer) const {
+  printer->Print(variables_,
+    "inline $type$ $classname$::$name$(int index) const {\n"
     "  // @@protoc_insertion_point(field_get:$full_name$)\n"
     "  return static_cast< $type$ >($name$_.Get(index));\n"
     "}\n"
-    "$inline$void $classname$::set_$name$(int index, $type$ value) {\n");
+    "inline void $classname$::set_$name$(int index, $type$ value) {\n");
   if (!HasPreservingUnknownEnumSemantics(descriptor_->file())) {
-    printer->Print(variables,
+    printer->Print(variables_,
     "  assert($type$_IsValid(value));\n");
   }
-  printer->Print(variables,
+  printer->Print(variables_,
     "  $name$_.Set(index, value);\n"
     "  // @@protoc_insertion_point(field_set:$full_name$)\n"
     "}\n"
-    "$inline$void $classname$::add_$name$($type$ value) {\n");
+    "inline void $classname$::add_$name$($type$ value) {\n");
   if (!HasPreservingUnknownEnumSemantics(descriptor_->file())) {
-    printer->Print(variables,
+    printer->Print(variables_,
     "  assert($type$_IsValid(value));\n");
   }
-  printer->Print(variables,
+  printer->Print(variables_,
     "  $name$_.Add(value);\n"
     "  // @@protoc_insertion_point(field_add:$full_name$)\n"
     "}\n"
-    "$inline$const ::google::protobuf::RepeatedField<int>&\n"
+    "inline const ::google::protobuf::RepeatedField<int>&\n"
     "$classname$::$name$() const {\n"
     "  // @@protoc_insertion_point(field_list:$full_name$)\n"
     "  return $name$_;\n"
     "}\n"
-    "$inline$::google::protobuf::RepeatedField<int>*\n"
+    "inline ::google::protobuf::RepeatedField<int>*\n"
     "$classname$::mutable_$name$() {\n"
     "  // @@protoc_insertion_point(field_mutable_list:$full_name$)\n"
     "  return &$name$_;\n"

src/google/protobuf/compiler/cpp/cpp_enum_field.h

@@ -52,8 +52,7 @@ class EnumFieldGenerator : public FieldGenerator {
   // implements FieldGenerator ---------------------------------------
   void GeneratePrivateMembers(io::Printer* printer) const;
   void GenerateAccessorDeclarations(io::Printer* printer) const;
-  void GenerateInlineAccessorDefinitions(io::Printer* printer,
-                                         bool is_inline) const;
+  void GenerateInlineAccessorDefinitions(io::Printer* printer) const;
   void GenerateClearingCode(io::Printer* printer) const;
   void GenerateMergingCode(io::Printer* printer) const;
   void GenerateSwappingCode(io::Printer* printer) const;
@@ -79,8 +78,7 @@ class EnumOneofFieldGenerator : public EnumFieldGenerator {
   ~EnumOneofFieldGenerator();
 
   // implements FieldGenerator ---------------------------------------
-  void GenerateInlineAccessorDefinitions(io::Printer* printer,
-                                         bool is_inline) const;
+  void GenerateInlineAccessorDefinitions(io::Printer* printer) const;
   void GenerateClearingCode(io::Printer* printer) const;
   void GenerateSwappingCode(io::Printer* printer) const;
   void GenerateConstructorCode(io::Printer* printer) const;
@@ -98,8 +96,7 @@ class RepeatedEnumFieldGenerator : public FieldGenerator {
   // implements FieldGenerator ---------------------------------------
   void GeneratePrivateMembers(io::Printer* printer) const;
   void GenerateAccessorDeclarations(io::Printer* printer) const;
-  void GenerateInlineAccessorDefinitions(io::Printer* printer,
-                                         bool is_inline) const;
+  void GenerateInlineAccessorDefinitions(io::Printer* printer) const;
   void GenerateClearingCode(io::Printer* printer) const;
   void GenerateMergingCode(io::Printer* printer) const;
   void GenerateSwappingCode(io::Printer* printer) const;

src/google/protobuf/compiler/cpp/cpp_field.h

@@ -109,19 +109,19 @@ class FieldGenerator {
   // Generate inline definitions of depenent accessor functions for this field.
   // These are placed inside the header after all class definitions.
   virtual void GenerateDependentInlineAccessorDefinitions(
-    io::Printer* printer) const {}
+      io::Printer* printer) const {}
 
   // Generate inline definitions of accessor functions for this field.
   // These are placed inside the header after all class definitions.
   // In non-.proto.h mode, this generates dependent accessor functions as well.
   virtual void GenerateInlineAccessorDefinitions(
-    io::Printer* printer, bool is_inline) const = 0;
+      io::Printer* printer) const = 0;
 
   // Generate definitions of accessors that aren't inlined.  These are
   // placed somewhere in the .cc file.
   // Most field types don't need this, so the default implementation is empty.
   virtual void GenerateNonInlineAccessorDefinitions(
-    io::Printer* /*printer*/) const {}
+      io::Printer* /*printer*/) const {}
 
   // Generate lines of code (statements, not declarations) which clear the
   // field.  This is used to define the clear_$name$() method

src/google/protobuf/compiler/cpp/cpp_file.cc

@@ -303,6 +303,18 @@ void FileGenerator::GenerateSourceIncludes(io::Printer* printer) {
     }
   }
 
+  // TODO(gerbens) Remove this when all code in google is using the same
+  // proto library. This is a temporary hack to force build errors if
+  // the proto library is compiled with GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
+  // and is also linking internal proto2. This is to prevent regressions while
+  // we work cleaning up the code base. After this is completed and we have
+  // one proto lib all code uses this should be removed.
+  printer->Print(
+    "// This is a temporary google only hack\n"
+    "#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS\n"
+    "#include \"third_party/protobuf/version.h\"\n"
+    "#endif\n");
+
   printer->Print(
     "// @@protoc_insertion_point(includes)\n");
 }
@@ -385,6 +397,10 @@ void FileGenerator::GenerateSourceForMessage(int idx, io::Printer* printer) {
 
     // Define default instances
     GenerateSourceDefaultInstance(idx, printer);
+    if (UsingImplicitWeakFields(file_, options_)) {
+      printer->Print("void $classname$_ReferenceStrong() {}\n", "classname",
+                     message_generators_[idx]->classname_);
+    }
 
     // Generate classes.
     printer->Print("\n");
@@ -452,7 +468,7 @@ void FileGenerator::GenerateSource(io::Printer* printer) {
     for (int i = 0; i < message_generators_.size(); i++) {
       GenerateSourceDefaultInstance(i, printer);
       if (UsingImplicitWeakFields(file_, options_)) {
-        printer->Print("void $classname$_Reference() {}\n", "classname",
+        printer->Print("void $classname$_ReferenceStrong() {}\n", "classname",
                        message_generators_[i]->classname_);
       }
     }
@@ -564,7 +580,7 @@ class FileGenerator::ForwardDeclarations {
           "classname",
           it->first);
       if (options.lite_implicit_weak_fields) {
-        printer->Print("void $classname$_Reference();\n",
+        printer->Print("void $classname$_ReferenceStrong();\n",
                        "classname", it->first);
       }
     }
@@ -827,8 +843,12 @@ void FileGenerator::GenerateInitForSCC(const SCC* scc, io::Printer* printer) {
   printer->Print(
       "void InitDefaults$scc_name$Impl() {\n"
       "  GOOGLE_PROTOBUF_VERIFY_VERSION;\n\n"
+      "#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS\n"
+      "  ::google::protobuf::internal::InitProtobufDefaultsForceUnique();\n"
+      "#else\n"
+      "  ::google::protobuf::internal::InitProtobufDefaults();\n"
+      "#endif  // GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS\n",
         // Force initialization of primitive values we depend on.
-      "  ::google::protobuf::internal::InitProtobufDefaults();\n",
       "scc_name", scc_name);
 
   printer->Indent();
@@ -1317,8 +1337,7 @@ void FileGenerator::GenerateInlineFunctionDefinitions(io::Printer* printer) {
       printer->Print(kThinSeparator);
       printer->Print("\n");
     }
-    message_generators_[i]->GenerateInlineMethods(printer,
-                                                  /* is_inline = */ true);
+    message_generators_[i]->GenerateInlineMethods(printer);
   }
   printer->Print(
     "#ifdef __GNUC__\n"

src/google/protobuf/compiler/cpp/cpp_helpers.cc

@@ -216,7 +216,7 @@ string DefaultInstanceName(const Descriptor* descriptor) {
 }
 
 string ReferenceFunctionName(const Descriptor* descriptor) {
-  return QualifiedClassName(descriptor) + "_Reference";
+  return QualifiedClassName(descriptor) + "_ReferenceStrong";
 }
 
 string DependentBaseClassTemplateName(const Descriptor* descriptor) {
@@ -753,8 +753,7 @@ bool IsImplicitWeakField(const FieldDescriptor* field, const Options& options) {
   return UsingImplicitWeakFields(field->file(), options) &&
          field->type() == FieldDescriptor::TYPE_MESSAGE &&
          !field->is_required() && !field->is_repeated() && !field->is_map() &&
-         field->containing_oneof() == NULL &&
-         field->message_type()->file() != field->file();
+         field->containing_oneof() == NULL;
 }
 
 struct CompareDescriptors {

src/google/protobuf/compiler/cpp/cpp_helpers.h

@@ -305,6 +305,11 @@ inline bool SupportsArenas(const FieldDescriptor* field) {
   return SupportsArenas(field->file());
 }
 
+inline bool IsCrossFileMessage(const FieldDescriptor* field) {
+  return field->type() == FieldDescriptor::TYPE_MESSAGE &&
+         field->message_type()->file() != field->file();
+}
+
 bool IsAnyMessage(const FileDescriptor* descriptor);
 bool IsAnyMessage(const Descriptor* descriptor);
 

src/google/protobuf/compiler/cpp/cpp_map_field.cc

@@ -137,17 +137,14 @@ GenerateAccessorDeclarations(io::Printer* printer) const {
 }
 
 void MapFieldGenerator::
-GenerateInlineAccessorDefinitions(io::Printer* printer,
-                                  bool is_inline) const {
-  std::map<string, string> variables(variables_);
-  variables["inline"] = is_inline ? "inline" : "";
-  printer->Print(variables,
-      "$inline$ const ::google::protobuf::Map< $key_cpp$, $val_cpp$ >&\n"
+GenerateInlineAccessorDefinitions(io::Printer* printer) const {
+  printer->Print(variables_,
+      "inline const ::google::protobuf::Map< $key_cpp$, $val_cpp$ >&\n"
       "$classname$::$name$() const {\n"
       "  // @@protoc_insertion_point(field_map:$full_name$)\n"
       "  return $name$_.GetMap();\n"
       "}\n"
-      "$inline$ ::google::protobuf::Map< $key_cpp$, $val_cpp$ >*\n"
+      "inline ::google::protobuf::Map< $key_cpp$, $val_cpp$ >*\n"
       "$classname$::mutable_$name$() {\n"
       "  // @@protoc_insertion_point(field_mutable_map:$full_name$)\n"
       "  return $name$_.MutableMap();\n"
@@ -156,9 +153,7 @@ GenerateInlineAccessorDefinitions(io::Printer* printer,
 
 void MapFieldGenerator::
 GenerateClearingCode(io::Printer* printer) const {
-  std::map<string, string> variables(variables_);
-  variables["this_message"] = dependent_field_ ? DependentBaseDownCast() : "";
-  printer->Print(variables, "$this_message$$name$_.Clear();\n");
+  printer->Print(variables_, "$name$_.Clear();\n");
 }
 
 void MapFieldGenerator::

src/google/protobuf/compiler/cpp/cpp_map_field.h

@@ -49,8 +49,7 @@ class MapFieldGenerator : public FieldGenerator {
   // implements FieldGenerator ---------------------------------------
   void GeneratePrivateMembers(io::Printer* printer) const;
   void GenerateAccessorDeclarations(io::Printer* printer) const;
-  void GenerateInlineAccessorDefinitions(io::Printer* printer,
-                                         bool is_inline) const;
+  void GenerateInlineAccessorDefinitions(io::Printer* printer) const;
   void GenerateClearingCode(io::Printer* printer) const;
   void GenerateMergingCode(io::Printer* printer) const;
   void GenerateSwappingCode(io::Printer* printer) const;

src/google/protobuf/compiler/cpp/cpp_message.cc

@@ -306,6 +306,25 @@ void SetUnknkownFieldsVariable(const Descriptor* descriptor,
       "_internal_metadata_.mutable_unknown_fields()";
 }
 
+bool IsCrossFileMapField(const FieldDescriptor* field) {
+  if (!field->is_map()) {
+    return false;
+  }
+
+  const Descriptor* d = field->message_type();
+  const FieldDescriptor* value = d->FindFieldByNumber(2);
+
+  return IsCrossFileMessage(value);
+}
+
+bool IsCrossFileMaybeMap(const FieldDescriptor* field) {
+  if (IsCrossFileMapField(field)) {
+    return true;
+  }
+
+  return IsCrossFileMessage(field);
+}
+
 }  // anonymous namespace
 
 // ===================================================================
@@ -426,12 +445,6 @@ GenerateDependentFieldAccessorDeclarations(io::Printer* printer) {
     std::map<string, string> vars;
     SetCommonFieldVariables(field, &vars, options_);
 
-    if (use_dependent_base_ && IsFieldDependent(field)) {
-      // If the message is dependent, the inline clear_*() method will need
-      // to delete the message type, so it must be in the dependent base
-      // class. (See also GenerateFieldAccessorDeclarations.)
-      printer->Print(vars, "$deprecated_attr$void clear_$name$();\n");
-    }
     // Generate type-specific accessor declarations.
     field_generators_.get(field).GenerateDependentAccessorDeclarations(printer);
     printer->Print("\n");
@@ -498,12 +511,8 @@ GenerateFieldAccessorDeclarations(io::Printer* printer) {
       printer->Annotate("{", "}", field);
     }
 
-    if (!dependent_field) {
-      // If this field is dependent, then its clear_() method is in the
-      // depenent base class. (See also GenerateDependentAccessorDeclarations.)
-      printer->Print(vars, "$deprecated_attr$void ${$clear_$name$$}$();\n");
-      printer->Annotate("{", "}", field);
-    }
+    printer->Print(vars, "$deprecated_attr$void ${$clear_$name$$}$();\n");
+    printer->Annotate("{", "}", field);
     printer->Print(vars,
                    "$deprecated_attr$static const int $constant_name$ = "
                    "$number$;\n");
@@ -545,36 +554,6 @@ GenerateDependentFieldAccessorDefinitions(io::Printer* printer) {
     if (field->options().weak()) continue;
 
     PrintFieldComment(printer, field);
-
-    // These functions are not really dependent: they are part of the
-    // (non-dependent) derived class. However, they need to live outside
-    // any #ifdef guards, so we treat them as if they were dependent.
-    //
-    // See the comment in FileGenerator::GenerateInlineFunctionDefinitions
-    // for a more complete explanation.
-    if (use_dependent_base_ && IsFieldDependent(field)) {
-      std::map<string, string> vars;
-      SetCommonFieldVariables(field, &vars, options_);
-      vars["inline"] = "inline ";
-      if (field->containing_oneof()) {
-        vars["field_name"] = UnderscoresToCamelCase(field->name(), true);
-        vars["oneof_name"] = field->containing_oneof()->name();
-        vars["oneof_index"] = SimpleItoa(field->containing_oneof()->index());
-        GenerateOneofMemberHasBits(field, vars, printer);
-      } else if (!field->is_repeated()) {
-        // There will be no header guard, so this always has to be inline.
-        GenerateSingularFieldHasBits(field, vars, printer);
-      }
-      // vars needed for clear_(), which is in the dependent base:
-      // (See also GenerateDependentFieldAccessorDeclarations.)
-      vars["tmpl"] = "template<class T>\n";
-      vars["dependent_classname"] =
-          DependentBaseClassTemplateName(descriptor_) + "<T>";
-      vars["this_message"] = DependentBaseDownCast();
-      vars["this_const_message"] = DependentBaseConstDownCast();
-      GenerateFieldClear(field, vars, printer);
-    }
-
     // Generate type-specific accessors.
     field_generators_.get(field)
         .GenerateDependentInlineAccessorDefinitions(printer);
@@ -585,7 +564,7 @@ GenerateDependentFieldAccessorDefinitions(io::Printer* printer) {
   // Generate has_$name$() and clear_has_$name$() functions for oneofs
   // Similar to other has-bits, these must always be in the header if we
   // are using a dependent base class.
-  GenerateOneofHasBits(printer, true /* is_inline */);
+  GenerateOneofHasBits(printer);
 }
 
 void MessageGenerator::
@@ -595,8 +574,7 @@ GenerateSingularFieldHasBits(const FieldDescriptor* field,
   if (field->options().weak()) {
     printer->Print(
         vars,
-        "$inline$"
-        "bool $classname$::has_$name$() const {\n"
+        "inline bool $classname$::has_$name$() const {\n"
         "  return _weak_field_map_.Has($number$);\n"
         "}\n");
     return;
@@ -611,16 +589,13 @@ GenerateSingularFieldHasBits(const FieldDescriptor* field,
     vars["has_mask"] = StrCat(strings::Hex(1u << (has_bit_index % 32),
                                            strings::ZERO_PAD_8));
     printer->Print(vars,
-      "$inline$"
-      "bool $classname$::has_$name$() const {\n"
+      "inline bool $classname$::has_$name$() const {\n"
       "  return (_has_bits_[$has_array_index$] & 0x$has_mask$u) != 0;\n"
       "}\n"
-      "$inline$"
-      "void $classname$::set_has_$name$() {\n"
+      "inline void $classname$::set_has_$name$() {\n"
       "  _has_bits_[$has_array_index$] |= 0x$has_mask$u;\n"
       "}\n"
-      "$inline$"
-      "void $classname$::clear_has_$name$() {\n"
+      "inline void $classname$::clear_has_$name$() {\n"
       "  _has_bits_[$has_array_index$] &= ~0x$has_mask$u;\n"
       "}\n");
   } else {
@@ -629,15 +604,13 @@ GenerateSingularFieldHasBits(const FieldDescriptor* field,
       bool is_lazy = false;
       if (is_lazy) {
         printer->Print(vars,
-          "$inline$"
-          "bool $classname$::has_$name$() const {\n"
+          "inline bool $classname$::has_$name$() const {\n"
           "  return !$name$_.IsCleared();\n"
           "}\n");
       } else {
         printer->Print(
             vars,
-            "$inline$"
-            "bool $classname$::has_$name$() const {\n"
+            "inline bool $classname$::has_$name$() const {\n"
             "  return this != internal_default_instance() && $name$_ != NULL;\n"
             "}\n");
       }
@@ -646,7 +619,7 @@ GenerateSingularFieldHasBits(const FieldDescriptor* field,
 }
 
 void MessageGenerator::
-GenerateOneofHasBits(io::Printer* printer, bool is_inline) {
+GenerateOneofHasBits(io::Printer* printer) {
   for (int i = 0; i < descriptor_->oneof_decl_count(); i++) {
     std::map<string, string> vars;
     vars["oneof_name"] = descriptor_->oneof_decl(i)->name();
@@ -654,15 +627,12 @@ GenerateOneofHasBits(io::Printer* printer, bool is_inline) {
     vars["cap_oneof_name"] =
         ToUpper(descriptor_->oneof_decl(i)->name());
     vars["classname"] = classname_;
-    vars["inline"] = (is_inline ? "inline " : "");
     printer->Print(
         vars,
-        "$inline$"
-        "bool $classname$::has_$oneof_name$() const {\n"
+        "inline bool $classname$::has_$oneof_name$() const {\n"
         "  return $oneof_name$_case() != $cap_oneof_name$_NOT_SET;\n"
         "}\n"
-        "$inline$"
-        "void $classname$::clear_has_$oneof_name$() {\n"
+        "inline void $classname$::clear_has_$oneof_name$() {\n"
         "  _oneof_case_[$oneof_index$] = $cap_oneof_name$_NOT_SET;\n"
         "}\n");
   }
@@ -679,13 +649,11 @@ GenerateOneofMemberHasBits(const FieldDescriptor* field,
   // method, so that generated code is slightly cleaner (vs.  comparing
   // _oneof_case_[index] against a constant everywhere).
   printer->Print(vars,
-    "$inline$"
-    "bool $classname$::has_$name$() const {\n"
+    "inline bool $classname$::has_$name$() const {\n"
     "  return $oneof_name$_case() == k$field_name$;\n"
     "}\n");
   printer->Print(vars,
-    "$inline$"
-    "void $classname$::set_has_$name$() {\n"
+    "inline void $classname$::set_has_$name$() {\n"
     "  _oneof_case_[$oneof_index$] = k$field_name$;\n"
     "}\n");
 }
@@ -693,14 +661,14 @@ GenerateOneofMemberHasBits(const FieldDescriptor* field,
 void MessageGenerator::
 GenerateFieldClear(const FieldDescriptor* field,
                    const std::map<string, string>& vars,
+                   bool is_inline,
                    io::Printer* printer) {
-  // Generate clear_$name$() (See GenerateFieldAccessorDeclarations and
-  // GenerateDependentFieldAccessorDeclarations, $dependent_classname$ is
-  // set by the Generate*Definitions functions.)
+  // Generate clear_$name$().
+  if (is_inline) {
+    printer->Print("inline ");
+  }
   printer->Print(vars,
-    "$tmpl$"
-    "$inline$"
-    "void $dependent_classname$::clear_$name$() {\n");
+    "void $classname$::clear_$name$() {\n");
 
   printer->Indent();
 
@@ -708,12 +676,12 @@ GenerateFieldClear(const FieldDescriptor* field,
     // Clear this field only if it is the active field in this oneof,
     // otherwise ignore
     printer->Print(vars,
-      "if ($this_message$has_$name$()) {\n");
+      "if (has_$name$()) {\n");
     printer->Indent();
     field_generators_.get(field)
         .GenerateClearingCode(printer);
     printer->Print(vars,
-      "$this_message$clear_has_$oneof_name$();\n");
+      "clear_has_$oneof_name$();\n");
     printer->Outdent();
     printer->Print("}\n");
   } else {
@@ -721,8 +689,7 @@ GenerateFieldClear(const FieldDescriptor* field,
         .GenerateClearingCode(printer);
     if (HasFieldPresence(descriptor_->file())) {
       if (!field->is_repeated() && !field->options().weak()) {
-        printer->Print(vars,
-                       "$this_message$clear_has_$name$();\n");
+        printer->Print(vars, "clear_has_$name$();\n");
       }
     }
   }
@@ -732,7 +699,7 @@ GenerateFieldClear(const FieldDescriptor* field,
 }
 
 void MessageGenerator::
-GenerateFieldAccessorDefinitions(io::Printer* printer, bool is_inline) {
+GenerateFieldAccessorDefinitions(io::Printer* printer) {
   printer->Print("// $classname$\n\n", "classname", classname_);
 
   for (int i = 0; i < descriptor_->field_count(); i++) {
@@ -742,7 +709,6 @@ GenerateFieldAccessorDefinitions(io::Printer* printer, bool is_inline) {
 
     std::map<string, string> vars;
     SetCommonFieldVariables(field, &vars, options_);
-    vars["inline"] = is_inline ? "inline " : "";
     if (use_dependent_base_ && IsFieldDependent(field)) {
       vars["tmpl"] = "template<class T>\n";
       vars["dependent_classname"] =
@@ -759,31 +725,25 @@ GenerateFieldAccessorDefinitions(io::Printer* printer, bool is_inline) {
     // Generate has_$name$() or $name$_size().
     if (field->is_repeated()) {
       printer->Print(vars,
-        "$inline$"
-        "int $classname$::$name$_size() const {\n"
+        "inline int $classname$::$name$_size() const {\n"
         "  return $name$_.size();\n"
         "}\n");
     } else if (field->containing_oneof()) {
       vars["field_name"] = UnderscoresToCamelCase(field->name(), true);
       vars["oneof_name"] = field->containing_oneof()->name();
       vars["oneof_index"] = SimpleItoa(field->containing_oneof()->index());
-      if (!use_dependent_base_ || !IsFieldDependent(field)) {
-        GenerateOneofMemberHasBits(field, vars, printer);
-      }
+      GenerateOneofMemberHasBits(field, vars, printer);
     } else {
       // Singular field.
-      if (!use_dependent_base_ || !IsFieldDependent(field)) {
-        GenerateSingularFieldHasBits(field, vars, printer);
-      }
+      GenerateSingularFieldHasBits(field, vars, printer);
     }
 
-    if (!use_dependent_base_ || !IsFieldDependent(field)) {
-      GenerateFieldClear(field, vars, printer);
+    if (!IsCrossFileMaybeMap(field)) {
+      GenerateFieldClear(field, vars, true, printer);
     }
 
     // Generate type-specific accessors.
-    field_generators_.get(field).GenerateInlineAccessorDefinitions(
-        printer, /* is_inline = */ true);
+    field_generators_.get(field).GenerateInlineAccessorDefinitions(printer);
 
     printer->Print("\n");
   }
@@ -792,7 +752,7 @@ GenerateFieldAccessorDefinitions(io::Printer* printer, bool is_inline) {
     // Generate has_$name$() and clear_has_$name$() functions for oneofs
     // If we aren't using a dependent base, they can be with the other functions
     // that are #ifdef-guarded.
-    GenerateOneofHasBits(printer, is_inline);
+    GenerateOneofHasBits(printer);
   }
 }
 
@@ -1381,9 +1341,9 @@ GenerateDependentInlineMethods(io::Printer* printer) {
 }
 
 void MessageGenerator::
-GenerateInlineMethods(io::Printer* printer, bool is_inline) {
+GenerateInlineMethods(io::Printer* printer) {
   if (IsMapEntryMessage(descriptor_)) return;
-  GenerateFieldAccessorDefinitions(printer,  /* is_inline = */ true);
+  GenerateFieldAccessorDefinitions(printer);
 
   // Generate oneof_case() functions.
   for (int i = 0; i < descriptor_->oneof_decl_count(); i++) {
@@ -1393,11 +1353,9 @@ GenerateInlineMethods(io::Printer* printer, bool is_inline) {
         descriptor_->oneof_decl(i)->name(), true);
     vars["oneof_name"] = descriptor_->oneof_decl(i)->name();
     vars["oneof_index"] = SimpleItoa(descriptor_->oneof_decl(i)->index());
-    vars["inline"] = is_inline ? "inline " : "";
     printer->Print(
         vars,
-        "$inline$"
-        "$class_name$::$camel_oneof_name$Case $class_name$::"
+        "inline $class_name$::$camel_oneof_name$Case $class_name$::"
         "$oneof_name$_case() const {\n"
         "  return $class_name$::$camel_oneof_name$Case("
         "_oneof_case_[$oneof_index$]);\n"
@@ -1852,8 +1810,17 @@ GenerateClassMethods(io::Printer* printer) {
 
   // Generate non-inline field definitions.
   for (int i = 0; i < descriptor_->field_count(); i++) {
-    field_generators_.get(descriptor_->field(i))
+    const FieldDescriptor* field = descriptor_->field(i);
+    field_generators_.get(field)
                      .GenerateNonInlineAccessorDefinitions(printer);
+    if (IsCrossFileMaybeMap(field)) {
+      std::map<string, string> vars;
+      SetCommonFieldVariables(field, &vars, options_);
+      if (field->containing_oneof()) {
+        SetCommonOneofFieldVariables(field, &vars);
+      }
+      GenerateFieldClear(field, vars, false, printer);
+    }
   }
 
   // Generate field number constants.
@@ -2244,16 +2211,9 @@ GenerateSharedDestructorCode(io::Printer* printer) {
     "classname", classname_);
   printer->Indent();
   if (SupportsArenas(descriptor_)) {
-    // Do nothing when the message is allocated in an arena.
     printer->Print(
-      "::google::protobuf::Arena* arena = GetArenaNoVirtual();\n"
-      "GOOGLE_DCHECK(arena == NULL);\n"
-      "if (arena != NULL) {\n"
-      "  return;\n"
-      "}\n"
-      "\n");
+      "GOOGLE_DCHECK(GetArenaNoVirtual() == NULL);\n");
   }
-
   // Write the destructors for each field except oneof members.
   // optimized_order_ does not contain oneof fields.
   for (int i = 0; i < optimized_order_.size(); i++) {
@@ -2748,11 +2708,7 @@ GenerateClear(io::Printer* printer) {
         break;
       }
 
-      if (use_dependent_base_ && IsFieldDependent(field)) {
-        printer->Print("clear_$name$();\n", "name", FieldName(field));
-      } else {
-        generator.GenerateMessageClearingCode(printer);
-      }
+      generator.GenerateMessageClearingCode(printer);
     }
 
     // Step 3: Greedily seek runs of fields that can be cleared by
@@ -2780,8 +2736,7 @@ GenerateClear(io::Printer* printer) {
       if (last_chunk == -1) {
         last_chunk = chunk;
         last_chunk_start = i;
-      } else if ((memset_run_start == -1 || unconditional_budget < 0) &&
-                 chunk != last_chunk) {
+      } else if (chunk != last_chunk) {
         // Emit the fields for this chunk so far.
         break;
       }
@@ -2900,6 +2855,12 @@ flush:
         if (should_check_bit &&
             // If no field presence, then always clear strings/messages as well.
             HasFieldPresence(descriptor_->file())) {
+          if (!field->options().weak() &&
+              cached_has_bit_index != (has_bit_indices_[field->index()] / 32)) {
+            cached_has_bit_index = (has_bit_indices_[field->index()] / 32);
+            printer->Print("cached_has_bits = _has_bits_[$new_index$];\n",
+                           "new_index", SimpleItoa(cached_has_bit_index));
+          }
           if (!MaybeGenerateOptionalFieldCondition(printer, field,
                                                    cached_has_bit_index)) {
             printer->Print(

src/google/protobuf/compiler/cpp/cpp_message.h

@@ -85,7 +85,7 @@ class MessageGenerator {
 
   // Generate definitions of inline methods (placed at the end of the header
   // file).
-  void GenerateInlineMethods(io::Printer* printer, bool is_inline);
+  void GenerateInlineMethods(io::Printer* printer);
 
   // Dependent methods are always inline.
   void GenerateDependentInlineMethods(io::Printer* printer);
@@ -112,7 +112,7 @@ class MessageGenerator {
   void GenerateDependentFieldAccessorDeclarations(io::Printer* printer);
   void GenerateFieldAccessorDeclarations(io::Printer* printer);
   void GenerateDependentFieldAccessorDefinitions(io::Printer* printer);
-  void GenerateFieldAccessorDefinitions(io::Printer* printer, bool is_inline);
+  void GenerateFieldAccessorDefinitions(io::Printer* printer);
 
   // Generate the table-driven parsing array.  Returns the number of entries
   // generated.
@@ -189,7 +189,7 @@ class MessageGenerator {
                                     std::map<string, string> vars,
                                     io::Printer* printer);
   // Generates has_foo() functions and variables for oneof field has-bits.
-  void GenerateOneofHasBits(io::Printer* printer, bool is_inline);
+  void GenerateOneofHasBits(io::Printer* printer);
   // Generates has_foo_bar() functions for oneof members.
   void GenerateOneofMemberHasBits(const FieldDescriptor* field,
                                   const std::map<string, string>& vars,
@@ -197,6 +197,7 @@ class MessageGenerator {
   // Generates the clear_foo() method for a field.
   void GenerateFieldClear(const FieldDescriptor* field,
                           const std::map<string, string>& vars,
+                          bool is_inline,
                           io::Printer* printer);
 
   void GenerateConstructorBody(io::Printer* printer,

src/google/protobuf/compiler/cpp/cpp_message_field.h

@@ -57,8 +57,7 @@ class MessageFieldGenerator : public FieldGenerator {
   void GenerateDependentAccessorDeclarations(io::Printer* printer) const;
   void GenerateAccessorDeclarations(io::Printer* printer) const;
   void GenerateDependentInlineAccessorDefinitions(io::Printer* printer) const;
-  void GenerateInlineAccessorDefinitions(io::Printer* printer,
-                                         bool is_inline) const;
+  void GenerateInlineAccessorDefinitions(io::Printer* printer) const;
   void GenerateNonInlineAccessorDefinitions(io::Printer* printer) const;
   void GenerateClearingCode(io::Printer* printer) const;
   void GenerateMessageClearingCode(io::Printer* printer) const;
@@ -73,11 +72,6 @@ class MessageFieldGenerator : public FieldGenerator {
   void GenerateByteSize(io::Printer* printer) const;
 
  protected:
-  void GenerateArenaManipulationCode(const std::map<string, string>& variables,
-                                     io::Printer* printer) const;
-
-  virtual void GenerateGetterDeclaration(io::Printer* printer) const;
-
   const FieldDescriptor* descriptor_;
   const bool dependent_field_;
   const bool implicit_weak_field_;
@@ -94,11 +88,9 @@ class MessageOneofFieldGenerator : public MessageFieldGenerator {
   ~MessageOneofFieldGenerator();
 
   // implements FieldGenerator ---------------------------------------
-  void GenerateDependentAccessorDeclarations(io::Printer* printer) const;
   void GenerateDependentInlineAccessorDefinitions(io::Printer* printer) const;
-  void GenerateInlineAccessorDefinitions(io::Printer* printer,
-                                         bool is_inline) const;
-  void GenerateNonInlineAccessorDefinitions(io::Printer* printer) const { }
+  void GenerateInlineAccessorDefinitions(io::Printer* printer) const;
+  void GenerateNonInlineAccessorDefinitions(io::Printer* printer) const;
   void GenerateClearingCode(io::Printer* printer) const;
 
   // MessageFieldGenerator, from which we inherit, overrides this so we need to
@@ -108,9 +100,6 @@ class MessageOneofFieldGenerator : public MessageFieldGenerator {
   void GenerateDestructorCode(io::Printer* printer) const;
   void GenerateConstructorCode(io::Printer* printer) const;
 
- protected:
-  void GenerateGetterDeclaration(io::Printer* printer) const;
-
  private:
   void InternalGenerateInlineAccessorDefinitions(
       const std::map<string, string>& variables, io::Printer* printer) const;
@@ -130,8 +119,7 @@ class RepeatedMessageFieldGenerator : public FieldGenerator {
   void GenerateDependentAccessorDeclarations(io::Printer* printer) const;
   void GenerateAccessorDeclarations(io::Printer* printer) const;
   void GenerateDependentInlineAccessorDefinitions(io::Printer* printer) const;
-  void GenerateInlineAccessorDefinitions(io::Printer* printer,
-                                         bool is_inline) const;
+  void GenerateInlineAccessorDefinitions(io::Printer* printer) const;
   void GenerateClearingCode(io::Printer* printer) const;
   void GenerateMergingCode(io::Printer* printer) const;
   void GenerateSwappingCode(io::Printer* printer) const;

src/google/protobuf/compiler/cpp/cpp_plugin_unittest.cc

@@ -132,7 +132,7 @@ class TestGenerator : public CodeGenerator {
     // Check field accessors for a message inside oneof{}:
     TryInsert("test.pb.h", "field_get:foo.Bar.oneOfMessage", context);
     TryInsert("test.pb.h", "field_mutable:foo.Bar.oneOfMessage", context);
-    TryInsert("test.pb.h", "field_set_allocated:foo.Bar.oneOfMessage", context);
+    TryInsert("test.pb.cc", "field_set_allocated:foo.Bar.oneOfMessage", context);
 
     // Check field accessors for an optional enum:
     TryInsert("test.pb.h", "field_get:foo.Bar.optEnum", context);

src/google/protobuf/compiler/cpp/cpp_primitive_field.cc

@@ -123,15 +123,13 @@ GenerateAccessorDeclarations(io::Printer* printer) const {
 }
 
 void PrimitiveFieldGenerator::
-GenerateInlineAccessorDefinitions(io::Printer* printer, bool is_inline) const {
-  std::map<string, string> variables(variables_);
-  variables["inline"] = is_inline ? "inline " : "";
-  printer->Print(variables,
-    "$inline$$type$ $classname$::$name$() const {\n"
+GenerateInlineAccessorDefinitions(io::Printer* printer) const {
+  printer->Print(variables_,
+    "inline $type$ $classname$::$name$() const {\n"
     "  // @@protoc_insertion_point(field_get:$full_name$)\n"
     "  return $name$_;\n"
     "}\n"
-    "$inline$void $classname$::set_$name$($type$ value) {\n"
+    "inline void $classname$::set_$name$($type$ value) {\n"
     "  $set_hasbit$\n"
     "  $name$_ = value;\n"
     "  // @@protoc_insertion_point(field_set:$full_name$)\n"
@@ -212,18 +210,16 @@ PrimitiveOneofFieldGenerator(const FieldDescriptor* descriptor,
 PrimitiveOneofFieldGenerator::~PrimitiveOneofFieldGenerator() {}
 
 void PrimitiveOneofFieldGenerator::
-GenerateInlineAccessorDefinitions(io::Printer* printer, bool is_inline) const {
-  std::map<string, string> variables(variables_);
-  variables["inline"] = is_inline ? "inline " : "";
-  printer->Print(variables,
-    "$inline$$type$ $classname$::$name$() const {\n"
+GenerateInlineAccessorDefinitions(io::Printer* printer) const {
+  printer->Print(variables_,
+    "inline $type$ $classname$::$name$() const {\n"
     "  // @@protoc_insertion_point(field_get:$full_name$)\n"
     "  if (has_$name$()) {\n"
     "    return $oneof_prefix$$name$_;\n"
     "  }\n"
     "  return $default$;\n"
     "}\n"
-    "$inline$void $classname$::set_$name$($type$ value) {\n"
+    "inline void $classname$::set_$name$($type$ value) {\n"
     "  if (!has_$name$()) {\n"
     "    clear_$oneof_name$();\n"
     "    set_has_$name$();\n"
@@ -311,28 +307,26 @@ GenerateAccessorDeclarations(io::Printer* printer) const {
 }
 
 void RepeatedPrimitiveFieldGenerator::
-GenerateInlineAccessorDefinitions(io::Printer* printer, bool is_inline) const {
-  std::map<string, string> variables(variables_);
-  variables["inline"] = is_inline ? "inline " : "";
-  printer->Print(variables,
-    "$inline$$type$ $classname$::$name$(int index) const {\n"
+GenerateInlineAccessorDefinitions(io::Printer* printer) const {
+  printer->Print(variables_,
+    "inline $type$ $classname$::$name$(int index) const {\n"
     "  // @@protoc_insertion_point(field_get:$full_name$)\n"
     "  return $name$_.Get(index);\n"
     "}\n"
-    "$inline$void $classname$::set_$name$(int index, $type$ value) {\n"
+    "inline void $classname$::set_$name$(int index, $type$ value) {\n"
     "  $name$_.Set(index, value);\n"
     "  // @@protoc_insertion_point(field_set:$full_name$)\n"
     "}\n"
-    "$inline$void $classname$::add_$name$($type$ value) {\n"
+    "inline void $classname$::add_$name$($type$ value) {\n"
     "  $name$_.Add(value);\n"
     "  // @@protoc_insertion_point(field_add:$full_name$)\n"
     "}\n"
-    "$inline$const ::google::protobuf::RepeatedField< $type$ >&\n"
+    "inline const ::google::protobuf::RepeatedField< $type$ >&\n"
     "$classname$::$name$() const {\n"
     "  // @@protoc_insertion_point(field_list:$full_name$)\n"
     "  return $name$_;\n"
     "}\n"
-    "$inline$::google::protobuf::RepeatedField< $type$ >*\n"
+    "inline ::google::protobuf::RepeatedField< $type$ >*\n"
     "$classname$::mutable_$name$() {\n"
     "  // @@protoc_insertion_point(field_mutable_list:$full_name$)\n"
     "  return &$name$_;\n"

src/google/protobuf/compiler/cpp/cpp_primitive_field.h

@@ -53,8 +53,7 @@ class PrimitiveFieldGenerator : public FieldGenerator {
   // implements FieldGenerator ---------------------------------------
   void GeneratePrivateMembers(io::Printer* printer) const;
   void GenerateAccessorDeclarations(io::Printer* printer) const;
-  void GenerateInlineAccessorDefinitions(io::Printer* printer,
-                                         bool is_inline) const;
+  void GenerateInlineAccessorDefinitions(io::Printer* printer) const;
   void GenerateClearingCode(io::Printer* printer) const;
   void GenerateMergingCode(io::Printer* printer) const;
   void GenerateSwappingCode(io::Printer* printer) const;
@@ -80,8 +79,7 @@ class PrimitiveOneofFieldGenerator : public PrimitiveFieldGenerator {
   ~PrimitiveOneofFieldGenerator();
 
   // implements FieldGenerator ---------------------------------------
-  void GenerateInlineAccessorDefinitions(io::Printer* printer,
-                                         bool is_inline) const;
+  void GenerateInlineAccessorDefinitions(io::Printer* printer) const;
   void GenerateClearingCode(io::Printer* printer) const;
   void GenerateSwappingCode(io::Printer* printer) const;
   void GenerateConstructorCode(io::Printer* printer) const;
@@ -100,8 +98,7 @@ class RepeatedPrimitiveFieldGenerator : public FieldGenerator {
   // implements FieldGenerator ---------------------------------------
   void GeneratePrivateMembers(io::Printer* printer) const;
   void GenerateAccessorDeclarations(io::Printer* printer) const;
-  void GenerateInlineAccessorDefinitions(io::Printer* printer,
-                                         bool is_inline) const;
+  void GenerateInlineAccessorDefinitions(io::Printer* printer) const;
   void GenerateClearingCode(io::Printer* printer) const;
   void GenerateMergingCode(io::Printer* printer) const;
   void GenerateSwappingCode(io::Printer* printer) const;

src/google/protobuf/compiler/cpp/cpp_string_field.cc

@@ -180,16 +180,21 @@ GenerateAccessorDeclarations(io::Printer* printer) const {
   if (SupportsArenas(descriptor_)) {
     printer->Print(
         variables_,
-        "$deprecated_attr$::std::string* ${$unsafe_arena_release_$name$$}$();\n");
+        "PROTOBUF_RUNTIME_DEPRECATED(\"The unsafe_arena_ accessors for\"\n"
+        "\"    string fields are deprecated and will be removed in a\"\n"
+        "\"    future release.\")\n"
+        "::std::string* ${$unsafe_arena_release_$name$$}$();\n");
     printer->Annotate("{", "}", descriptor_);
     printer->Print(
         variables_,
-        "$deprecated_attr$void ${$unsafe_arena_set_allocated_$name$$}$(\n"
+        "PROTOBUF_RUNTIME_DEPRECATED(\"The unsafe_arena_ accessors for\"\n"
+        "\"    string fields are deprecated and will be removed in a\"\n"
+        "\"    future release.\")\n"
+        "void ${$unsafe_arena_set_allocated_$name$$}$(\n"
         "    ::std::string* $name$);\n");
     printer->Annotate("{", "}", descriptor_);
   }
 
-
   if (unknown_ctype) {
     printer->Outdent();
     printer->Print(" public:\n");
@@ -198,38 +203,35 @@ GenerateAccessorDeclarations(io::Printer* printer) const {
 }
 
 void StringFieldGenerator::
-GenerateInlineAccessorDefinitions(io::Printer* printer,
-                                  bool is_inline) const {
-  std::map<string, string> variables(variables_);
-  variables["inline"] = is_inline ? "inline " : "";
+GenerateInlineAccessorDefinitions(io::Printer* printer) const {
   if (SupportsArenas(descriptor_)) {
     printer->Print(
-        variables,
-        "$inline$const ::std::string& $classname$::$name$() const {\n"
+        variables_,
+        "inline const ::std::string& $classname$::$name$() const {\n"
         "  // @@protoc_insertion_point(field_get:$full_name$)\n"
         "  return $name$_.Get();\n"
         "}\n"
-        "$inline$void $classname$::set_$name$(const ::std::string& value) {\n"
+        "inline void $classname$::set_$name$(const ::std::string& value) {\n"
         "  $set_hasbit$\n"
         "  $name$_.Set$lite$($default_variable$, value, GetArenaNoVirtual());\n"
         "  // @@protoc_insertion_point(field_set:$full_name$)\n"
         "}\n"
         "#if LANG_CXX11\n"
-        "$inline$void $classname$::set_$name$(::std::string&& value) {\n"
+        "inline void $classname$::set_$name$(::std::string&& value) {\n"
         "  $set_hasbit$\n"
         "  $name$_.Set$lite$(\n"
         "    $default_variable$, ::std::move(value), GetArenaNoVirtual());\n"
         "  // @@protoc_insertion_point(field_set_rvalue:$full_name$)\n"
         "}\n"
         "#endif\n"
-        "$inline$void $classname$::set_$name$(const char* value) {\n"
+        "inline void $classname$::set_$name$(const char* value) {\n"
         "  $null_check$"
         "  $set_hasbit$\n"
         "  $name$_.Set$lite$($default_variable$, $string_piece$(value),\n"
         "              GetArenaNoVirtual());\n"
         "  // @@protoc_insertion_point(field_set_char:$full_name$)\n"
         "}\n"
-        "$inline$"
+        "inline "
         "void $classname$::set_$name$(const $pointer_type$* value,\n"
         "    size_t size) {\n"
         "  $set_hasbit$\n"
@@ -238,25 +240,17 @@ GenerateInlineAccessorDefinitions(io::Printer* printer,
         "GetArenaNoVirtual());\n"
         "  // @@protoc_insertion_point(field_set_pointer:$full_name$)\n"
         "}\n"
-        "$inline$::std::string* $classname$::mutable_$name$() {\n"
+        "inline ::std::string* $classname$::mutable_$name$() {\n"
         "  $set_hasbit$\n"
         "  // @@protoc_insertion_point(field_mutable:$full_name$)\n"
         "  return $name$_.Mutable($default_variable$, GetArenaNoVirtual());\n"
         "}\n"
-        "$inline$::std::string* $classname$::$release_name$() {\n"
+        "inline ::std::string* $classname$::$release_name$() {\n"
         "  // @@protoc_insertion_point(field_release:$full_name$)\n"
         "  $clear_hasbit$\n"
         "  return $name$_.Release($default_variable$, GetArenaNoVirtual());\n"
         "}\n"
-        "$inline$::std::string* $classname$::unsafe_arena_release_$name$() {\n"
-        "  // "
-        "@@protoc_insertion_point(field_unsafe_arena_release:$full_name$)\n"
-        "  GOOGLE_DCHECK(GetArenaNoVirtual() != NULL);\n"
-        "  $clear_hasbit$\n"
-        "  return $name$_.UnsafeArenaRelease($default_variable$,\n"
-        "      GetArenaNoVirtual());\n"
-        "}\n"
-        "$inline$void $classname$::set_allocated_$name$(::std::string* $name$) {\n"
+        "inline void $classname$::set_allocated_$name$(::std::string* $name$) {\n"
         "  if ($name$ != NULL) {\n"
         "    $set_hasbit$\n"
         "  } else {\n"
@@ -266,7 +260,15 @@ GenerateInlineAccessorDefinitions(io::Printer* printer,
         "      GetArenaNoVirtual());\n"
         "  // @@protoc_insertion_point(field_set_allocated:$full_name$)\n"
         "}\n"
-        "$inline$void $classname$::unsafe_arena_set_allocated_$name$(\n"
+        "inline ::std::string* $classname$::unsafe_arena_release_$name$() {\n"
+        "  // "
+        "@@protoc_insertion_point(field_unsafe_arena_release:$full_name$)\n"
+        "  GOOGLE_DCHECK(GetArenaNoVirtual() != NULL);\n"
+        "  $clear_hasbit$\n"
+        "  return $name$_.UnsafeArenaRelease($default_variable$,\n"
+        "      GetArenaNoVirtual());\n"
+        "}\n"
+        "inline void $classname$::unsafe_arena_set_allocated_$name$(\n"
         "    ::std::string* $name$) {\n"
         "  GOOGLE_DCHECK(GetArenaNoVirtual() != NULL);\n"
         "  if ($name$ != NULL) {\n"
@@ -282,31 +284,31 @@ GenerateInlineAccessorDefinitions(io::Printer* printer,
   } else {
     // No-arena case.
     printer->Print(
-        variables,
-        "$inline$const ::std::string& $classname$::$name$() const {\n"
+        variables_,
+        "inline const ::std::string& $classname$::$name$() const {\n"
         "  // @@protoc_insertion_point(field_get:$full_name$)\n"
         "  return $name$_.GetNoArena();\n"
         "}\n"
-        "$inline$void $classname$::set_$name$(const ::std::string& value) {\n"
+        "inline void $classname$::set_$name$(const ::std::string& value) {\n"
         "  $set_hasbit$\n"
         "  $name$_.SetNoArena($default_variable$, value);\n"
         "  // @@protoc_insertion_point(field_set:$full_name$)\n"
         "}\n"
         "#if LANG_CXX11\n"
-        "$inline$void $classname$::set_$name$(::std::string&& value) {\n"
+        "inline void $classname$::set_$name$(::std::string&& value) {\n"
         "  $set_hasbit$\n"
         "  $name$_.SetNoArena(\n"
         "    $default_variable$, ::std::move(value));\n"
         "  // @@protoc_insertion_point(field_set_rvalue:$full_name$)\n"
         "}\n"
         "#endif\n"
-        "$inline$void $classname$::set_$name$(const char* value) {\n"
+        "inline void $classname$::set_$name$(const char* value) {\n"
         "  $null_check$"
         "  $set_hasbit$\n"
         "  $name$_.SetNoArena($default_variable$, $string_piece$(value));\n"
         "  // @@protoc_insertion_point(field_set_char:$full_name$)\n"
         "}\n"
-        "$inline$"
+        "inline "
         "void $classname$::set_$name$(const $pointer_type$* value, "
         "size_t size) {\n"
         "  $set_hasbit$\n"
@@ -314,17 +316,17 @@ GenerateInlineAccessorDefinitions(io::Printer* printer,
         "      $string_piece$(reinterpret_cast<const char*>(value), size));\n"
         "  // @@protoc_insertion_point(field_set_pointer:$full_name$)\n"
         "}\n"
-        "$inline$::std::string* $classname$::mutable_$name$() {\n"
+        "inline ::std::string* $classname$::mutable_$name$() {\n"
         "  $set_hasbit$\n"
         "  // @@protoc_insertion_point(field_mutable:$full_name$)\n"
         "  return $name$_.MutableNoArena($default_variable$);\n"
         "}\n"
-        "$inline$::std::string* $classname$::$release_name$() {\n"
+        "inline ::std::string* $classname$::$release_name$() {\n"
         "  // @@protoc_insertion_point(field_release:$full_name$)\n"
         "  $clear_hasbit$\n"
         "  return $name$_.ReleaseNoArena($default_variable$);\n"
         "}\n"
-        "$inline$void $classname$::set_allocated_$name$(::std::string* $name$) {\n"
+        "inline void $classname$::set_allocated_$name$(::std::string* $name$) {\n"
         "  if ($name$ != NULL) {\n"
         "    $set_hasbit$\n"
         "  } else {\n"
@@ -470,15 +472,8 @@ GenerateCopyConstructorCode(io::Printer* printer) const {
 
 void StringFieldGenerator::
 GenerateDestructorCode(io::Printer* printer) const {
-  if (SupportsArenas(descriptor_)) {
-    // The variable |arena| is defined by the enclosing code.
-    // See MessageGenerator::GenerateSharedDestructorCode.
-    printer->Print(variables_,
-      "$name$_.Destroy($default_variable$, arena);\n");
-  } else {
-    printer->Print(variables_,
-      "$name$_.DestroyNoArena($default_variable$);\n");
-  }
+  printer->Print(variables_,
+    "$name$_.DestroyNoArena($default_variable$);\n");
 }
 
 void StringFieldGenerator::
@@ -557,21 +552,18 @@ StringOneofFieldGenerator(const FieldDescriptor* descriptor,
 StringOneofFieldGenerator::~StringOneofFieldGenerator() {}
 
 void StringOneofFieldGenerator::
-GenerateInlineAccessorDefinitions(io::Printer* printer,
-                                  bool is_inline) const {
-  std::map<string, string> variables(variables_);
-  variables["inline"] = is_inline ? "inline " : "";
+GenerateInlineAccessorDefinitions(io::Printer* printer) const {
   if (SupportsArenas(descriptor_)) {
     printer->Print(
-        variables,
-        "$inline$const ::std::string& $classname$::$name$() const {\n"
+        variables_,
+        "inline const ::std::string& $classname$::$name$() const {\n"
         "  // @@protoc_insertion_point(field_get:$full_name$)\n"
         "  if (has_$name$()) {\n"
         "    return $oneof_prefix$$name$_.Get();\n"
         "  }\n"
         "  return *$default_variable$;\n"
         "}\n"
-        "$inline$void $classname$::set_$name$(const ::std::string& value) {\n"
+        "inline void $classname$::set_$name$(const ::std::string& value) {\n"
         "  if (!has_$name$()) {\n"
         "    clear_$oneof_name$();\n"
         "    set_has_$name$();\n"
@@ -582,7 +574,7 @@ GenerateInlineAccessorDefinitions(io::Printer* printer,
         "  // @@protoc_insertion_point(field_set:$full_name$)\n"
         "}\n"
         "#if LANG_CXX11\n"
-        "$inline$void $classname$::set_$name$(::std::string&& value) {\n"
+        "inline void $classname$::set_$name$(::std::string&& value) {\n"
         "  // @@protoc_insertion_point(field_set:$full_name$)\n"
         "  if (!has_$name$()) {\n"
         "    clear_$oneof_name$();\n"
@@ -594,7 +586,7 @@ GenerateInlineAccessorDefinitions(io::Printer* printer,
         "  // @@protoc_insertion_point(field_set_rvalue:$full_name$)\n"
         "}\n"
         "#endif\n"
-        "$inline$void $classname$::set_$name$(const char* value) {\n"
+        "inline void $classname$::set_$name$(const char* value) {\n"
         "  $null_check$"
         "  if (!has_$name$()) {\n"
         "    clear_$oneof_name$();\n"
@@ -605,7 +597,7 @@ GenerateInlineAccessorDefinitions(io::Printer* printer,
         "      $string_piece$(value), GetArenaNoVirtual());\n"
         "  // @@protoc_insertion_point(field_set_char:$full_name$)\n"
         "}\n"
-        "$inline$"
+        "inline "
         "void $classname$::set_$name$(const $pointer_type$* value,\n"
         "                             size_t size) {\n"
         "  if (!has_$name$()) {\n"
@@ -619,7 +611,7 @@ GenerateInlineAccessorDefinitions(io::Printer* printer,
         "      GetArenaNoVirtual());\n"
         "  // @@protoc_insertion_point(field_set_pointer:$full_name$)\n"
         "}\n"
-        "$inline$::std::string* $classname$::mutable_$name$() {\n"
+        "inline ::std::string* $classname$::mutable_$name$() {\n"
         "  if (!has_$name$()) {\n"
         "    clear_$oneof_name$();\n"
         "    set_has_$name$();\n"
@@ -629,7 +621,7 @@ GenerateInlineAccessorDefinitions(io::Printer* printer,
         "      GetArenaNoVirtual());\n"
         "  // @@protoc_insertion_point(field_mutable:$full_name$)\n"
         "}\n"
-        "$inline$::std::string* $classname$::$release_name$() {\n"
+        "inline ::std::string* $classname$::$release_name$() {\n"
         "  // @@protoc_insertion_point(field_release:$full_name$)\n"
         "  if (has_$name$()) {\n"
         "    clear_has_$oneof_name$();\n"
@@ -639,19 +631,7 @@ GenerateInlineAccessorDefinitions(io::Printer* printer,
         "    return NULL;\n"
         "  }\n"
         "}\n"
-        "$inline$::std::string* $classname$::unsafe_arena_release_$name$() {\n"
-        "  // "
-        "@@protoc_insertion_point(field_unsafe_arena_release:$full_name$)\n"
-        "  GOOGLE_DCHECK(GetArenaNoVirtual() != NULL);\n"
-        "  if (has_$name$()) {\n"
-        "    clear_has_$oneof_name$();\n"
-        "    return $oneof_prefix$$name$_.UnsafeArenaRelease(\n"
-        "        $default_variable$, GetArenaNoVirtual());\n"
-        "  } else {\n"
-        "    return NULL;\n"
-        "  }\n"
-        "}\n"
-        "$inline$void $classname$::set_allocated_$name$(::std::string* $name$) {\n"
+        "inline void $classname$::set_allocated_$name$(::std::string* $name$) {\n"
         "  if (!has_$name$()) {\n"
         "    $oneof_prefix$$name$_.UnsafeSetDefault($default_variable$);\n"
         "  }\n"
@@ -663,7 +643,19 @@ GenerateInlineAccessorDefinitions(io::Printer* printer,
         "  }\n"
         "  // @@protoc_insertion_point(field_set_allocated:$full_name$)\n"
         "}\n"
-        "$inline$void $classname$::unsafe_arena_set_allocated_$name$("
+        "inline ::std::string* $classname$::unsafe_arena_release_$name$() {\n"
+        "  // "
+        "@@protoc_insertion_point(field_unsafe_arena_release:$full_name$)\n"
+        "  GOOGLE_DCHECK(GetArenaNoVirtual() != NULL);\n"
+        "  if (has_$name$()) {\n"
+        "    clear_has_$oneof_name$();\n"
+        "    return $oneof_prefix$$name$_.UnsafeArenaRelease(\n"
+        "        $default_variable$, GetArenaNoVirtual());\n"
+        "  } else {\n"
+        "    return NULL;\n"
+        "  }\n"
+        "}\n"
+        "inline void $classname$::unsafe_arena_set_allocated_$name$("
         "::std::string* $name$) {\n"
         "  GOOGLE_DCHECK(GetArenaNoVirtual() != NULL);\n"
         "  if (!has_$name$()) {\n"
@@ -681,15 +673,15 @@ GenerateInlineAccessorDefinitions(io::Printer* printer,
   } else {
     // No-arena case.
     printer->Print(
-        variables,
-        "$inline$const ::std::string& $classname$::$name$() const {\n"
+        variables_,
+        "inline const ::std::string& $classname$::$name$() const {\n"
         "  // @@protoc_insertion_point(field_get:$full_name$)\n"
         "  if (has_$name$()) {\n"
         "    return $oneof_prefix$$name$_.GetNoArena();\n"
         "  }\n"
         "  return *$default_variable$;\n"
         "}\n"
-        "$inline$void $classname$::set_$name$(const ::std::string& value) {\n"
+        "inline void $classname$::set_$name$(const ::std::string& value) {\n"
         "  // @@protoc_insertion_point(field_set:$full_name$)\n"
         "  if (!has_$name$()) {\n"
         "    clear_$oneof_name$();\n"
@@ -700,7 +692,7 @@ GenerateInlineAccessorDefinitions(io::Printer* printer,
         "  // @@protoc_insertion_point(field_set:$full_name$)\n"
         "}\n"
         "#if LANG_CXX11\n"
-        "$inline$void $classname$::set_$name$(::std::string&& value) {\n"
+        "inline void $classname$::set_$name$(::std::string&& value) {\n"
         "  // @@protoc_insertion_point(field_set:$full_name$)\n"
         "  if (!has_$name$()) {\n"
         "    clear_$oneof_name$();\n"
@@ -712,7 +704,7 @@ GenerateInlineAccessorDefinitions(io::Printer* printer,
         "  // @@protoc_insertion_point(field_set_rvalue:$full_name$)\n"
         "}\n"
         "#endif\n"
-        "$inline$void $classname$::set_$name$(const char* value) {\n"
+        "inline void $classname$::set_$name$(const char* value) {\n"
         "  $null_check$"
         "  if (!has_$name$()) {\n"
         "    clear_$oneof_name$();\n"
@@ -723,7 +715,7 @@ GenerateInlineAccessorDefinitions(io::Printer* printer,
         "      $string_piece$(value));\n"
         "  // @@protoc_insertion_point(field_set_char:$full_name$)\n"
         "}\n"
-        "$inline$"
+        "inline "
         "void $classname$::set_$name$(const $pointer_type$* value, size_t "
         "size) {\n"
         "  if (!has_$name$()) {\n"
@@ -736,7 +728,7 @@ GenerateInlineAccessorDefinitions(io::Printer* printer,
         "      reinterpret_cast<const char*>(value), size));\n"
         "  // @@protoc_insertion_point(field_set_pointer:$full_name$)\n"
         "}\n"
-        "$inline$::std::string* $classname$::mutable_$name$() {\n"
+        "inline ::std::string* $classname$::mutable_$name$() {\n"
         "  if (!has_$name$()) {\n"
         "    clear_$oneof_name$();\n"
         "    set_has_$name$();\n"
@@ -745,7 +737,7 @@ GenerateInlineAccessorDefinitions(io::Printer* printer,
         "  // @@protoc_insertion_point(field_mutable:$full_name$)\n"
         "  return $oneof_prefix$$name$_.MutableNoArena($default_variable$);\n"
         "}\n"
-        "$inline$::std::string* $classname$::$release_name$() {\n"
+        "inline ::std::string* $classname$::$release_name$() {\n"
         "  // @@protoc_insertion_point(field_release:$full_name$)\n"
         "  if (has_$name$()) {\n"
         "    clear_has_$oneof_name$();\n"
@@ -754,7 +746,7 @@ GenerateInlineAccessorDefinitions(io::Printer* printer,
         "    return NULL;\n"
         "  }\n"
         "}\n"
-        "$inline$void $classname$::set_allocated_$name$(::std::string* $name$) {\n"
+        "inline void $classname$::set_allocated_$name$(::std::string* $name$) {\n"
         "  if (!has_$name$()) {\n"
         "    $oneof_prefix$$name$_.UnsafeSetDefault($default_variable$);\n"
         "  }\n"
@@ -771,29 +763,13 @@ GenerateInlineAccessorDefinitions(io::Printer* printer,
 
 void StringOneofFieldGenerator::
 GenerateClearingCode(io::Printer* printer) const {
-  std::map<string, string> variables(variables_);
-  if (dependent_field_) {
-    variables["this_message"] = DependentBaseDownCast();
-    // This clearing code may be in the dependent base class. If the default
-    // value is an empty string, then the $default_variable$ is a global
-    // singleton. If the default is not empty, we need to down-cast to get the
-    // default value's global singleton instance. See SetStringVariables() for
-    // possible values of default_variable.
-    if (!descriptor_->default_value_string().empty()) {
-      variables["default_variable"] = "&" + DependentBaseDownCast() +
-                                      variables["default_variable_name"] +
-                                      ".get()";
-    }
-  } else {
-    variables["this_message"] = "";
-  }
   if (SupportsArenas(descriptor_)) {
-    printer->Print(variables,
-      "$this_message$$oneof_prefix$$name$_.Destroy($default_variable$,\n"
-      "    $this_message$GetArenaNoVirtual());\n");
+    printer->Print(variables_,
+      "$oneof_prefix$$name$_.Destroy($default_variable$,\n"
+      "    GetArenaNoVirtual());\n");
   } else {
-    printer->Print(variables,
-      "$this_message$$oneof_prefix$$name$_."
+    printer->Print(variables_,
+      "$oneof_prefix$$name$_."
       "DestroyNoArena($default_variable$);\n");
   }
 }
@@ -818,18 +794,10 @@ GenerateConstructorCode(io::Printer* printer) const {
 
 void StringOneofFieldGenerator::
 GenerateDestructorCode(io::Printer* printer) const {
-  if (SupportsArenas(descriptor_)) {
-    printer->Print(variables_,
-      "if (has_$name$()) {\n"
-      "  $oneof_prefix$$name$_.Destroy($default_variable$,\n"
-      "      GetArenaNoVirtual());\n"
-      "}\n");
-  } else {
-    printer->Print(variables_,
-      "if (has_$name$()) {\n"
-      "  $oneof_prefix$$name$_.DestroyNoArena($default_variable$);\n"
-      "}\n");
-  }
+  printer->Print(variables_,
+    "if (has_$name$()) {\n"
+    "  $oneof_prefix$$name$_.DestroyNoArena($default_variable$);\n"
+    "}\n");
 }
 
 void StringOneofFieldGenerator::
@@ -943,71 +911,68 @@ GenerateAccessorDeclarations(io::Printer* printer) const {
 }
 
 void RepeatedStringFieldGenerator::
-GenerateInlineAccessorDefinitions(io::Printer* printer,
-                                  bool is_inline) const {
-  std::map<string, string> variables(variables_);
-  variables["inline"] = is_inline ? "inline " : "";
-  printer->Print(variables,
-    "$inline$const ::std::string& $classname$::$name$(int index) const {\n"
+GenerateInlineAccessorDefinitions(io::Printer* printer) const {
+  printer->Print(variables_,
+    "inline const ::std::string& $classname$::$name$(int index) const {\n"
     "  // @@protoc_insertion_point(field_get:$full_name$)\n"
     "  return $name$_.$cppget$(index);\n"
     "}\n"
-    "$inline$::std::string* $classname$::mutable_$name$(int index) {\n"
+    "inline ::std::string* $classname$::mutable_$name$(int index) {\n"
     "  // @@protoc_insertion_point(field_mutable:$full_name$)\n"
     "  return $name$_.Mutable(index);\n"
     "}\n"
-    "$inline$void $classname$::set_$name$(int index, const ::std::string& value) {\n"
+    "inline void $classname$::set_$name$(int index, const ::std::string& value) {\n"
     "  // @@protoc_insertion_point(field_set:$full_name$)\n"
     "  $name$_.Mutable(index)->assign(value);\n"
     "}\n"
     "#if LANG_CXX11\n"
-    "$inline$void $classname$::set_$name$(int index, ::std::string&& value) {\n"
+    "inline void $classname$::set_$name$(int index, ::std::string&& value) {\n"
     "  // @@protoc_insertion_point(field_set:$full_name$)\n"
     "  $name$_.Mutable(index)->assign(std::move(value));\n"
     "}\n"
     "#endif\n"
-    "$inline$void $classname$::set_$name$(int index, const char* value) {\n"
+    "inline void $classname$::set_$name$(int index, const char* value) {\n"
     "  $null_check$"
     "  $name$_.Mutable(index)->assign(value);\n"
     "  // @@protoc_insertion_point(field_set_char:$full_name$)\n"
     "}\n"
-    "$inline$void "
+    "inline void "
     "$classname$::set_$name$"
     "(int index, const $pointer_type$* value, size_t size) {\n"
     "  $name$_.Mutable(index)->assign(\n"
     "    reinterpret_cast<const char*>(value), size);\n"
     "  // @@protoc_insertion_point(field_set_pointer:$full_name$)\n"
     "}\n"
-    "$inline$::std::string* $classname$::add_$name$() {\n"
+    "inline ::std::string* $classname$::add_$name$() {\n"
     "  // @@protoc_insertion_point(field_add_mutable:$full_name$)\n"
     "  return $name$_.Add();\n"
     "}\n"
-    "$inline$void $classname$::add_$name$(const ::std::string& value) {\n"
+    "inline void $classname$::add_$name$(const ::std::string& value) {\n"
     "  $name$_.Add()->assign(value);\n"
     "  // @@protoc_insertion_point(field_add:$full_name$)\n"
     "}\n"
     "#if LANG_CXX11\n"
-    "$inline$void $classname$::add_$name$(::std::string&& value) {\n"
+    "inline void $classname$::add_$name$(::std::string&& value) {\n"
     "  $name$_.Add(std::move(value));\n"
     "  // @@protoc_insertion_point(field_add:$full_name$)\n"
     "}\n"
     "#endif\n"
-    "$inline$void $classname$::add_$name$(const char* value) {\n"
+    "inline void $classname$::add_$name$(const char* value) {\n"
     "  $null_check$"
     "  $name$_.Add()->assign(value);\n"
     "  // @@protoc_insertion_point(field_add_char:$full_name$)\n"
     "}\n"
-    "$inline$void "
+    "inline void "
     "$classname$::add_$name$(const $pointer_type$* value, size_t size) {\n"
     "  $name$_.Add()->assign(reinterpret_cast<const char*>(value), size);\n"
     "  // @@protoc_insertion_point(field_add_pointer:$full_name$)\n"
     "}\n"
-    "$inline$const ::google::protobuf::RepeatedPtrField< ::std::string>&\n"
+    "inline const ::google::protobuf::RepeatedPtrField< ::std::string>&\n"
     "$classname$::$name$() const {\n"
     "  // @@protoc_insertion_point(field_list:$full_name$)\n"
     "  return $name$_;\n"
     "}\n"
-    "$inline$::google::protobuf::RepeatedPtrField< ::std::string>*\n"
+    "inline ::google::protobuf::RepeatedPtrField< ::std::string>*\n"
     "$classname$::mutable_$name$() {\n"
     "  // @@protoc_insertion_point(field_mutable_list:$full_name$)\n"
     "  return &$name$_;\n"

src/google/protobuf/compiler/cpp/cpp_string_field.h

@@ -54,8 +54,7 @@ class StringFieldGenerator : public FieldGenerator {
   void GeneratePrivateMembers(io::Printer* printer) const;
   void GenerateStaticMembers(io::Printer* printer) const;
   void GenerateAccessorDeclarations(io::Printer* printer) const;
-  void GenerateInlineAccessorDefinitions(io::Printer* printer,
-                                         bool is_inline) const;
+  void GenerateInlineAccessorDefinitions(io::Printer* printer) const;
   void GenerateNonInlineAccessorDefinitions(io::Printer* printer) const;
   void GenerateClearingCode(io::Printer* printer) const;
   void GenerateMessageClearingCode(io::Printer* printer) const;
@@ -86,8 +85,7 @@ class StringOneofFieldGenerator : public StringFieldGenerator {
   ~StringOneofFieldGenerator();
 
   // implements FieldGenerator ---------------------------------------
-  void GenerateInlineAccessorDefinitions(io::Printer* printer,
-                                         bool is_inline) const;
+  void GenerateInlineAccessorDefinitions(io::Printer* printer) const;
   void GenerateClearingCode(io::Printer* printer) const;
 
   // StringFieldGenerator, from which we inherit, overrides this so we need to
@@ -112,8 +110,7 @@ class RepeatedStringFieldGenerator : public FieldGenerator {
   // implements FieldGenerator ---------------------------------------
   void GeneratePrivateMembers(io::Printer* printer) const;
   void GenerateAccessorDeclarations(io::Printer* printer) const;
-  void GenerateInlineAccessorDefinitions(io::Printer* printer,
-                                         bool is_inline) const;
+  void GenerateInlineAccessorDefinitions(io::Printer* printer) const;
   void GenerateClearingCode(io::Printer* printer) const;
   void GenerateMergingCode(io::Printer* printer) const;
   void GenerateSwappingCode(io::Printer* printer) const;

src/google/protobuf/compiler/cpp/metadata_test.cc

@@ -35,10 +35,8 @@
 
 #include <google/protobuf/compiler/cpp/cpp_helpers.h>
 #include <google/protobuf/compiler/cpp/cpp_generator.h>
+#include <google/protobuf/compiler/annotation_test_util.h>
 #include <google/protobuf/compiler/command_line_interface.h>
-#include <google/protobuf/io/zero_copy_stream.h>
-#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
-#include <google/protobuf/io/printer.h>
 #include <google/protobuf/descriptor.pb.h>
 
 #include <google/protobuf/testing/file.h>
@@ -47,37 +45,15 @@
 #include <gtest/gtest.h>
 
 namespace google {
+namespace atu = ::google::protobuf::compiler::annotation_test_util;
+
 namespace protobuf {
 namespace compiler {
 namespace cpp {
 namespace {
 
-// A CodeGenerator that captures the FileDescriptor it's passed as a
-// FileDescriptorProto.
-class DescriptorCapturingGenerator : public CodeGenerator {
- public:
-  // Does not own file; file must outlive the Generator.
-  explicit DescriptorCapturingGenerator(FileDescriptorProto* file)
-      : file_(file) {}
-
-  virtual bool Generate(const FileDescriptor* file, const string& parameter,
-                        GeneratorContext* context, string* error) const {
-    file->CopyTo(file_);
-    return true;
-  }
-
- private:
-  FileDescriptorProto* file_;
-};
-
 class CppMetadataTest : public ::testing::Test {
  public:
-  // Adds a file with name `filename` and content `data`.
-  void AddFile(const string& filename, const string& data) {
-    GOOGLE_CHECK_OK(File::SetContents(TestTempDir() + "/" + filename, data,
-                               true));
-  }
-
   // Tries to capture a FileDescriptorProto, GeneratedCodeInfo, and output
   // code from the previously added file with name `filename`. Returns true on
   // success. If pb_h is non-null, expects a .pb.h and a .pb.h.meta (copied to
@@ -87,26 +63,21 @@ class CppMetadataTest : public ::testing::Test {
                        string* proto_h, GeneratedCodeInfo* proto_h_info,
                        string* pb_cc) {
     google::protobuf::compiler::CommandLineInterface cli;
-    cli.SetInputsAreProtoPathRelative(true);
-
     CppGenerator cpp_generator;
-    DescriptorCapturingGenerator capturing_generator(file);
     cli.RegisterGenerator("--cpp_out", &cpp_generator, "");
-    cli.RegisterGenerator("--capture_out", &capturing_generator, "");
-
-    string proto_path = "-I" + TestTempDir();
     string cpp_out =
         "--cpp_out=annotate_headers=true,"
         "annotation_pragma_name=pragma_name,"
         "annotation_guard_name=guard_name:" +
         TestTempDir();
-    string capture_out = "--capture_out=" + TestTempDir();
 
-    const char* argv[] = {"protoc", proto_path.c_str(), cpp_out.c_str(),
-                          capture_out.c_str(), filename.c_str()};
+    const bool result =
+        atu::CaptureMetadata(filename, cpp_out,
+                             /* meta_file_suffix */ "", &cli, file,
+                             /* outputs */ NULL);
 
-    if (cli.Run(5, argv) != 0) {
-      return false;
+    if (!result) {
+      return result;
     }
 
     string output_base = TestTempDir() + "/" + StripProto(filename);
@@ -119,7 +90,7 @@ class CppMetadataTest : public ::testing::Test {
     if (pb_h != NULL && pb_h_info != NULL) {
       GOOGLE_CHECK_OK(
           File::GetContents(output_base + ".pb.h", pb_h, true));
-      if (!DecodeMetadata(output_base + ".pb.h.meta", pb_h_info)) {
+      if (!atu::DecodeMetadata(output_base + ".pb.h.meta", pb_h_info)) {
         return false;
       }
     }
@@ -127,23 +98,13 @@ class CppMetadataTest : public ::testing::Test {
     if (proto_h != NULL && proto_h_info != NULL) {
       GOOGLE_CHECK_OK(File::GetContents(output_base + ".proto.h", proto_h,
                                  true));
-      if (!DecodeMetadata(output_base + ".proto.h.meta", proto_h_info)) {
+      if (!atu::DecodeMetadata(output_base + ".proto.h.meta", proto_h_info)) {
         return false;
       }
     }
 
     return true;
   }
-
- private:
-  // Decodes GeneratedCodeInfo stored in path and copies it to info.
-  // Returns true on success.
-  bool DecodeMetadata(const string& path, GeneratedCodeInfo* info) {
-    string data;
-    GOOGLE_CHECK_OK(File::GetContents(path, &data, true));
-    io::ArrayInputStream input(data.data(), data.size());
-    return info->ParseFromZeroCopyStream(&input);
-  }
 };
 
 const char kSmallTestFile[] =
@@ -152,48 +113,11 @@ const char kSmallTestFile[] =
     "enum Enum { VALUE = 0; }\n"
     "message Message { }\n";
 
-// Finds the Annotation for a given source file and path (or returns null if it
-// couldn't).
-const GeneratedCodeInfo::Annotation* FindAnnotationOnPath(
-    const GeneratedCodeInfo& info, const string& source_file,
-    const std::vector<int>& path) {
-  for (int i = 0; i < info.annotation_size(); ++i) {
-    const GeneratedCodeInfo::Annotation* annotation = &info.annotation(i);
-    if (annotation->source_file() != source_file ||
-        annotation->path_size() != path.size()) {
-      continue;
-    }
-    int node = 0;
-    for (; node < path.size(); ++node) {
-      if (annotation->path(node) != path[node]) {
-        break;
-      }
-    }
-    if (node == path.size()) {
-      return annotation;
-    }
-  }
-  return NULL;
-}
-
-// Returns true if the provided annotation covers a given substring in
-// file_content.
-bool AnnotationMatchesSubstring(const string& file_content,
-                                const GeneratedCodeInfo::Annotation* annotation,
-                                const string& expected_text) {
-  uint32 begin = annotation->begin();
-  uint32 end = annotation->end();
-  if (end < begin || end > file_content.size()) {
-    return false;
-  }
-  return file_content.substr(begin, end - begin) == expected_text;
-}
-
 TEST_F(CppMetadataTest, CapturesEnumNames) {
   FileDescriptorProto file;
   GeneratedCodeInfo info;
   string pb_h;
-  AddFile("test.proto", kSmallTestFile);
+  atu::AddFile("test.proto", kSmallTestFile);
   EXPECT_TRUE(
       CaptureMetadata("test.proto", &file, &pb_h, &info, NULL, NULL, NULL));
   EXPECT_EQ("Enum", file.enum_type(0).name());
@@ -201,16 +125,16 @@ TEST_F(CppMetadataTest, CapturesEnumNames) {
   enum_path.push_back(FileDescriptorProto::kEnumTypeFieldNumber);
   enum_path.push_back(0);
   const GeneratedCodeInfo::Annotation* enum_annotation =
-      FindAnnotationOnPath(info, "test.proto", enum_path);
+      atu::FindAnnotationOnPath(info, "test.proto", enum_path);
   EXPECT_TRUE(NULL != enum_annotation);
-  EXPECT_TRUE(AnnotationMatchesSubstring(pb_h, enum_annotation, "Enum"));
+  EXPECT_TRUE(atu::AnnotationMatchesSubstring(pb_h, enum_annotation, "Enum"));
 }
 
 TEST_F(CppMetadataTest, AddsPragma) {
   FileDescriptorProto file;
   GeneratedCodeInfo info;
   string pb_h;
-  AddFile("test.proto", kSmallTestFile);
+  atu::AddFile("test.proto", kSmallTestFile);
   EXPECT_TRUE(
       CaptureMetadata("test.proto", &file, &pb_h, &info, NULL, NULL, NULL));
   EXPECT_TRUE(pb_h.find("#ifdef guard_name") != string::npos);
@@ -222,7 +146,7 @@ TEST_F(CppMetadataTest, CapturesMessageNames) {
   FileDescriptorProto file;
   GeneratedCodeInfo info;
   string pb_h;
-  AddFile("test.proto", kSmallTestFile);
+  atu::AddFile("test.proto", kSmallTestFile);
   EXPECT_TRUE(
       CaptureMetadata("test.proto", &file, &pb_h, &info, NULL, NULL, NULL));
   EXPECT_EQ("Message", file.message_type(0).name());
@@ -230,9 +154,10 @@ TEST_F(CppMetadataTest, CapturesMessageNames) {
   message_path.push_back(FileDescriptorProto::kMessageTypeFieldNumber);
   message_path.push_back(0);
   const GeneratedCodeInfo::Annotation* message_annotation =
-      FindAnnotationOnPath(info, "test.proto", message_path);
+      atu::FindAnnotationOnPath(info, "test.proto", message_path);
   EXPECT_TRUE(NULL != message_annotation);
-  EXPECT_TRUE(AnnotationMatchesSubstring(pb_h, message_annotation, "Message"));
+  EXPECT_TRUE(
+      atu::AnnotationMatchesSubstring(pb_h, message_annotation, "Message"));
 }
 
 }  // namespace

src/google/protobuf/compiler/java/java_helpers.cc

@@ -37,6 +37,7 @@
 #include <limits>
 #include <vector>
 
+#include <google/protobuf/stubs/stringprintf.h>
 #include <google/protobuf/compiler/java/java_helpers.h>
 #include <google/protobuf/compiler/java/java_name_resolver.h>
 #include <google/protobuf/descriptor.pb.h>
@@ -783,6 +784,137 @@ bool HasRepeatedFields(const Descriptor* descriptor) {
   return false;
 }
 
+// Encode an unsigned 32-bit value into a sequence of UTF-16 characters.
+//
+// If the value is in [0x0000, 0xD7FF], we encode it with a single character
+// with the same numeric value.
+//
+// If the value is larger than 0xD7FF, we encode its lowest 13 bits into a
+// character in the range [0xE000, 0xFFFF] by combining these 13 bits with
+// 0xE000 using logic-or. Then we shift the value to the right by 13 bits, and
+// encode the remaining value by repeating this same process until we get to
+// a value in [0x0000, 0xD7FF] where we will encode it using a character with
+// the same numeric value.
+//
+// Note that we only use code points in [0x0000, 0xD7FF] and [0xE000, 0xFFFF].
+// There will be no surrogate pairs in the encoded character sequence.
+void WriteUInt32ToUtf16CharSequence(uint32 number,
+                                    std::vector<uint16>* output) {
+  // For values in [0x0000, 0xD7FF], only use one char to encode it.
+  if (number < 0xD800) {
+    output->push_back(static_cast<uint16>(number));
+    return;
+  }
+  // Encode into multiple chars. All except the last char will be in the range
+  // [0xE000, 0xFFFF], and the last char will be in the range [0x0000, 0xD7FF].
+  // Note that we don't use any value in range [0xD800, 0xDFFF] because they
+  // have to come in pairs and the encoding is just more space-efficient w/o
+  // them.
+  while (number >= 0xD800) {
+    // [0xE000, 0xFFFF] can represent 13 bits of info.
+    output->push_back(static_cast<uint16>(0xE000 | (number & 0x1FFF)));
+    number >>= 13;
+  }
+  output->push_back(static_cast<uint16>(number));
+}
+
+int GetExperimentalJavaFieldTypeForSingular(const FieldDescriptor* field) {
+  // j/c/g/protobuf/FieldType.java lists field types in a slightly different
+  // order from FieldDescriptor::Type so we can't do a simple cast.
+  //
+  // TODO(xiaofeng): Make j/c/g/protobuf/FieldType.java follow the same order.
+  int result = field->type();
+  if (result == FieldDescriptor::TYPE_GROUP) {
+    return 17;
+  } else if (result < FieldDescriptor::TYPE_GROUP) {
+    return result - 1;
+  } else {
+    return result - 2;
+  }
+}
+
+int GetExperimentalJavaFieldTypeForRepeated(const FieldDescriptor* field) {
+  if (field->type() == FieldDescriptor::TYPE_GROUP) {
+    return 49;
+  } else {
+    return GetExperimentalJavaFieldTypeForSingular(field) + 18;
+  }
+}
+
+int GetExperimentalJavaFieldTypeForPacked(const FieldDescriptor* field) {
+  int result = field->type();
+  if (result < FieldDescriptor::TYPE_STRING) {
+    return result + 34;
+  } else if (result > FieldDescriptor::TYPE_BYTES) {
+    return result + 30;
+  } else {
+    GOOGLE_LOG(FATAL) << field->full_name() << " can't be packed.";
+    return 0;
+  }
+}
+
+int GetExperimentalJavaFieldType(const FieldDescriptor* field) {
+  static const int kMapFieldType = 50;
+  static const int kOneofFieldTypeOffset = 51;
+  static const int kRequiredBit = 0x100;
+  static const int kUtf8CheckBit = 0x200;
+  static const int kCheckInitialized = 0x400;
+  static const int kMapWithProto2EnumValue = 0x800;
+  int extra_bits = field->is_required() ? kRequiredBit : 0;
+  if (field->type() == FieldDescriptor::TYPE_STRING && CheckUtf8(field)) {
+    extra_bits |= kUtf8CheckBit;
+  }
+  if (field->is_required() || (GetJavaType(field) == JAVATYPE_MESSAGE &&
+                               HasRequiredFields(field->message_type()))) {
+    extra_bits |= kCheckInitialized;
+  }
+
+  if (field->is_map()) {
+    if (SupportFieldPresence(field->file())) {
+      const FieldDescriptor* value =
+          field->message_type()->FindFieldByName("value");
+      if (GetJavaType(value) == JAVATYPE_ENUM) {
+        extra_bits |= kMapWithProto2EnumValue;
+      }
+    }
+    return kMapFieldType | extra_bits;
+  } else if (field->is_packed()) {
+    return GetExperimentalJavaFieldTypeForPacked(field);
+  } else if (field->is_repeated()) {
+    return GetExperimentalJavaFieldTypeForRepeated(field) | extra_bits;
+  } else if (field->containing_oneof() != NULL) {
+    return (GetExperimentalJavaFieldTypeForSingular(field) +
+            kOneofFieldTypeOffset) |
+           extra_bits;
+  } else {
+    return GetExperimentalJavaFieldTypeForSingular(field) | extra_bits;
+  }
+}
+
+// Escape a UTF-16 character to be embedded in a Java string.
+void EscapeUtf16ToString(uint16 code, string* output) {
+  if (code == '\t') {
+    output->append("\\t");
+  } else if (code == '\b') {
+    output->append("\\b");
+  } else if (code == '\n') {
+    output->append("\\n");
+  } else if (code == '\r') {
+    output->append("\\r");
+  } else if (code == '\f') {
+    output->append("\\f");
+  } else if (code == '\'') {
+    output->append("\\'");
+  } else if (code == '\"') {
+    output->append("\\\"");
+  } else if (code == '\\') {
+    output->append("\\\\");
+  } else if (code >= 0x20 && code <= 0x7f) {
+    output->push_back(static_cast<char>(code));
+  } else {
+    output->append(StringPrintf("\\u%04x", code));
+  }
+}
 }  // namespace java
 }  // namespace compiler
 }  // namespace protobuf

src/google/protobuf/compiler/java/java_helpers.h

@@ -394,6 +394,26 @@ inline string GeneratedCodeVersionSuffix() {
 inline bool EnableExperimentalRuntime(Context* context) {
   return false;
 }
+
+void WriteUInt32ToUtf16CharSequence(uint32 number, std::vector<uint16>* output);
+
+inline void WriteIntToUtf16CharSequence(int value,
+                                        std::vector<uint16>* output) {
+  WriteUInt32ToUtf16CharSequence(static_cast<uint32>(value), output);
+}
+
+// Escape a UTF-16 character so it can be embedded in a Java string literal.
+void EscapeUtf16ToString(uint16 code, string* output);
+
+// Only the lowest two bytes of the return value are used. The lowest byte
+// is the integer value of a j/c/g/protobuf/FieldType enum. For the other
+// byte:
+//    bit 0: whether the field is required.
+//    bit 1: whether the field requires UTF-8 validation.
+//    bit 2: whether the field needs isInitialized check.
+//    bit 3: whether the field is a map field with proto2 enum value.
+//    bits 4-7: unused
+int GetExperimentalJavaFieldType(const FieldDescriptor* field);
 }  // namespace java
 }  // namespace compiler
 }  // namespace protobuf

src/google/protobuf/compiler/java/java_message.cc

@@ -371,6 +371,7 @@ void ImmutableMessageGenerator::Generate(io::Printer* printer) {
     "}\n"
     "\n");
 
+
   printer->Print(
     "@java.lang.Override\n"
     "public final com.google.protobuf.UnknownFieldSet\n"

src/google/protobuf/compiler/java/java_message_lite.cc

@@ -69,6 +69,14 @@ using internal::WireFormat;
 using internal::WireFormatLite;
 
 namespace {
+bool EnableExperimentalRuntimeForLite() {
+#ifdef PROTOBUF_EXPERIMENT
+  return PROTOBUF_EXPERIMENT;
+#else   // PROTOBUF_EXPERIMENT
+  return false;
+#endif  // !PROTOBUF_EXPERIMENT
+}
+
 bool GenerateHasBits(const Descriptor* descriptor) {
   return SupportFieldPresence(descriptor->file()) ||
       HasRepeatedFields(descriptor);
@@ -361,14 +369,14 @@ void ImmutableMessageLiteGenerator::Generate(io::Printer* printer) {
   printer->Indent();
   printer->Indent();
 
-  printer->Print(
-    "case IS_INITIALIZED: {\n");
+  printer->Print("case IS_INITIALIZED: {\n");
   printer->Indent();
   GenerateDynamicMethodIsInitialized(printer);
   printer->Outdent();
 
+  printer->Print("}\n");
+
   printer->Print(
-    "}\n"
     "case MAKE_IMMUTABLE: {\n");
 
   printer->Indent();
@@ -383,13 +391,15 @@ void ImmutableMessageLiteGenerator::Generate(io::Printer* printer) {
   GenerateDynamicMethodNewBuilder(printer);
   printer->Outdent();
 
-  printer->Print(
-    "}\n"
-    "case VISIT: {\n");
+  if (!EnableExperimentalRuntimeForLite()) {
+    printer->Print(
+        "}\n"
+        "case VISIT: {\n");
 
-  printer->Indent();
-  GenerateDynamicMethodVisit(printer);
-  printer->Outdent();
+    printer->Indent();
+    GenerateDynamicMethodVisit(printer);
+    printer->Outdent();
+  }
 
   printer->Print(
     "}\n"
@@ -470,6 +480,17 @@ void ImmutableMessageLiteGenerator::Generate(io::Printer* printer) {
     "}\n"
     "\n",
     "classname", descriptor_->name());
+  if (EnableExperimentalRuntimeForLite()) {
+    // Register the default instance in a map. This map will be used by
+    // experimental runtime to lookup default instance given a class instance
+    // without using Java reflection.
+    printer->Print(
+        "static {\n"
+        "  com.google.protobuf.GeneratedMessageLite.registerDefaultInstance(\n"
+        "    $classname$.class, DEFAULT_INSTANCE);\n"
+        "}\n",
+        "classname", descriptor_->name());
+  }
   printer->Print(
       "public static $classname$ getDefaultInstance() {\n"
       "  return DEFAULT_INSTANCE;\n"
@@ -502,6 +523,7 @@ void ImmutableMessageLiteGenerator::Generate(io::Printer* printer) {
   printer->Print("}\n\n");
 }
 
+
 // ===================================================================
 
 void ImmutableMessageLiteGenerator::
@@ -708,10 +730,10 @@ void ImmutableMessageLiteGenerator::GenerateSerializeOneExtensionRange(
 void ImmutableMessageLiteGenerator::GenerateBuilder(io::Printer* printer) {
   printer->Print(
     "public static Builder newBuilder() {\n"
-    "  return DEFAULT_INSTANCE.toBuilder();\n"
+    "  return (Builder) DEFAULT_INSTANCE.createBuilder();\n"
     "}\n"
     "public static Builder newBuilder($classname$ prototype) {\n"
-    "  return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype);\n"
+    "  return (Builder) DEFAULT_INSTANCE.createBuilder(prototype);\n"
     "}\n"
     "\n",
     "classname", name_resolver_->GetImmutableClassName(descriptor_));

src/google/protobuf/compiler/js/js_generator.cc

@@ -244,22 +244,35 @@ string GetPrefix(const GeneratorOptions& options,
   return prefix;
 }
 
+// Returns the fully normalized JavaScript path prefix for the given
+// message descriptor.
+string GetMessagePathPrefix(const GeneratorOptions& options,
+                            const Descriptor* descriptor) {
+  return GetPrefix(
+      options, descriptor->file(),
+      descriptor->containing_type());
+}
+
 // Returns the fully normalized JavaScript path for the given
 // message descriptor.
 string GetMessagePath(const GeneratorOptions& options,
                       const Descriptor* descriptor) {
-  return GetPrefix(
-      options, descriptor->file(),
-      descriptor->containing_type()) + descriptor->name();
+  return GetMessagePathPrefix(options, descriptor) + descriptor->name();
+}
+
+// Returns the fully normalized JavaScript path prefix for the given
+// enumeration descriptor.
+string GetEnumPathPrefix(const GeneratorOptions& options,
+                   const EnumDescriptor* enum_descriptor) {
+  return GetPrefix(options, enum_descriptor->file(),
+      enum_descriptor->containing_type());
 }
 
 // Returns the fully normalized JavaScript path for the given
 // enumeration descriptor.
 string GetEnumPath(const GeneratorOptions& options,
                    const EnumDescriptor* enum_descriptor) {
-  return GetPrefix(
-      options, enum_descriptor->file(),
-      enum_descriptor->containing_type()) + enum_descriptor->name();
+  return GetEnumPathPrefix(options, enum_descriptor) + enum_descriptor->name();
 }
 
 string MaybeCrossFileRef(const GeneratorOptions& options,
@@ -1930,8 +1943,10 @@ void Generator::GenerateClassConstructor(const GeneratorOptions& options,
       " * @extends {jspb.Message}\n"
       " * @constructor\n"
       " */\n"
-      "$classname$ = function(opt_data) {\n",
-      "classname", GetMessagePath(options, desc));
+      "$classprefix$$classname$ = function(opt_data) {\n",
+      "classprefix", GetMessagePathPrefix(options, desc),
+      "classname", desc->name());
+  printer->Annotate("classname", desc);
   string message_id = GetMessageId(desc);
   printer->Print(
       "  jspb.Message.initialize(this, opt_data, $messageId$, $pivot$, "
@@ -2413,12 +2428,13 @@ void Generator::GenerateClassField(const GeneratorOptions& options,
         "keytype", key_type,
         "valuetype", value_type);
     printer->Print(
-        "$class$.prototype.get$name$ = function(opt_noLazyCreate) {\n"
+        "$class$.prototype.$gettername$ = function(opt_noLazyCreate) {\n"
         "  return /** @type {!jspb.Map<$keytype$,$valuetype$>} */ (\n",
         "class", GetMessagePath(options, field->containing_type()),
-        "name", JSGetterName(options, field),
+        "gettername", "get" + JSGetterName(options, field),
         "keytype", key_type,
         "valuetype", value_type);
+    printer->Annotate("gettername", field);
     printer->Print(
         "      jspb.Message.getMapField(this, $index$, opt_noLazyCreate",
         "index", JSFieldIndex(field));
@@ -2457,7 +2473,7 @@ void Generator::GenerateClassField(const GeneratorOptions& options,
                                       /* force_present = */ false,
                                       /* singular_if_not_packed = */ false));
     printer->Print(
-        "$class$.prototype.get$name$ = function() {\n"
+        "$class$.prototype.$gettername$ = function() {\n"
         "  return /** @type{$type$} */ (\n"
         "    jspb.Message.get$rpt$WrapperField(this, $wrapperclass$, "
         "$index$$required$));\n"
@@ -2465,7 +2481,7 @@ void Generator::GenerateClassField(const GeneratorOptions& options,
         "\n"
         "\n",
         "class", GetMessagePath(options, field->containing_type()),
-        "name", JSGetterName(options, field),
+        "gettername", "get" + JSGetterName(options, field),
         "type", JSFieldTypeAnnotation(options, field,
                                       /* is_setter_argument = */ false,
                                       /* force_present = */ false,
@@ -2475,9 +2491,10 @@ void Generator::GenerateClassField(const GeneratorOptions& options,
         "wrapperclass", SubmessageTypeRef(options, field),
         "required", (field->label() == FieldDescriptor::LABEL_REQUIRED ?
                      ", 1" : ""));
+    printer->Annotate("gettername", field);
     printer->Print(
         "/** @param {$optionaltype$} value$returndoc$ */\n"
-        "$class$.prototype.set$name$ = function(value) {\n"
+        "$class$.prototype.$settername$ = function(value) {\n"
         "  jspb.Message.set$oneoftag$$repeatedtag$WrapperField(",
         "optionaltype",
         JSFieldTypeAnnotation(options, field,
@@ -2486,9 +2503,10 @@ void Generator::GenerateClassField(const GeneratorOptions& options,
                               /* singular_if_not_packed = */ false),
         "returndoc", JSReturnDoc(options, field),
         "class", GetMessagePath(options, field->containing_type()),
-        "name", JSGetterName(options, field),
+        "settername", "set" + JSGetterName(options, field),
         "oneoftag", (field->containing_oneof() ? "Oneof" : ""),
         "repeatedtag", (field->is_repeated() ? "Repeated" : ""));
+    printer->Annotate("settername", field);
 
     printer->Print(
         "this, $index$$oneofgroup$, value);$returnvalue$\n"
@@ -2540,9 +2558,10 @@ void Generator::GenerateClassField(const GeneratorOptions& options,
     }
 
     printer->Print(
-        "$class$.prototype.get$name$ = function() {\n",
+        "$class$.prototype.$gettername$ = function() {\n",
         "class", GetMessagePath(options, field->containing_type()),
-        "name", JSGetterName(options, field));
+        "gettername", "get" + JSGetterName(options, field));
+    printer->Annotate("gettername", field);
 
     if (untyped) {
       printer->Print(
@@ -2610,24 +2629,27 @@ void Generator::GenerateClassField(const GeneratorOptions& options,
       // Proto3 non-repeated and non-map fields without presence use the
       // setProto3*Field function.
       printer->Print(
-          "$class$.prototype.set$name$ = function(value) {\n"
+          "$class$.prototype.$settername$ = function(value) {\n"
           "  jspb.Message.setProto3$typetag$Field(this, $index$, "
           "value);$returnvalue$\n"
           "};\n"
           "\n"
           "\n",
-          "class", GetMessagePath(options, field->containing_type()), "name",
-          JSGetterName(options, field), "typetag", JSTypeTag(field), "index",
-          JSFieldIndex(field), "returnvalue", JSReturnClause(field));
+          "class", GetMessagePath(options, field->containing_type()),
+          "settername", "set" + JSGetterName(options, field), "typetag",
+          JSTypeTag(field), "index", JSFieldIndex(field), "returnvalue",
+          JSReturnClause(field));
+      printer->Annotate("settername", field);
     } else {
       // Otherwise, use the regular setField function.
       printer->Print(
-          "$class$.prototype.set$name$ = function(value) {\n"
+          "$class$.prototype.$settername$ = function(value) {\n"
           "  jspb.Message.set$oneoftag$Field(this, $index$",
-          "class", GetMessagePath(options, field->containing_type()), "name",
-          JSGetterName(options, field), "oneoftag",
+          "class", GetMessagePath(options, field->containing_type()),
+          "settername", "set" + JSGetterName(options, field), "oneoftag",
           (field->containing_oneof() ? "Oneof" : ""), "index",
           JSFieldIndex(field));
+      printer->Annotate("settername", field);
       printer->Print(
           "$oneofgroup$, $type$value$rptvalueinit$$typeclose$);$returnvalue$\n"
           "};\n"
@@ -2660,41 +2682,46 @@ void Generator::GenerateClassField(const GeneratorOptions& options,
   // fields with presence.
   if (IsMap(options, field)) {
     printer->Print(
-        "$class$.prototype.clear$name$ = function() {\n"
-        "  this.get$name$().clear();$returnvalue$\n"
+        "$class$.prototype.$clearername$ = function() {\n"
+        "  this.$gettername$().clear();$returnvalue$\n"
         "};\n"
         "\n"
         "\n",
         "class", GetMessagePath(options, field->containing_type()),
-        "name", JSGetterName(options, field),
+        "clearername", "clear" + JSGetterName(options, field),
+        "gettername", "get" + JSGetterName(options, field),
         "returnvalue", JSReturnClause(field));
+    printer->Annotate("clearername", field);
   } else if (field->is_repeated() ||
              (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE &&
               !field->is_required())) {
     // Fields where we can delegate to the regular setter.
     printer->Print(
-        "$class$.prototype.clear$name$ = function() {\n"
-        "  this.set$name$($clearedvalue$);$returnvalue$\n"
+        "$class$.prototype.$clearername$ = function() {\n"
+        "  this.$settername$($clearedvalue$);$returnvalue$\n"
         "};\n"
         "\n"
         "\n",
         "class", GetMessagePath(options, field->containing_type()),
-        "name", JSGetterName(options, field),
+        "clearername", "clear" + JSGetterName(options, field),
+        "settername", "set" + JSGetterName(options, field),
         "clearedvalue", (field->is_repeated() ? "[]" : "undefined"),
         "returnvalue", JSReturnClause(field));
+    printer->Annotate("clearername", field);
   } else if (HasFieldPresence(options, field)) {
     // Fields where we can't delegate to the regular setter because it doesn't
     // accept "undefined" as an argument.
     printer->Print(
-        "$class$.prototype.clear$name$ = function() {\n"
+        "$class$.prototype.$clearername$ = function() {\n"
         "  jspb.Message.set$maybeoneof$Field(this, "
         "$index$$maybeoneofgroup$, ",
         "class", GetMessagePath(options, field->containing_type()),
-        "name", JSGetterName(options, field),
+        "clearername", "clear" + JSGetterName(options, field),
         "maybeoneof", (field->containing_oneof() ? "Oneof" : ""),
         "maybeoneofgroup", (field->containing_oneof() ?
                            (", " + JSOneofArray(options, field)) : ""),
         "index", JSFieldIndex(field));
+    printer->Annotate("clearername", field);
     printer->Print(
         "$clearedvalue$);$returnvalue$\n"
         "};\n"
@@ -2710,14 +2737,15 @@ void Generator::GenerateClassField(const GeneratorOptions& options,
         " * Returns whether this field is set.\n"
         " * @return {!boolean}\n"
         " */\n"
-        "$class$.prototype.has$name$ = function() {\n"
+        "$class$.prototype.$hasername$ = function() {\n"
         "  return jspb.Message.getField(this, $index$) != null;\n"
         "};\n"
         "\n"
         "\n",
         "class", GetMessagePath(options, field->containing_type()),
-        "name", JSGetterName(options, field),
+        "hasername", "has" + JSGetterName(options, field),
         "index", JSFieldIndex(field));
+    printer->Annotate("hasername", field);
   }
 }
 
@@ -2729,13 +2757,14 @@ void Generator::GenerateRepeatedPrimitiveHelperMethods(
       " * @param {!$optionaltype$} value\n"
       " * @param {number=} opt_index\n"
       " */\n"
-      "$class$.prototype.add$name$ = function(value, opt_index) {\n"
+      "$class$.prototype.$addername$ = function(value, opt_index) {\n"
       "  jspb.Message.addToRepeatedField(this, $index$",
-      "class", GetMessagePath(options, field->containing_type()),
-      "name", JSGetterName(options, field, BYTES_DEFAULT,
-                   /* drop_list = */ true),
+      "class", GetMessagePath(options, field->containing_type()), "addername",
+      "add" + JSGetterName(options, field, BYTES_DEFAULT,
+                           /* drop_list = */ true),
       "optionaltype", JSTypeName(options, field, BYTES_DEFAULT), "index",
       JSFieldIndex(field));
+  printer->Annotate("addername", field);
   printer->Print(
       "$oneofgroup$, $type$value$rptvalueinit$$typeclose$, opt_index);\n"
       "};\n"
@@ -3133,8 +3162,10 @@ void Generator::GenerateEnum(const GeneratorOptions& options,
       "/**\n"
       " * @enum {number}\n"
       " */\n"
-      "$name$ = {\n",
-      "name", GetEnumPath(options, enumdesc));
+      "$enumprefix$$name$ = {\n",
+      "enumprefix", GetEnumPathPrefix(options, enumdesc),
+      "name", enumdesc->name());
+  printer->Annotate("name", enumdesc);
 
   for (int i = 0; i < enumdesc->value_count(); i++) {
     const EnumValueDescriptor* value = enumdesc->value(i);
@@ -3143,6 +3174,7 @@ void Generator::GenerateEnum(const GeneratorOptions& options,
         "name", ToEnumCase(value->name()),
         "value", SimpleItoa(value->number()),
         "comma", (i == enumdesc->value_count() - 1) ? "" : ",");
+    printer->Annotate("name", value);
   }
 
   printer->Print(
@@ -3282,6 +3314,12 @@ bool GeneratorOptions::ParseFromOptions(
         return false;
       }
       one_output_file_per_input_file = true;
+    } else if (options[i].first == "annotate_code") {
+      if (!options[i].second.empty()) {
+        *error = "Unexpected option value for annotate_code";
+        return false;
+      }
+      annotate_code = true;
     } else {
       // Assume any other option is an output directory, as long as it is a bare
       // `key` rather than a `key=value` option.
@@ -3582,16 +3620,27 @@ bool Generator::GenerateAll(const std::vector<const FileDescriptor*>& files,
           options.output_dir + "/" + GetJSFilename(options, file->name());
       google::protobuf::scoped_ptr<io::ZeroCopyOutputStream> output(context->Open(filename));
       GOOGLE_CHECK(output.get());
-      io::Printer printer(output.get(), '$');
+      GeneratedCodeInfo annotations;
+      io::AnnotationProtoCollector<GeneratedCodeInfo> annotation_collector(
+          &annotations);
+      io::Printer printer(output.get(), '$',
+                          options.annotate_code ? &annotation_collector : NULL);
+
 
       GenerateFile(options, &printer, file);
 
       if (printer.failed()) {
         return false;
       }
+
+      if (options.annotate_code) {
+        const string meta_file = filename + ".meta";
+        google::protobuf::scoped_ptr<io::ZeroCopyOutputStream> info_output(
+          context->Open(meta_file));
+        annotations.SerializeToZeroCopyStream(info_output.get());
+      }
     }
   }
-
   return true;
 }
 

src/google/protobuf/compiler/js/js_generator.h

@@ -79,7 +79,8 @@ struct GeneratorOptions {
         library(""),
         error_on_name_conflict(false),
         extension(".js"),
-        one_output_file_per_input_file(false) {}
+        one_output_file_per_input_file(false),
+        annotate_code(false) {}
 
   bool ParseFromOptions(
       const std::vector< std::pair< string, string > >& options,
@@ -118,6 +119,9 @@ struct GeneratorOptions {
   string extension;
   // Create a separate output file for each input file?
   bool one_output_file_per_input_file;
+  // If true, we should build .meta files that contain annotations for
+  // generated code. See GeneratedCodeInfo in descriptor.proto.
+  bool annotate_code;
 };
 
 // CodeGenerator implementation which generates a JavaScript source file and

src/google/protobuf/compiler/plugin.pb.cc

@@ -14,6 +14,10 @@
 #include <google/protobuf/generated_message_reflection.h>
 #include <google/protobuf/reflection_ops.h>
 #include <google/protobuf/wire_format.h>
+// This is a temporary google only hack
+#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
+#include "third_party/protobuf/version.h"
+#endif
 // @@protoc_insertion_point(includes)
 namespace google {
 namespace protobuf {
@@ -45,7 +49,11 @@ namespace protobuf_google_2fprotobuf_2fcompiler_2fplugin_2eproto {
 void InitDefaultsVersionImpl() {
   GOOGLE_PROTOBUF_VERIFY_VERSION;
 
+#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
+  ::google::protobuf::internal::InitProtobufDefaultsForceUnique();
+#else
   ::google::protobuf::internal::InitProtobufDefaults();
+#endif  // GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
   {
     void* ptr = &::google::protobuf::compiler::_Version_default_instance_;
     new (ptr) ::google::protobuf::compiler::Version();
@@ -62,7 +70,11 @@ void InitDefaultsVersion() {
 void InitDefaultsCodeGeneratorRequestImpl() {
   GOOGLE_PROTOBUF_VERIFY_VERSION;
 
+#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
+  ::google::protobuf::internal::InitProtobufDefaultsForceUnique();
+#else
   ::google::protobuf::internal::InitProtobufDefaults();
+#endif  // GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
   protobuf_google_2fprotobuf_2fdescriptor_2eproto::InitDefaultsFileDescriptorProto();
   protobuf_google_2fprotobuf_2fcompiler_2fplugin_2eproto::InitDefaultsVersion();
   {
@@ -81,7 +93,11 @@ void InitDefaultsCodeGeneratorRequest() {
 void InitDefaultsCodeGeneratorResponse_FileImpl() {
   GOOGLE_PROTOBUF_VERIFY_VERSION;
 
+#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
+  ::google::protobuf::internal::InitProtobufDefaultsForceUnique();
+#else
   ::google::protobuf::internal::InitProtobufDefaults();
+#endif  // GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
   {
     void* ptr = &::google::protobuf::compiler::_CodeGeneratorResponse_File_default_instance_;
     new (ptr) ::google::protobuf::compiler::CodeGeneratorResponse_File();
@@ -98,7 +114,11 @@ void InitDefaultsCodeGeneratorResponse_File() {
 void InitDefaultsCodeGeneratorResponseImpl() {
   GOOGLE_PROTOBUF_VERIFY_VERSION;
 
+#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
+  ::google::protobuf::internal::InitProtobufDefaultsForceUnique();
+#else
   ::google::protobuf::internal::InitProtobufDefaults();
+#endif  // GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
   protobuf_google_2fprotobuf_2fcompiler_2fplugin_2eproto::InitDefaultsCodeGeneratorResponse_File();
   {
     void* ptr = &::google::protobuf::compiler::_CodeGeneratorResponse_default_instance_;
@@ -319,11 +339,11 @@ void Version::Clear() {
   // Prevent compiler warnings about cached_has_bits being unused
   (void) cached_has_bits;
 
-  if (has_suffix()) {
+  cached_has_bits = _has_bits_[0];
+  if (cached_has_bits & 0x00000001u) {
     GOOGLE_DCHECK(!suffix_.IsDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()));
     (*suffix_.UnsafeRawStringPointer())->clear();
   }
-  cached_has_bits = _has_bits_[0];
   if (cached_has_bits & 14u) {
     ::memset(&major_, 0, static_cast<size_t>(
         reinterpret_cast<char*>(&patch_) -
@@ -634,6 +654,9 @@ void CodeGeneratorRequest::InitAsDefaultInstance() {
   ::google::protobuf::compiler::_CodeGeneratorRequest_default_instance_._instance.get_mutable()->compiler_version_ = const_cast< ::google::protobuf::compiler::Version*>(
       ::google::protobuf::compiler::Version::internal_default_instance());
 }
+void CodeGeneratorRequest::clear_proto_file() {
+  proto_file_.Clear();
+}
 #if !defined(_MSC_VER) || _MSC_VER >= 1900
 const int CodeGeneratorRequest::kFileToGenerateFieldNumber;
 const int CodeGeneratorRequest::kParameterFieldNumber;
@@ -724,7 +747,7 @@ void CodeGeneratorRequest::Clear() {
     }
     if (cached_has_bits & 0x00000002u) {
       GOOGLE_DCHECK(compiler_version_ != NULL);
-      compiler_version_->::google::protobuf::compiler::Version::Clear();
+      compiler_version_->Clear();
     }
   }
   _has_bits_.Clear();
@@ -778,7 +801,7 @@ bool CodeGeneratorRequest::MergePartialFromCodedStream(
       case 3: {
         if (static_cast< ::google::protobuf::uint8>(tag) ==
             static_cast< ::google::protobuf::uint8>(26u /* 26 & 0xFF */)) {
-          DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual(
+          DO_(::google::protobuf::internal::WireFormatLite::ReadMessage(
                input, mutable_compiler_version()));
         } else {
           goto handle_unusual;
@@ -790,8 +813,7 @@ bool CodeGeneratorRequest::MergePartialFromCodedStream(
       case 15: {
         if (static_cast< ::google::protobuf::uint8>(tag) ==
             static_cast< ::google::protobuf::uint8>(122u /* 122 & 0xFF */)) {
-          DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual(
-                input, add_proto_file()));
+          DO_(::google::protobuf::internal::WireFormatLite::ReadMessage(input, add_proto_file()));
         } else {
           goto handle_unusual;
         }
@@ -897,7 +919,7 @@ void CodeGeneratorRequest::SerializeWithCachedSizes(
   // optional .google.protobuf.compiler.Version compiler_version = 3;
   if (cached_has_bits & 0x00000002u) {
     target = ::google::protobuf::internal::WireFormatLite::
-      InternalWriteMessageNoVirtualToArray(
+      InternalWriteMessageToArray(
         3, *this->compiler_version_, deterministic, target);
   }
 
@@ -905,7 +927,7 @@ void CodeGeneratorRequest::SerializeWithCachedSizes(
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->proto_file_size()); i < n; i++) {
     target = ::google::protobuf::internal::WireFormatLite::
-      InternalWriteMessageNoVirtualToArray(
+      InternalWriteMessageToArray(
         15, this->proto_file(static_cast<int>(i)), deterministic, target);
   }
 
@@ -940,7 +962,7 @@ size_t CodeGeneratorRequest::ByteSizeLong() const {
     total_size += 1UL * count;
     for (unsigned int i = 0; i < count; i++) {
       total_size +=
-        ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual(
+        ::google::protobuf::internal::WireFormatLite::MessageSize(
           this->proto_file(static_cast<int>(i)));
     }
   }
@@ -956,7 +978,7 @@ size_t CodeGeneratorRequest::ByteSizeLong() const {
     // optional .google.protobuf.compiler.Version compiler_version = 3;
     if (has_compiler_version()) {
       total_size += 1 +
-        ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual(
+        ::google::protobuf::internal::WireFormatLite::MessageSize(
           *this->compiler_version_);
     }
 
@@ -1511,7 +1533,8 @@ void CodeGeneratorResponse::Clear() {
   (void) cached_has_bits;
 
   file_.Clear();
-  if (has_error()) {
+  cached_has_bits = _has_bits_[0];
+  if (cached_has_bits & 0x00000001u) {
     GOOGLE_DCHECK(!error_.IsDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()));
     (*error_.UnsafeRawStringPointer())->clear();
   }
@@ -1549,8 +1572,7 @@ bool CodeGeneratorResponse::MergePartialFromCodedStream(
       case 15: {
         if (static_cast< ::google::protobuf::uint8>(tag) ==
             static_cast< ::google::protobuf::uint8>(122u /* 122 & 0xFF */)) {
-          DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual(
-                input, add_file()));
+          DO_(::google::protobuf::internal::WireFormatLite::ReadMessage(input, add_file()));
         } else {
           goto handle_unusual;
         }
@@ -1631,7 +1653,7 @@ void CodeGeneratorResponse::SerializeWithCachedSizes(
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->file_size()); i < n; i++) {
     target = ::google::protobuf::internal::WireFormatLite::
-      InternalWriteMessageNoVirtualToArray(
+      InternalWriteMessageToArray(
         15, this->file(static_cast<int>(i)), deterministic, target);
   }
 
@@ -1658,7 +1680,7 @@ size_t CodeGeneratorResponse::ByteSizeLong() const {
     total_size += 1UL * count;
     for (unsigned int i = 0; i < count; i++) {
       total_size +=
-        ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual(
+        ::google::protobuf::internal::WireFormatLite::MessageSize(
           this->file(static_cast<int>(i)));
     }
   }

src/google/protobuf/compiler/plugin.pb.h

@@ -379,8 +379,8 @@ class LIBPROTOC_EXPORT CodeGeneratorRequest : public ::google::protobuf::Message
   void clear_compiler_version();
   static const int kCompilerVersionFieldNumber = 3;
   const ::google::protobuf::compiler::Version& compiler_version() const;
-  ::google::protobuf::compiler::Version* mutable_compiler_version();
   ::google::protobuf::compiler::Version* release_compiler_version();
+  ::google::protobuf::compiler::Version* mutable_compiler_version();
   void set_allocated_compiler_version(::google::protobuf::compiler::Version* compiler_version);
 
   // @@protoc_insertion_point(class_scope:google.protobuf.compiler.CodeGeneratorRequest)
@@ -973,9 +973,6 @@ inline void CodeGeneratorRequest::set_allocated_parameter(::std::string* paramet
 inline int CodeGeneratorRequest::proto_file_size() const {
   return proto_file_.size();
 }
-inline void CodeGeneratorRequest::clear_proto_file() {
-  proto_file_.Clear();
-}
 inline const ::google::protobuf::FileDescriptorProto& CodeGeneratorRequest::proto_file(int index) const {
   // @@protoc_insertion_point(field_get:google.protobuf.compiler.CodeGeneratorRequest.proto_file)
   return proto_file_.Get(index);
@@ -1010,7 +1007,7 @@ inline void CodeGeneratorRequest::clear_has_compiler_version() {
   _has_bits_[0] &= ~0x00000002u;
 }
 inline void CodeGeneratorRequest::clear_compiler_version() {
-  if (compiler_version_ != NULL) compiler_version_->::google::protobuf::compiler::Version::Clear();
+  if (compiler_version_ != NULL) compiler_version_->Clear();
   clear_has_compiler_version();
 }
 inline const ::google::protobuf::compiler::Version& CodeGeneratorRequest::compiler_version() const {
@@ -1019,6 +1016,13 @@ inline const ::google::protobuf::compiler::Version& CodeGeneratorRequest::compil
   return p != NULL ? *p : *reinterpret_cast<const ::google::protobuf::compiler::Version*>(
       &::google::protobuf::compiler::_Version_default_instance_);
 }
+inline ::google::protobuf::compiler::Version* CodeGeneratorRequest::release_compiler_version() {
+  // @@protoc_insertion_point(field_release:google.protobuf.compiler.CodeGeneratorRequest.compiler_version)
+  clear_has_compiler_version();
+  ::google::protobuf::compiler::Version* temp = compiler_version_;
+  compiler_version_ = NULL;
+  return temp;
+}
 inline ::google::protobuf::compiler::Version* CodeGeneratorRequest::mutable_compiler_version() {
   set_has_compiler_version();
   if (compiler_version_ == NULL) {
@@ -1027,21 +1031,22 @@ inline ::google::protobuf::compiler::Version* CodeGeneratorRequest::mutable_comp
   // @@protoc_insertion_point(field_mutable:google.protobuf.compiler.CodeGeneratorRequest.compiler_version)
   return compiler_version_;
 }
-inline ::google::protobuf::compiler::Version* CodeGeneratorRequest::release_compiler_version() {
-  // @@protoc_insertion_point(field_release:google.protobuf.compiler.CodeGeneratorRequest.compiler_version)
-  clear_has_compiler_version();
-  ::google::protobuf::compiler::Version* temp = compiler_version_;
-  compiler_version_ = NULL;
-  return temp;
-}
 inline void CodeGeneratorRequest::set_allocated_compiler_version(::google::protobuf::compiler::Version* compiler_version) {
-  delete compiler_version_;
-  compiler_version_ = compiler_version;
+  ::google::protobuf::Arena* message_arena = GetArenaNoVirtual();
+  if (message_arena == NULL) {
+    delete compiler_version_;
+  }
   if (compiler_version) {
+    ::google::protobuf::Arena* submessage_arena = NULL;
+    if (message_arena != submessage_arena) {
+      compiler_version = ::google::protobuf::internal::GetOwnedMessage(
+          message_arena, compiler_version, submessage_arena);
+    }
     set_has_compiler_version();
   } else {
     clear_has_compiler_version();
   }
+  compiler_version_ = compiler_version;
   // @@protoc_insertion_point(field_set_allocated:google.protobuf.compiler.CodeGeneratorRequest.compiler_version)
 }
 

src/google/protobuf/descriptor.proto

@@ -417,10 +417,12 @@ message FileOptions {
   // determining the namespace.
   optional string php_namespace = 41;
 
-  // The parser stores options it doesn't recognize here. See above.
+  // The parser stores options it doesn't recognize here.
+  // See the documentation for the "Options" section above.
   repeated UninterpretedOption uninterpreted_option = 999;
 
-  // Clients can define custom options in extensions of this message. See above.
+  // Clients can define custom options in extensions of this message.
+  // See the documentation for the "Options" section above.
   extensions 1000 to max;
 
   reserved 38;

src/google/protobuf/duration.pb.cc

@@ -14,6 +14,10 @@
 #include <google/protobuf/generated_message_reflection.h>
 #include <google/protobuf/reflection_ops.h>
 #include <google/protobuf/wire_format.h>
+// This is a temporary google only hack
+#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
+#include "third_party/protobuf/version.h"
+#endif
 // @@protoc_insertion_point(includes)
 namespace google {
 namespace protobuf {
@@ -28,7 +32,11 @@ namespace protobuf_google_2fprotobuf_2fduration_2eproto {
 void InitDefaultsDurationImpl() {
   GOOGLE_PROTOBUF_VERIFY_VERSION;
 
+#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
+  ::google::protobuf::internal::InitProtobufDefaultsForceUnique();
+#else
   ::google::protobuf::internal::InitProtobufDefaults();
+#endif  // GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
   {
     void* ptr = &::google::protobuf::_Duration_default_instance_;
     new (ptr) ::google::protobuf::Duration();
@@ -159,12 +167,7 @@ Duration::~Duration() {
 }
 
 void Duration::SharedDtor() {
-  ::google::protobuf::Arena* arena = GetArenaNoVirtual();
-  GOOGLE_DCHECK(arena == NULL);
-  if (arena != NULL) {
-    return;
-  }
-
+  GOOGLE_DCHECK(GetArenaNoVirtual() == NULL);
 }
 
 void Duration::ArenaDtor(void* object) {

src/google/protobuf/empty.pb.cc

@@ -14,6 +14,10 @@
 #include <google/protobuf/generated_message_reflection.h>
 #include <google/protobuf/reflection_ops.h>
 #include <google/protobuf/wire_format.h>
+// This is a temporary google only hack
+#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
+#include "third_party/protobuf/version.h"
+#endif
 // @@protoc_insertion_point(includes)
 namespace google {
 namespace protobuf {
@@ -28,7 +32,11 @@ namespace protobuf_google_2fprotobuf_2fempty_2eproto {
 void InitDefaultsEmptyImpl() {
   GOOGLE_PROTOBUF_VERIFY_VERSION;
 
+#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
+  ::google::protobuf::internal::InitProtobufDefaultsForceUnique();
+#else
   ::google::protobuf::internal::InitProtobufDefaults();
+#endif  // GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
   {
     void* ptr = &::google::protobuf::_Empty_default_instance_;
     new (ptr) ::google::protobuf::Empty();
@@ -148,12 +156,7 @@ Empty::~Empty() {
 }
 
 void Empty::SharedDtor() {
-  ::google::protobuf::Arena* arena = GetArenaNoVirtual();
-  GOOGLE_DCHECK(arena == NULL);
-  if (arena != NULL) {
-    return;
-  }
-
+  GOOGLE_DCHECK(GetArenaNoVirtual() == NULL);
 }
 
 void Empty::ArenaDtor(void* object) {

src/google/protobuf/field_mask.pb.cc

@@ -14,6 +14,10 @@
 #include <google/protobuf/generated_message_reflection.h>
 #include <google/protobuf/reflection_ops.h>
 #include <google/protobuf/wire_format.h>
+// This is a temporary google only hack
+#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
+#include "third_party/protobuf/version.h"
+#endif
 // @@protoc_insertion_point(includes)
 namespace google {
 namespace protobuf {
@@ -28,7 +32,11 @@ namespace protobuf_google_2fprotobuf_2ffield_5fmask_2eproto {
 void InitDefaultsFieldMaskImpl() {
   GOOGLE_PROTOBUF_VERIFY_VERSION;
 
+#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
+  ::google::protobuf::internal::InitProtobufDefaultsForceUnique();
+#else
   ::google::protobuf::internal::InitProtobufDefaults();
+#endif  // GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
   {
     void* ptr = &::google::protobuf::_FieldMask_default_instance_;
     new (ptr) ::google::protobuf::FieldMask();

src/google/protobuf/field_mask.proto

@@ -240,6 +240,12 @@ option go_package = "google.golang.org/genproto/protobuf/field_mask;field_mask";
 //
 // Note that oneof type names ("test_oneof" in this case) cannot be used in
 // paths.
+//
+// ## Field Mask Verification
+//
+// The implementation of the all the API methods, which have any FieldMask type
+// field in the request, should verify the included field paths, and return
+// `INVALID_ARGUMENT` error if any path is duplicated or unmappable.
 message FieldMask {
   // The set of field mask paths.
   repeated string paths = 1;

src/google/protobuf/generated_message_reflection.h

@@ -45,7 +45,6 @@
 // TODO(jasonh): Remove this once the compiler change to directly include this
 // is released to components.
 #include <google/protobuf/generated_enum_reflection.h>
-#include <google/protobuf/generated_message_util.h>
 #include <google/protobuf/message.h>
 #include <google/protobuf/metadata.h>
 #include <google/protobuf/unknown_field_set.h>

src/google/protobuf/generated_message_util.cc

@@ -43,14 +43,17 @@
 #include <google/protobuf/extension_set.h>
 #include <google/protobuf/message_lite.h>
 #include <google/protobuf/metadata_lite.h>
+#include <google/protobuf/stubs/port.h>
 #include <google/protobuf/repeated_field.h>
 #include <google/protobuf/wire_format_lite.h>
 #include <google/protobuf/wire_format_lite_inl.h>
 
 namespace google {
+
 namespace protobuf {
 namespace internal {
 
+
 double Infinity() {
   return std::numeric_limits<double>::infinity();
 }
@@ -695,6 +698,33 @@ void UnknownFieldSerializerLite(const uint8* ptr, uint32 offset, uint32 tag,
           ->unknown_fields());
 }
 
+MessageLite* DuplicateIfNonNullInternal(MessageLite* message, Arena* arena) {
+  if (message) {
+    MessageLite* ret = message->New(arena);
+    ret->CheckTypeAndMergeFrom(*message);
+    return ret;
+  } else {
+    return NULL;
+  }
+}
+
+// Returns a message owned by this Arena.  This may require Own()ing or
+// duplicating the message.
+MessageLite* GetOwnedMessageInternal(Arena* message_arena,
+                                     MessageLite* submessage,
+                                     Arena* submessage_arena) {
+  GOOGLE_DCHECK(submessage->GetArena() == submessage_arena);
+  GOOGLE_DCHECK(message_arena != submessage_arena);
+  if (message_arena != NULL && submessage_arena == NULL) {
+    message_arena->Own(submessage);
+    return submessage;
+  } else {
+    MessageLite* ret = submessage->New(message_arena);
+    ret->CheckTypeAndMergeFrom(*submessage);
+    return ret;
+  }
+}
+
 }  // namespace internal
 }  // namespace protobuf
 }  // namespace google

src/google/protobuf/generated_message_util.h

@@ -274,6 +274,45 @@ void MapFieldSerializer(const uint8* base, uint32 offset, uint32 tag,
   }
 }
 
+LIBPROTOBUF_EXPORT MessageLite* DuplicateIfNonNullInternal(MessageLite* message, Arena* arena);
+LIBPROTOBUF_EXPORT MessageLite* GetOwnedMessageInternal(Arena* message_arena,
+                                     MessageLite* submessage,
+                                     Arena* submessage_arena);
+
+template <typename T>
+T* DuplicateIfNonNull(T* message, Arena* arena) {
+  // The casts must be reinterpret_cast<> because T might be a forward-declared
+  // type that the compiler doesn't know is related to MessageLite.
+  return reinterpret_cast<T*>(DuplicateIfNonNullInternal(
+      reinterpret_cast<MessageLite*>(message), arena));
+}
+
+template <typename T>
+T* GetOwnedMessage(Arena* message_arena, T* submessage,
+                   Arena* submessage_arena) {
+  // The casts must be reinterpret_cast<> because T might be a forward-declared
+  // type that the compiler doesn't know is related to MessageLite.
+  return reinterpret_cast<T*>(GetOwnedMessageInternal(
+      message_arena, reinterpret_cast<MessageLite*>(submessage),
+      submessage_arena));
+}
+
+// Returns a message owned by this Arena.  This may require Own()ing or
+// duplicating the message.
+template <typename T>
+T* GetOwnedMessage(T* message, Arena* arena) {
+  GOOGLE_DCHECK(message);
+  Arena* message_arena = google::protobuf::Arena::GetArena(message);
+  if (message_arena == arena) {
+    return message;
+  } else if (arena != NULL && message_arena == NULL) {
+    arena->Own(message);
+    return message;
+  } else {
+    return DuplicateIfNonNull(message, arena);
+  }
+}
+
 }  // namespace internal
 }  // namespace protobuf
 

src/google/protobuf/io/coded_stream.h

@@ -862,11 +862,11 @@ class LIBPROTOBUF_EXPORT CodedOutputStream {
   bool IsSerializationDeterministic() const {
     return serialization_deterministic_is_overridden_ ?
         serialization_deterministic_override_ :
-        default_serialization_deterministic_;
+        IsDefaultSerializationDeterministic();
   }
 
   static bool IsDefaultSerializationDeterministic() {
-    return google::protobuf::internal::Acquire_Load(&default_serialization_deterministic_);
+    return google::protobuf::internal::NoBarrier_Load(&default_serialization_deterministic_);
   }
 
  private:
@@ -882,6 +882,7 @@ class LIBPROTOBUF_EXPORT CodedOutputStream {
   bool serialization_deterministic_is_overridden_;
   bool serialization_deterministic_override_;
   // Conceptually, default_serialization_deterministic_ is an atomic bool.
+  // TODO(haberman): replace with std::atomic<bool> when we move to C++11.
   static google::protobuf::internal::AtomicWord default_serialization_deterministic_;
 
   // Advance the buffer by a given number of bytes.
@@ -909,7 +910,7 @@ class LIBPROTOBUF_EXPORT CodedOutputStream {
   // thread has done so.
   friend void ::google::protobuf::internal::MapTestForceDeterministic();
   static void SetDefaultSerializationDeterministic() {
-    google::protobuf::internal::Release_Store(&default_serialization_deterministic_, 1);
+    google::protobuf::internal::NoBarrier_Store(&default_serialization_deterministic_, 1);
   }
 };
 

src/google/protobuf/lite_arena_unittest.cc

@@ -39,16 +39,16 @@ namespace {
 
 class LiteArenaTest : public testing::Test {
  protected:
-  // We create an Arena with a large initial block of memory, so that tests can
-  // verify that no new allocations are made.
-  LiteArenaTest() : arena_block_(128 * 1024) {
+  LiteArenaTest() {
     ArenaOptions options;
-    options.initial_block = &arena_block_[0];
-    options.initial_block_size = arena_block_.size();
+    options.start_block_size = 128 * 1024;
+    options.max_block_size = 128 * 1024;
     arena_.reset(new Arena(options));
+    // Trigger the allocation of the first arena block, so that further use of
+    // the arena will not require any heap allocations.
+    google::protobuf::Arena::CreateArray<char>(arena_.get(), 1);
   }
 
-  std::vector<char> arena_block_;
   google::protobuf::scoped_ptr<Arena> arena_;
 };
 

src/google/protobuf/map.h

@@ -142,6 +142,26 @@ class Map {
     insert(other.begin(), other.end());
   }
 
+#if LANG_CXX11
+  Map(Map&& other) noexcept : Map() {
+    if (other.arena_) {
+      *this = other;
+    } else {
+      swap(other);
+    }
+  }
+  Map& operator=(Map&& other) noexcept {
+    if (this != &other) {
+      if (arena_ != other.arena_) {
+        *this = other;
+      } else {
+        swap(other);
+      }
+    }
+    return *this;
+  }
+#endif
+
   template <class InputIt>
   Map(const InputIt& first, const InputIt& last)
       : arena_(NULL), default_enum_value_(0) {
@@ -1036,12 +1056,12 @@ class Map {
   }
   const T& at(const key_type& key) const {
     const_iterator it = find(key);
-    GOOGLE_CHECK(it != end());
+    GOOGLE_CHECK(it != end()) << "key not found: " << key;
     return it->second;
   }
   T& at(const key_type& key) {
     iterator it = find(key);
-    GOOGLE_CHECK(it != end());
+    GOOGLE_CHECK(it != end()) << "key not found: " << key;
     return it->second;
   }
 

src/google/protobuf/map_entry_lite.h

@@ -603,7 +603,7 @@ template <>
 struct FromHelper<WireFormatLite::TYPE_STRING> {
   static ArenaStringPtr From(const string& x) {
     ArenaStringPtr res;
-    res.UnsafeArenaSetAllocated(NULL, const_cast<string*>(&x), NULL);
+    *res.UnsafeRawStringPointer() = const_cast<string*>(&x);
     return res;
   }
 };
@@ -611,7 +611,7 @@ template <>
 struct FromHelper<WireFormatLite::TYPE_BYTES> {
   static ArenaStringPtr From(const string& x) {
     ArenaStringPtr res;
-    res.UnsafeArenaSetAllocated(NULL, const_cast<string*>(&x), NULL);
+    *res.UnsafeRawStringPointer() = const_cast<string*>(&x);
     return res;
   }
 };

src/google/protobuf/map_test.cc

@@ -3283,6 +3283,44 @@ TEST(ArenaTest, IsInitialized) {
   EXPECT_EQ(0, (*message->mutable_map_int32_int32())[0]);
 }
 
+#if LANG_CXX11
+TEST(MoveTest, MoveConstructorWorks) {
+  Map<int32, TestAllTypes> original_map;
+  original_map[42].mutable_optional_nested_message()->set_bb(42);
+  original_map[43].mutable_optional_nested_message()->set_bb(43);
+  const auto* nested_msg42_ptr = &original_map[42].optional_nested_message();
+  const auto* nested_msg43_ptr = &original_map[43].optional_nested_message();
+
+  Map<int32, TestAllTypes> moved_to_map(std::move(original_map));
+  EXPECT_TRUE(original_map.empty());
+  EXPECT_EQ(2, moved_to_map.size());
+  EXPECT_EQ(42, moved_to_map[42].optional_nested_message().bb());
+  EXPECT_EQ(43, moved_to_map[43].optional_nested_message().bb());
+  // This test takes advantage of the fact that pointers are swapped, so there
+  // should be pointer stability.
+  EXPECT_EQ(nested_msg42_ptr, &moved_to_map[42].optional_nested_message());
+  EXPECT_EQ(nested_msg43_ptr, &moved_to_map[43].optional_nested_message());
+}
+
+TEST(MoveTest, MoveAssignmentWorks) {
+  Map<int32, TestAllTypes> original_map;
+  original_map[42].mutable_optional_nested_message()->set_bb(42);
+  original_map[43].mutable_optional_nested_message()->set_bb(43);
+  const auto* nested_msg42_ptr = &original_map[42].optional_nested_message();
+  const auto* nested_msg43_ptr = &original_map[43].optional_nested_message();
+
+  Map<int32, TestAllTypes> moved_to_map = std::move(original_map);
+  EXPECT_TRUE(original_map.empty());
+  EXPECT_EQ(2, moved_to_map.size());
+  EXPECT_EQ(42, moved_to_map[42].optional_nested_message().bb());
+  EXPECT_EQ(43, moved_to_map[43].optional_nested_message().bb());
+  // This test takes advantage of the fact that pointers are swapped, so there
+  // should be pointer stability.
+  EXPECT_EQ(nested_msg42_ptr, &moved_to_map[42].optional_nested_message());
+  EXPECT_EQ(nested_msg43_ptr, &moved_to_map[43].optional_nested_message());
+}
+#endif
+
 }  // namespace internal
 }  // namespace protobuf
 }  // namespace google

src/google/protobuf/message_lite.cc

@@ -394,7 +394,8 @@ void GenericTypeHandler<string>::Merge(const string& from,
   *to = from;
 }
 
-bool proto3_preserve_unknown_ = false;
+bool proto3_preserve_unknown_ = true;
+
 void SetProto3PreserveUnknownsDefault(bool preserve) {
   proto3_preserve_unknown_ = preserve;
 }

src/google/protobuf/message_unittest.cc

@@ -409,6 +409,7 @@ TEST(MessageTest, MessageIsStillValidAfterParseFails) {
   }
 }
 
+
 namespace {
 
 void ExpectMessageMerged(const unittest::TestAllTypes& message) {

src/google/protobuf/metadata_lite.h

@@ -33,7 +33,7 @@
 
 #include <google/protobuf/stubs/common.h>
 #include <google/protobuf/arena.h>
-#include <google/protobuf/generated_message_util.h>
+#include <google/protobuf/message_lite.h>
 #include <google/protobuf/stubs/port.h>
 
 namespace google {

src/google/protobuf/repeated_field.h

@@ -376,7 +376,7 @@ struct TypeImplementsMergeBehaviorProbeForMergeFrom {
       CheckType<U, bool, &U::MergeFrom>*);
   template<typename U> static HasNoMerge Check(...);
 
-  // Resovles to either google::protobuf::internal::true_type or google::protobuf::internal::false_type.
+  // Resolves to either google::protobuf::internal::true_type or google::protobuf::internal::false_type.
   typedef google::protobuf::internal::integral_constant<bool,
                (sizeof(Check<T>(0)) == sizeof(HasMerge))> type;
 };
@@ -528,11 +528,9 @@ class LIBPROTOBUF_EXPORT RepeatedPtrFieldBase {
   inline void InternalSwap(RepeatedPtrFieldBase* other);
 
   template <typename TypeHandler>
-  void AddAllocatedInternal(typename TypeHandler::Type* value,
-                            google::protobuf::internal::true_type);
+  void AddAllocatedInternal(typename TypeHandler::Type* value, google::protobuf::internal::true_type);
   template <typename TypeHandler>
-  void AddAllocatedInternal(typename TypeHandler::Type* value,
-                            google::protobuf::internal::false_type);
+  void AddAllocatedInternal(typename TypeHandler::Type* value, google::protobuf::internal::false_type);
 
   template <typename TypeHandler> GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE
   void AddAllocatedSlowWithCopy(typename TypeHandler::Type* value,
@@ -1735,7 +1733,6 @@ void RepeatedPtrFieldBase::AddAllocatedInternal(
     elems[current_size_] = value;
     current_size_ = current_size_ + 1;
     rep_->allocated_size = rep_->allocated_size + 1;
-    return;
   } else {
     AddAllocatedSlowWithCopy<TypeHandler>(
         value, TypeHandler::GetArena(value), arena);
@@ -1782,7 +1779,6 @@ void RepeatedPtrFieldBase::AddAllocatedInternal(
     elems[current_size_] = value;
     current_size_ = current_size_ + 1;
     ++rep_->allocated_size;
-    return;
   } else {
     UnsafeArenaAddAllocated<TypeHandler>(value);
   }

src/google/protobuf/source_context.pb.cc

@@ -14,6 +14,10 @@
 #include <google/protobuf/generated_message_reflection.h>
 #include <google/protobuf/reflection_ops.h>
 #include <google/protobuf/wire_format.h>
+// This is a temporary google only hack
+#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
+#include "third_party/protobuf/version.h"
+#endif
 // @@protoc_insertion_point(includes)
 namespace google {
 namespace protobuf {
@@ -28,7 +32,11 @@ namespace protobuf_google_2fprotobuf_2fsource_5fcontext_2eproto {
 void InitDefaultsSourceContextImpl() {
   GOOGLE_PROTOBUF_VERIFY_VERSION;
 
+#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
+  ::google::protobuf::internal::InitProtobufDefaultsForceUnique();
+#else
   ::google::protobuf::internal::InitProtobufDefaults();
+#endif  // GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
   {
     void* ptr = &::google::protobuf::_SourceContext_default_instance_;
     new (ptr) ::google::protobuf::SourceContext();

src/google/protobuf/struct.pb.cc

@@ -14,6 +14,10 @@
 #include <google/protobuf/generated_message_reflection.h>
 #include <google/protobuf/reflection_ops.h>
 #include <google/protobuf/wire_format.h>
+// This is a temporary google only hack
+#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
+#include "third_party/protobuf/version.h"
+#endif
 // @@protoc_insertion_point(includes)
 namespace google {
 namespace protobuf {
@@ -49,7 +53,11 @@ namespace protobuf_google_2fprotobuf_2fstruct_2eproto {
 void InitDefaultsListValueImpl() {
   GOOGLE_PROTOBUF_VERIFY_VERSION;
 
+#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
+  ::google::protobuf::internal::InitProtobufDefaultsForceUnique();
+#else
   ::google::protobuf::internal::InitProtobufDefaults();
+#endif  // GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
   {
     void* ptr = &::google::protobuf::_Struct_FieldsEntry_DoNotUse_default_instance_;
     new (ptr) ::google::protobuf::Struct_FieldsEntry_DoNotUse();
@@ -266,12 +274,7 @@ Struct::~Struct() {
 }
 
 void Struct::SharedDtor() {
-  ::google::protobuf::Arena* arena = GetArenaNoVirtual();
-  GOOGLE_DCHECK(arena == NULL);
-  if (arena != NULL) {
-    return;
-  }
-
+  GOOGLE_DCHECK(GetArenaNoVirtual() == NULL);
 }
 
 void Struct::ArenaDtor(void* object) {
@@ -628,6 +631,36 @@ void Value::InitAsDefaultInstance() {
   ::google::protobuf::_Value_default_instance_.list_value_ = const_cast< ::google::protobuf::ListValue*>(
       ::google::protobuf::ListValue::internal_default_instance());
 }
+void Value::set_allocated_struct_value(::google::protobuf::Struct* struct_value) {
+  ::google::protobuf::Arena* message_arena = GetArenaNoVirtual();
+  clear_kind();
+  if (struct_value) {
+    ::google::protobuf::Arena* submessage_arena =
+      ::google::protobuf::Arena::GetArena(struct_value);
+    if (message_arena != submessage_arena) {
+      struct_value = ::google::protobuf::internal::GetOwnedMessage(
+          message_arena, struct_value, submessage_arena);
+    }
+    set_has_struct_value();
+    kind_.struct_value_ = struct_value;
+  }
+  // @@protoc_insertion_point(field_set_allocated:google.protobuf.Value.struct_value)
+}
+void Value::set_allocated_list_value(::google::protobuf::ListValue* list_value) {
+  ::google::protobuf::Arena* message_arena = GetArenaNoVirtual();
+  clear_kind();
+  if (list_value) {
+    ::google::protobuf::Arena* submessage_arena =
+      ::google::protobuf::Arena::GetArena(list_value);
+    if (message_arena != submessage_arena) {
+      list_value = ::google::protobuf::internal::GetOwnedMessage(
+          message_arena, list_value, submessage_arena);
+    }
+    set_has_list_value();
+    kind_.list_value_ = list_value;
+  }
+  // @@protoc_insertion_point(field_set_allocated:google.protobuf.Value.list_value)
+}
 #if !defined(_MSC_VER) || _MSC_VER >= 1900
 const int Value::kNullValueFieldNumber;
 const int Value::kNumberValueFieldNumber;
@@ -702,12 +735,7 @@ Value::~Value() {
 }
 
 void Value::SharedDtor() {
-  ::google::protobuf::Arena* arena = GetArenaNoVirtual();
-  GOOGLE_DCHECK(arena == NULL);
-  if (arena != NULL) {
-    return;
-  }
-
+  GOOGLE_DCHECK(GetArenaNoVirtual() == NULL);
   if (has_kind()) {
     clear_kind();
   }
@@ -863,7 +891,7 @@ bool Value::MergePartialFromCodedStream(
       case 5: {
         if (static_cast< ::google::protobuf::uint8>(tag) ==
             static_cast< ::google::protobuf::uint8>(42u /* 42 & 0xFF */)) {
-          DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual(
+          DO_(::google::protobuf::internal::WireFormatLite::ReadMessage(
                input, mutable_struct_value()));
         } else {
           goto handle_unusual;
@@ -875,7 +903,7 @@ bool Value::MergePartialFromCodedStream(
       case 6: {
         if (static_cast< ::google::protobuf::uint8>(tag) ==
             static_cast< ::google::protobuf::uint8>(50u /* 50 & 0xFF */)) {
-          DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual(
+          DO_(::google::protobuf::internal::WireFormatLite::ReadMessage(
                input, mutable_list_value()));
         } else {
           goto handle_unusual;
@@ -991,14 +1019,14 @@ void Value::SerializeWithCachedSizes(
   // .google.protobuf.Struct struct_value = 5;
   if (has_struct_value()) {
     target = ::google::protobuf::internal::WireFormatLite::
-      InternalWriteMessageNoVirtualToArray(
+      InternalWriteMessageToArray(
         5, *kind_.struct_value_, deterministic, target);
   }
 
   // .google.protobuf.ListValue list_value = 6;
   if (has_list_value()) {
     target = ::google::protobuf::internal::WireFormatLite::
-      InternalWriteMessageNoVirtualToArray(
+      InternalWriteMessageToArray(
         6, *kind_.list_value_, deterministic, target);
   }
 
@@ -1046,14 +1074,14 @@ size_t Value::ByteSizeLong() const {
     // .google.protobuf.Struct struct_value = 5;
     case kStructValue: {
       total_size += 1 +
-        ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual(
+        ::google::protobuf::internal::WireFormatLite::MessageSize(
           *kind_.struct_value_);
       break;
     }
     // .google.protobuf.ListValue list_value = 6;
     case kListValue: {
       total_size += 1 +
-        ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual(
+        ::google::protobuf::internal::WireFormatLite::MessageSize(
           *kind_.list_value_);
       break;
     }
@@ -1216,12 +1244,7 @@ ListValue::~ListValue() {
 }
 
 void ListValue::SharedDtor() {
-  ::google::protobuf::Arena* arena = GetArenaNoVirtual();
-  GOOGLE_DCHECK(arena == NULL);
-  if (arena != NULL) {
-    return;
-  }
-
+  GOOGLE_DCHECK(GetArenaNoVirtual() == NULL);
 }
 
 void ListValue::ArenaDtor(void* object) {
@@ -1273,8 +1296,7 @@ bool ListValue::MergePartialFromCodedStream(
       case 1: {
         if (static_cast< ::google::protobuf::uint8>(tag) ==
             static_cast< ::google::protobuf::uint8>(10u /* 10 & 0xFF */)) {
-          DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual(
-                input, add_values()));
+          DO_(::google::protobuf::internal::WireFormatLite::ReadMessage(input, add_values()));
         } else {
           goto handle_unusual;
         }
@@ -1332,7 +1354,7 @@ void ListValue::SerializeWithCachedSizes(
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->values_size()); i < n; i++) {
     target = ::google::protobuf::internal::WireFormatLite::
-      InternalWriteMessageNoVirtualToArray(
+      InternalWriteMessageToArray(
         1, this->values(static_cast<int>(i)), deterministic, target);
   }
 
@@ -1359,7 +1381,7 @@ size_t ListValue::ByteSizeLong() const {
     total_size += 1UL * count;
     for (unsigned int i = 0; i < count; i++) {
       total_size +=
-        ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual(
+        ::google::protobuf::internal::WireFormatLite::MessageSize(
           this->values(static_cast<int>(i)));
     }
   }

src/google/protobuf/struct.pb.h

@@ -375,7 +375,13 @@ class LIBPROTOBUF_EXPORT Value : public ::google::protobuf::Message /* @@protoc_
   ::std::string* mutable_string_value();
   ::std::string* release_string_value();
   void set_allocated_string_value(::std::string* string_value);
+  PROTOBUF_RUNTIME_DEPRECATED("The unsafe_arena_ accessors for"
+  "    string fields are deprecated and will be removed in a"
+  "    future release.")
   ::std::string* unsafe_arena_release_string_value();
+  PROTOBUF_RUNTIME_DEPRECATED("The unsafe_arena_ accessors for"
+  "    string fields are deprecated and will be removed in a"
+  "    future release.")
   void unsafe_arena_set_allocated_string_value(
       ::std::string* string_value);
 
@@ -394,17 +400,14 @@ class LIBPROTOBUF_EXPORT Value : public ::google::protobuf::Message /* @@protoc_
   static const int kStructValueFieldNumber = 5;
   private:
   void _slow_mutable_struct_value();
-  void _slow_set_allocated_struct_value(
-      ::google::protobuf::Arena* message_arena, ::google::protobuf::Struct** struct_value);
-  ::google::protobuf::Struct* _slow_release_struct_value();
   public:
   const ::google::protobuf::Struct& struct_value() const;
-  ::google::protobuf::Struct* mutable_struct_value();
   ::google::protobuf::Struct* release_struct_value();
+  ::google::protobuf::Struct* mutable_struct_value();
   void set_allocated_struct_value(::google::protobuf::Struct* struct_value);
-  ::google::protobuf::Struct* unsafe_arena_release_struct_value();
   void unsafe_arena_set_allocated_struct_value(
       ::google::protobuf::Struct* struct_value);
+  ::google::protobuf::Struct* unsafe_arena_release_struct_value();
 
   // .google.protobuf.ListValue list_value = 6;
   bool has_list_value() const;
@@ -412,17 +415,14 @@ class LIBPROTOBUF_EXPORT Value : public ::google::protobuf::Message /* @@protoc_
   static const int kListValueFieldNumber = 6;
   private:
   void _slow_mutable_list_value();
-  void _slow_set_allocated_list_value(
-      ::google::protobuf::Arena* message_arena, ::google::protobuf::ListValue** list_value);
-  ::google::protobuf::ListValue* _slow_release_list_value();
   public:
   const ::google::protobuf::ListValue& list_value() const;
-  ::google::protobuf::ListValue* mutable_list_value();
   ::google::protobuf::ListValue* release_list_value();
+  ::google::protobuf::ListValue* mutable_list_value();
   void set_allocated_list_value(::google::protobuf::ListValue* list_value);
-  ::google::protobuf::ListValue* unsafe_arena_release_list_value();
   void unsafe_arena_set_allocated_list_value(
       ::google::protobuf::ListValue* list_value);
+  ::google::protobuf::ListValue* unsafe_arena_release_list_value();
 
   KindCase kind_case() const;
   // @@protoc_insertion_point(class_scope:google.protobuf.Value)
@@ -758,17 +758,6 @@ inline ::std::string* Value::release_string_value() {
     return NULL;
   }
 }
-inline ::std::string* Value::unsafe_arena_release_string_value() {
-  // @@protoc_insertion_point(field_unsafe_arena_release:google.protobuf.Value.string_value)
-  GOOGLE_DCHECK(GetArenaNoVirtual() != NULL);
-  if (has_string_value()) {
-    clear_has_kind();
-    return kind_.string_value_.UnsafeArenaRelease(
-        &::google::protobuf::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual());
-  } else {
-    return NULL;
-  }
-}
 inline void Value::set_allocated_string_value(::std::string* string_value) {
   if (!has_string_value()) {
     kind_.string_value_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
@@ -781,6 +770,17 @@ inline void Value::set_allocated_string_value(::std::string* string_value) {
   }
   // @@protoc_insertion_point(field_set_allocated:google.protobuf.Value.string_value)
 }
+inline ::std::string* Value::unsafe_arena_release_string_value() {
+  // @@protoc_insertion_point(field_unsafe_arena_release:google.protobuf.Value.string_value)
+  GOOGLE_DCHECK(GetArenaNoVirtual() != NULL);
+  if (has_string_value()) {
+    clear_has_kind();
+    return kind_.string_value_.UnsafeArenaRelease(
+        &::google::protobuf::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual());
+  } else {
+    return NULL;
+  }
+}
 inline void Value::unsafe_arena_set_allocated_string_value(::std::string* string_value) {
   GOOGLE_DCHECK(GetArenaNoVirtual() != NULL);
   if (!has_string_value()) {
@@ -838,58 +838,25 @@ inline void Value::clear_struct_value() {
     clear_has_kind();
   }
 }
-inline  const ::google::protobuf::Struct& Value::struct_value() const {
-  // @@protoc_insertion_point(field_get:google.protobuf.Value.struct_value)
-  return has_struct_value()
-      ? *kind_.struct_value_
-      : ::google::protobuf::Struct::default_instance();
-}
-inline ::google::protobuf::Struct* Value::mutable_struct_value() {
-  if (!has_struct_value()) {
-    clear_kind();
-    set_has_struct_value();
-    kind_.struct_value_ = 
-      ::google::protobuf::Arena::CreateMessage< ::google::protobuf::Struct >(
-      GetArenaNoVirtual());
-  }
-  // @@protoc_insertion_point(field_mutable:google.protobuf.Value.struct_value)
-  return kind_.struct_value_;
-}
 inline ::google::protobuf::Struct* Value::release_struct_value() {
   // @@protoc_insertion_point(field_release:google.protobuf.Value.struct_value)
   if (has_struct_value()) {
     clear_has_kind();
-    if (GetArenaNoVirtual() != NULL) {
-      ::google::protobuf::Struct* temp = new ::google::protobuf::Struct(*kind_.struct_value_);
-      kind_.struct_value_ = NULL;
-      return temp;
-    } else {
       ::google::protobuf::Struct* temp = kind_.struct_value_;
-      kind_.struct_value_ = NULL;
-      return temp;
+    if (GetArenaNoVirtual() != NULL) {
+      temp = ::google::protobuf::internal::DuplicateIfNonNull(temp, NULL);
     }
+    kind_.struct_value_ = NULL;
+    return temp;
   } else {
     return NULL;
   }
 }
-inline void Value::set_allocated_struct_value(::google::protobuf::Struct* struct_value) {
-  clear_kind();
-  if (struct_value) {
-    if (GetArenaNoVirtual() != NULL &&
-        ::google::protobuf::Arena::GetArena(struct_value) == NULL) {
-      GetArenaNoVirtual()->Own(struct_value);
-    } else if (GetArenaNoVirtual() !=
-               ::google::protobuf::Arena::GetArena(struct_value)) {
-      ::google::protobuf::Struct* new_struct_value = 
-          ::google::protobuf::Arena::CreateMessage< ::google::protobuf::Struct >(
-          GetArenaNoVirtual());
-      new_struct_value->CopyFrom(*struct_value);
-      struct_value = new_struct_value;
-    }
-    set_has_struct_value();
-    kind_.struct_value_ = struct_value;
-  }
-  // @@protoc_insertion_point(field_set_allocated:google.protobuf.Value.struct_value)
+inline const ::google::protobuf::Struct& Value::struct_value() const {
+  // @@protoc_insertion_point(field_get:google.protobuf.Value.struct_value)
+  return has_struct_value()
+      ? *kind_.struct_value_
+      : *reinterpret_cast< ::google::protobuf::Struct*>(&::google::protobuf::_Struct_default_instance_);
 }
 inline ::google::protobuf::Struct* Value::unsafe_arena_release_struct_value() {
   // @@protoc_insertion_point(field_unsafe_arena_release:google.protobuf.Value.struct_value)
@@ -902,7 +869,7 @@ inline ::google::protobuf::Struct* Value::unsafe_arena_release_struct_value() {
     return NULL;
   }
 }
-inline  void Value::unsafe_arena_set_allocated_struct_value(::google::protobuf::Struct* struct_value) {
+inline void Value::unsafe_arena_set_allocated_struct_value(::google::protobuf::Struct* struct_value) {
   clear_kind();
   if (struct_value) {
     set_has_struct_value();
@@ -910,6 +877,17 @@ inline  void Value::unsafe_arena_set_allocated_struct_value(::google::protobuf::
   }
   // @@protoc_insertion_point(field_unsafe_arena_set_allocated:google.protobuf.Value.struct_value)
 }
+inline ::google::protobuf::Struct* Value::mutable_struct_value() {
+  if (!has_struct_value()) {
+    clear_kind();
+    set_has_struct_value();
+    kind_.struct_value_ = 
+      ::google::protobuf::Arena::CreateMessage< ::google::protobuf::Struct >(
+      GetArenaNoVirtual());
+  }
+  // @@protoc_insertion_point(field_mutable:google.protobuf.Value.struct_value)
+  return kind_.struct_value_;
+}
 
 // .google.protobuf.ListValue list_value = 6;
 inline bool Value::has_list_value() const {
@@ -926,58 +904,25 @@ inline void Value::clear_list_value() {
     clear_has_kind();
   }
 }
-inline  const ::google::protobuf::ListValue& Value::list_value() const {
-  // @@protoc_insertion_point(field_get:google.protobuf.Value.list_value)
-  return has_list_value()
-      ? *kind_.list_value_
-      : ::google::protobuf::ListValue::default_instance();
-}
-inline ::google::protobuf::ListValue* Value::mutable_list_value() {
-  if (!has_list_value()) {
-    clear_kind();
-    set_has_list_value();
-    kind_.list_value_ = 
-      ::google::protobuf::Arena::CreateMessage< ::google::protobuf::ListValue >(
-      GetArenaNoVirtual());
-  }
-  // @@protoc_insertion_point(field_mutable:google.protobuf.Value.list_value)
-  return kind_.list_value_;
-}
 inline ::google::protobuf::ListValue* Value::release_list_value() {
   // @@protoc_insertion_point(field_release:google.protobuf.Value.list_value)
   if (has_list_value()) {
     clear_has_kind();
-    if (GetArenaNoVirtual() != NULL) {
-      ::google::protobuf::ListValue* temp = new ::google::protobuf::ListValue(*kind_.list_value_);
-      kind_.list_value_ = NULL;
-      return temp;
-    } else {
       ::google::protobuf::ListValue* temp = kind_.list_value_;
-      kind_.list_value_ = NULL;
-      return temp;
+    if (GetArenaNoVirtual() != NULL) {
+      temp = ::google::protobuf::internal::DuplicateIfNonNull(temp, NULL);
     }
+    kind_.list_value_ = NULL;
+    return temp;
   } else {
     return NULL;
   }
 }
-inline void Value::set_allocated_list_value(::google::protobuf::ListValue* list_value) {
-  clear_kind();
-  if (list_value) {
-    if (GetArenaNoVirtual() != NULL &&
-        ::google::protobuf::Arena::GetArena(list_value) == NULL) {
-      GetArenaNoVirtual()->Own(list_value);
-    } else if (GetArenaNoVirtual() !=
-               ::google::protobuf::Arena::GetArena(list_value)) {
-      ::google::protobuf::ListValue* new_list_value = 
-          ::google::protobuf::Arena::CreateMessage< ::google::protobuf::ListValue >(
-          GetArenaNoVirtual());
-      new_list_value->CopyFrom(*list_value);
-      list_value = new_list_value;
-    }
-    set_has_list_value();
-    kind_.list_value_ = list_value;
-  }
-  // @@protoc_insertion_point(field_set_allocated:google.protobuf.Value.list_value)
+inline const ::google::protobuf::ListValue& Value::list_value() const {
+  // @@protoc_insertion_point(field_get:google.protobuf.Value.list_value)
+  return has_list_value()
+      ? *kind_.list_value_
+      : *reinterpret_cast< ::google::protobuf::ListValue*>(&::google::protobuf::_ListValue_default_instance_);
 }
 inline ::google::protobuf::ListValue* Value::unsafe_arena_release_list_value() {
   // @@protoc_insertion_point(field_unsafe_arena_release:google.protobuf.Value.list_value)
@@ -990,7 +935,7 @@ inline ::google::protobuf::ListValue* Value::unsafe_arena_release_list_value() {
     return NULL;
   }
 }
-inline  void Value::unsafe_arena_set_allocated_list_value(::google::protobuf::ListValue* list_value) {
+inline void Value::unsafe_arena_set_allocated_list_value(::google::protobuf::ListValue* list_value) {
   clear_kind();
   if (list_value) {
     set_has_list_value();
@@ -998,6 +943,17 @@ inline  void Value::unsafe_arena_set_allocated_list_value(::google::protobuf::Li
   }
   // @@protoc_insertion_point(field_unsafe_arena_set_allocated:google.protobuf.Value.list_value)
 }
+inline ::google::protobuf::ListValue* Value::mutable_list_value() {
+  if (!has_list_value()) {
+    clear_kind();
+    set_has_list_value();
+    kind_.list_value_ = 
+      ::google::protobuf::Arena::CreateMessage< ::google::protobuf::ListValue >(
+      GetArenaNoVirtual());
+  }
+  // @@protoc_insertion_point(field_mutable:google.protobuf.Value.list_value)
+  return kind_.list_value_;
+}
 
 inline bool Value::has_kind() const {
   return kind_case() != KIND_NOT_SET;

src/google/protobuf/stubs/io_win32_unittest.cc

@@ -123,8 +123,7 @@ void IoWin32Test::SetUp() {
       // "\\?\" prefix (except on Windows 10 version 1607 and beyond, after
       // opting in to long paths by default [1]).
       //
-      // [1] https://msdn.microsoft.com/en-us/library/windows/ \
-      //   desktop/aa365247(v=vs.85).aspx#maxpath
+      // [1] https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#maxpath
       DWORD result = ::GetCurrentDirectoryA(MAX_PATH, buffer);
       if (result > 0) {
         test_tmpdir = string(buffer);
@@ -142,8 +141,7 @@ void IoWin32Test::SetUp() {
   test_tmpdir += "\\io_win32_unittest.tmp";
 
   // CreateDirectoryA's limit is 248 chars, see MSDN.
-  // https://msdn.microsoft.com/en-us/library/windows/ \
-  //   desktop/aa363855(v=vs.85).aspx
+  // https://msdn.microsoft.com/en-us/library/windows/desktop/aa363855(v=vs.85).aspx
   wtest_tmpdir = testonly_path_to_winpath(test_tmpdir);
   if (!DeleteAllUnder(wtest_tmpdir) || !CreateAllUnder(wtest_tmpdir)) {
     GOOGLE_CHECK_OK(false);

src/google/protobuf/text_format.h

@@ -211,7 +211,7 @@ class LIBPROTOBUF_EXPORT TextFormat {
       single_line_mode_ = single_line_mode;
     }
 
-    bool IsInSingleLineMode() {
+    bool IsInSingleLineMode() const {
       return single_line_mode_;
     }
 

src/google/protobuf/timestamp.pb.cc

@@ -14,6 +14,10 @@
 #include <google/protobuf/generated_message_reflection.h>
 #include <google/protobuf/reflection_ops.h>
 #include <google/protobuf/wire_format.h>
+// This is a temporary google only hack
+#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
+#include "third_party/protobuf/version.h"
+#endif
 // @@protoc_insertion_point(includes)
 namespace google {
 namespace protobuf {
@@ -28,7 +32,11 @@ namespace protobuf_google_2fprotobuf_2ftimestamp_2eproto {
 void InitDefaultsTimestampImpl() {
   GOOGLE_PROTOBUF_VERIFY_VERSION;
 
+#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
+  ::google::protobuf::internal::InitProtobufDefaultsForceUnique();
+#else
   ::google::protobuf::internal::InitProtobufDefaults();
+#endif  // GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
   {
     void* ptr = &::google::protobuf::_Timestamp_default_instance_;
     new (ptr) ::google::protobuf::Timestamp();
@@ -159,12 +167,7 @@ Timestamp::~Timestamp() {
 }
 
 void Timestamp::SharedDtor() {
-  ::google::protobuf::Arena* arena = GetArenaNoVirtual();
-  GOOGLE_DCHECK(arena == NULL);
-  if (arena != NULL) {
-    return;
-  }
-
+  GOOGLE_DCHECK(GetArenaNoVirtual() == NULL);
 }
 
 void Timestamp::ArenaDtor(void* object) {