down-integrate internal changes

.gitignore

@@ -44,6 +44,7 @@ src/.libs
 
 .dirstamp
 
+any_test.pb.*
 map*unittest.pb.*
 unittest*.pb.*
 cpp_test*.pb.*

Android.mk

@@ -90,14 +90,20 @@ COMPILER_SRC_FILES :=  \
     src/google/protobuf/compiler/cpp/cpp_string_field.cc \
     src/google/protobuf/compiler/java/java_enum.cc \
     src/google/protobuf/compiler/java/java_enum_field.cc \
+    src/google/protobuf/compiler/java/java_enum_field_lite.cc \
     src/google/protobuf/compiler/java/java_extension.cc \
     src/google/protobuf/compiler/java/java_field.cc \
     src/google/protobuf/compiler/java/java_file.cc \
     src/google/protobuf/compiler/java/java_generator.cc \
     src/google/protobuf/compiler/java/java_helpers.cc \
     src/google/protobuf/compiler/java/java_message.cc \
+    src/google/protobuf/compiler/java/java_message_lite.cc \
+    src/google/protobuf/compiler/java/java_message_builder.cc \
+    src/google/protobuf/compiler/java/java_message_builder_lite.cc \
     src/google/protobuf/compiler/java/java_message_field.cc \
+    src/google/protobuf/compiler/java/java_message_field_lite.cc \
     src/google/protobuf/compiler/java/java_primitive_field.cc \
+    src/google/protobuf/compiler/java/java_primitive_field_lite.cc \
     src/google/protobuf/compiler/java/java_service.cc \
     src/google/protobuf/compiler/javamicro/javamicro_enum.cc \
     src/google/protobuf/compiler/javamicro/javamicro_enum_field.cc \

Makefile.am

@@ -43,28 +43,34 @@ java_EXTRA_DIST=                                                             \
   java/src/main/java/com/google/protobuf/AbstractMessage.java                \
   java/src/main/java/com/google/protobuf/AbstractMessageLite.java            \
   java/src/main/java/com/google/protobuf/AbstractParser.java                 \
+  java/src/main/java/com/google/protobuf/AbstractProtobufList.java           \
   java/src/main/java/com/google/protobuf/BlockingRpcChannel.java             \
   java/src/main/java/com/google/protobuf/BlockingService.java                \
   java/src/main/java/com/google/protobuf/BoundedByteString.java              \
+  java/src/main/java/com/google/protobuf/BooleanArrayList.java               \
   java/src/main/java/com/google/protobuf/ByteString.java                     \
   java/src/main/java/com/google/protobuf/CodedInputStream.java               \
   java/src/main/java/com/google/protobuf/CodedOutputStream.java              \
   java/src/main/java/com/google/protobuf/Descriptors.java                    \
+  java/src/main/java/com/google/protobuf/DoubleArrayList.java                \
   java/src/main/java/com/google/protobuf/DynamicMessage.java                 \
   java/src/main/java/com/google/protobuf/Extension.java                      \
   java/src/main/java/com/google/protobuf/ExtensionLite.java                  \
   java/src/main/java/com/google/protobuf/ExtensionRegistry.java              \
   java/src/main/java/com/google/protobuf/ExtensionRegistryLite.java          \
   java/src/main/java/com/google/protobuf/FieldSet.java                       \
+  java/src/main/java/com/google/protobuf/FloatArrayList.java                 \
   java/src/main/java/com/google/protobuf/GeneratedMessage.java               \
   java/src/main/java/com/google/protobuf/GeneratedMessageLite.java           \
   java/src/main/java/com/google/protobuf/Internal.java                       \
+  java/src/main/java/com/google/protobuf/IntArrayList.java                   \
   java/src/main/java/com/google/protobuf/InvalidProtocolBufferException.java \
   java/src/main/java/com/google/protobuf/LazyField.java                      \
   java/src/main/java/com/google/protobuf/LazyFieldLite.java                  \
   java/src/main/java/com/google/protobuf/LazyStringArrayList.java            \
   java/src/main/java/com/google/protobuf/LazyStringList.java                 \
   java/src/main/java/com/google/protobuf/LiteralByteString.java              \
+  java/src/main/java/com/google/protobuf/LongArrayList.java                  \
   java/src/main/java/com/google/protobuf/MapEntry.java                       \
   java/src/main/java/com/google/protobuf/MapEntryLite.java                   \
   java/src/main/java/com/google/protobuf/MapField.java                       \
@@ -74,7 +80,9 @@ java_EXTRA_DIST=                                                             \
   java/src/main/java/com/google/protobuf/MessageLiteOrBuilder.java           \
   java/src/main/java/com/google/protobuf/MessageOrBuilder.java               \
   java/src/main/java/com/google/protobuf/MessageReflection.java              \
+  java/src/main/java/com/google/protobuf/MutabilityOracle.java               \
   java/src/main/java/com/google/protobuf/Parser.java                         \
+  java/src/main/java/com/google/protobuf/ProtobufArrayList.java              \
   java/src/main/java/com/google/protobuf/ProtocolMessageEnum.java            \
   java/src/main/java/com/google/protobuf/ProtocolStringList.java             \
   java/src/main/java/com/google/protobuf/RepeatedFieldBuilder.java           \
@@ -96,16 +104,20 @@ java_EXTRA_DIST=                                                             \
   java/src/main/java/com/google/protobuf/WireFormat.java                     \
   java/src/test/java/com/google/protobuf/AbstractMessageTest.java            \
   java/src/test/java/com/google/protobuf/BoundedByteStringTest.java          \
+  java/src/test/java/com/google/protobuf/BooleanArrayListTest.java           \
   java/src/test/java/com/google/protobuf/ByteStringTest.java                 \
   java/src/test/java/com/google/protobuf/CheckUtf8Test.java                  \
   java/src/test/java/com/google/protobuf/CodedInputStreamTest.java           \
   java/src/test/java/com/google/protobuf/CodedOutputStreamTest.java          \
   java/src/test/java/com/google/protobuf/DeprecatedFieldTest.java            \
   java/src/test/java/com/google/protobuf/DescriptorsTest.java                \
+  java/src/test/java/com/google/protobuf/DoubleArrayListTest.java            \
   java/src/test/java/com/google/protobuf/DynamicMessageTest.java             \
   java/src/test/java/com/google/protobuf/FieldPresenceTest.java              \
+  java/src/test/java/com/google/protobuf/FloatArrayListTest.java             \
   java/src/test/java/com/google/protobuf/ForceFieldBuildersPreRun.java       \
   java/src/test/java/com/google/protobuf/GeneratedMessageTest.java           \
+  java/src/test/java/com/google/protobuf/IntArrayListTest.java               \
   java/src/test/java/com/google/protobuf/IsValidUtf8Test.java                \
   java/src/test/java/com/google/protobuf/IsValidUtf8TestUtil.java            \
   java/src/test/java/com/google/protobuf/LazyFieldLiteTest.java              \
@@ -116,12 +128,14 @@ java_EXTRA_DIST=                                                             \
   java/src/test/java/com/google/protobuf/LiteEqualsAndHashTest.java          \
   java/src/test/java/com/google/protobuf/LiteralByteStringTest.java          \
   java/src/test/java/com/google/protobuf/LiteTest.java                       \
+  java/src/test/java/com/google/protobuf/LongArrayListTest.java              \
   java/src/test/java/com/google/protobuf/MapForProto2LiteTest.java           \
   java/src/test/java/com/google/protobuf/MapForProto2Test.java               \
   java/src/test/java/com/google/protobuf/MapTest.java                        \
   java/src/test/java/com/google/protobuf/MessageTest.java                    \
   java/src/test/java/com/google/protobuf/NestedBuildersTest.java             \
   java/src/test/java/com/google/protobuf/ParserTest.java                     \
+  java/src/test/java/com/google/protobuf/ProtobufArrayListTest.java          \
   java/src/test/java/com/google/protobuf/RepeatedFieldBuilderTest.java       \
   java/src/test/java/com/google/protobuf/RopeByteStringSubstringTest.java    \
   java/src/test/java/com/google/protobuf/RopeByteStringTest.java             \

conformance/conformance.proto

@@ -71,7 +71,7 @@ message ConformanceRequest {
   }
 
   // Which format should the testee serialize its message to?
-  optional RequestedOutput requested_output = 3;
+  RequestedOutput requested_output = 3;
 }
 
 // Represents a single test case's output.
@@ -103,8 +103,8 @@ message ConformanceResponse {
 // forms.
 message TestAllTypes {
   message NestedMessage {
-    optional int32 a = 1;
-    optional TestAllTypes corecursive = 2;
+    int32 a = 1;
+    TestAllTypes corecursive = 2;
   }
 
   enum NestedEnum {
@@ -115,36 +115,32 @@ message TestAllTypes {
   }
 
   // Singular
-  optional    int32 optional_int32    =  1;
-  optional    int64 optional_int64    =  2;
-  optional   uint32 optional_uint32   =  3;
-  optional   uint64 optional_uint64   =  4;
-  optional   sint32 optional_sint32   =  5;
-  optional   sint64 optional_sint64   =  6;
-  optional  fixed32 optional_fixed32  =  7;
-  optional  fixed64 optional_fixed64  =  8;
-  optional sfixed32 optional_sfixed32 =  9;
-  optional sfixed64 optional_sfixed64 = 10;
-  optional    float optional_float    = 11;
-  optional   double optional_double   = 12;
-  optional     bool optional_bool     = 13;
-  optional   string optional_string   = 14;
-  optional    bytes optional_bytes    = 15;
-
-  optional group OptionalGroup = 16 {
-    optional int32 a = 17;
-  }
-
-  optional NestedMessage                        optional_nested_message  = 18;
-  optional ForeignMessage                       optional_foreign_message = 19;
-
-  optional NestedEnum                           optional_nested_enum     = 21;
-  optional ForeignEnum                          optional_foreign_enum    = 22;
-
-  optional string optional_string_piece = 24 [ctype=STRING_PIECE];
-  optional string optional_cord = 25 [ctype=CORD];
-
-  optional TestAllTypes recursive_message = 27;
+  int32 optional_int32    =  1;
+  int64 optional_int64    =  2;
+  uint32 optional_uint32   =  3;
+  uint64 optional_uint64   =  4;
+  sint32 optional_sint32   =  5;
+  sint64 optional_sint64   =  6;
+  fixed32 optional_fixed32  =  7;
+  fixed64 optional_fixed64  =  8;
+  sfixed32 optional_sfixed32 =  9;
+  sfixed64 optional_sfixed64 = 10;
+  float optional_float    = 11;
+  double optional_double   = 12;
+  bool optional_bool     = 13;
+  string optional_string   = 14;
+  bytes optional_bytes    = 15;
+
+  NestedMessage                        optional_nested_message  = 18;
+  ForeignMessage                       optional_foreign_message = 19;
+
+  NestedEnum                           optional_nested_enum     = 21;
+  ForeignEnum                          optional_foreign_enum    = 22;
+
+  string optional_string_piece = 24 [ctype=STRING_PIECE];
+  string optional_cord = 25 [ctype=CORD];
+
+  TestAllTypes recursive_message = 27;
 
   // Repeated
   repeated    int32 repeated_int32    = 31;
@@ -163,10 +159,6 @@ message TestAllTypes {
   repeated   string repeated_string   = 44;
   repeated    bytes repeated_bytes    = 45;
 
-  repeated group RepeatedGroup = 46 {
-    optional int32 a = 47;
-  }
-
   repeated NestedMessage                        repeated_nested_message  = 48;
   repeated ForeignMessage                       repeated_foreign_message = 49;
 
@@ -206,7 +198,7 @@ message TestAllTypes {
 }
 
 message ForeignMessage {
-  optional int32 c = 1;
+  int32 c = 1;
 }
 
 enum ForeignEnum {

conformance/conformance_test.cc

@@ -295,6 +295,7 @@ void ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner,
   failures_ = 0;
 
   for (int i = 1; i <= FieldDescriptor::MAX_TYPE; i++) {
+    if (i == FieldDescriptor::TYPE_GROUP) continue;
     TestPrematureEOFForType(static_cast<WireFormatLite::FieldType>(i));
   }
 

editors/proto.vim

@@ -57,7 +57,7 @@ syn keyword pbSyntax     syntax import option
 syn keyword pbStructure  package message group oneof
 syn keyword pbRepeat     optional required repeated
 syn keyword pbDefault    default
-syn keyword pbExtend     extend extensions to max
+syn keyword pbExtend     extend extensions to max reserved
 syn keyword pbRPC        service rpc returns
 
 syn keyword pbType      int32 int64 uint32 uint64 sint32 sint64

editors/protobuf-mode.el

@@ -106,7 +106,7 @@
 ;; cc-mode.  So, we approximate as best we can.
 
 (c-lang-defconst c-type-list-kwds
-  protobuf '("extensions" "to"))
+  protobuf '("extensions" "to" "reserved"))
 
 (c-lang-defconst c-typeless-decl-kwds
   protobuf '("extend" "rpc" "option" "returns"))

generate_descriptor_proto.sh

@@ -43,8 +43,11 @@ declare -a RUNTIME_PROTO_FILES=(\
   google/protobuf/wrappers.proto)
 
 CORE_PROTO_IS_CORRECT=0
+PROCESS_ROUND=1
+echo "Updating descriptor protos..."
 while [ $CORE_PROTO_IS_CORRECT -ne 1 ]
 do
+  echo "Round $PROCESS_ROUND"
   CORE_PROTO_IS_CORRECT=1
   for PROTO_FILE in ${RUNTIME_PROTO_FILES[@]}; do
     BASE_NAME=${PROTO_FILE%.*}
@@ -86,5 +89,7 @@ do
   done
   rm google/protobuf/compiler/plugin.pb.h.tmp
   rm google/protobuf/compiler/plugin.pb.cc.tmp
+
+  PROCESS_ROUND=$((PROCESS_ROUND + 1))
 done
 cd ..

java/pom.xml

@@ -134,6 +134,7 @@
                   <arg value="src/test/java/com/google/protobuf/map_for_proto2_lite_test.proto" />
                   <arg value="src/test/java/com/google/protobuf/map_for_proto2_test.proto" />
                   <arg value="src/test/java/com/google/protobuf/map_test.proto" />
+                  <arg value="src/test/java/com/google/protobuf/map_initialization_order_test.proto" />
                 </exec>
               </tasks>
               <testSourceRoot>target/generated-test-sources</testSourceRoot>
@@ -227,25 +228,33 @@
               <includes>
                 <include>**/AbstractMessageLite.java</include>
                 <include>**/AbstractParser.java</include>
+                <include>**/AbstractProtobufList.java</include>
                 <include>**/BoundedByteString.java</include>
+                <include>**/BooleanArrayList.java</include>
                 <include>**/ByteString.java</include>
                 <include>**/CodedInputStream.java</include>
                 <include>**/CodedOutputStream.java</include>
+                <include>**/DoublerrayList.java</include>
                 <include>**/ExtensionLite.java</include>
                 <include>**/ExtensionRegistryLite.java</include>
                 <include>**/FieldSet.java</include>
+                <include>**/FloatArrayList.java</include>
                 <include>**/GeneratedMessageLite.java</include>
+                <include>**/IntArrayList.java</include>
                 <include>**/Internal.java</include>
                 <include>**/InvalidProtocolBufferException.java</include>
                 <include>**/LazyFieldLite.java</include>
                 <include>**/LazyStringArrayList.java</include>
                 <include>**/LazyStringList.java</include>
                 <include>**/LiteralByteString.java</include>
+                <include>**/LongArrayList.java</include>
                 <include>**/MapEntryLite.java</include>
                 <include>**/MapFieldLite.java</include>
                 <include>**/MessageLite.java</include>
                 <include>**/MessageLiteOrBuilder.java</include>
+                <include>**/MutabilityOracle.java</include>
                 <include>**/Parser.java</include>
+                <include>**/ProtobufArrayList.java</include>
                 <include>**/ProtocolStringList.java</include>
                 <include>**/RopeByteString.java</include>
                 <include>**/SmallSortedMap.java</include>
@@ -257,8 +266,14 @@
               </includes>
               <testIncludes>
                 <testInclude>**/*Lite.java</testInclude>
+                <testInclude>**/BooleanArrayListTest.java</testInclude>
+                <testInclude>**/DoubleArrayListTest.java</testInclude>
+                <testInclude>**/FloatArrayListTest.java</testInclude>
+                <testInclude>**/IntArrayListTest.java</testInclude>
                 <testInclude>**/LazyMessageLiteTest.java</testInclude>
                 <testInclude>**/LiteTest.java</testInclude>
+                <testInclude>**/LongArrayListTest.java</testInclude>
+                <testInclude>**/ProtobufArrayListTest.java</testInclude>
                 <testInclude>**/UnknownFieldSetLiteTest.java</testInclude>
               </testIncludes>
             </configuration>

java/src/main/java/com/google/protobuf/AbstractMessageLite.java

@@ -31,8 +31,8 @@
 package com.google.protobuf;
 
 import java.io.FilterInputStream;
-import java.io.InputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.OutputStream;
 import java.util.Collection;
 
@@ -109,6 +109,11 @@ public abstract class AbstractMessageLite implements MessageLite {
     }
   }
 
+  protected static <T> void addAll(final Iterable<T> values,
+      final Collection<? super T> list) {
+    Builder.addAll(values, list);
+  }
+  
   /**
    * A partial implementation of the {@link Message.Builder} interface which
    * implements as many methods of that interface as possible in terms of
@@ -320,12 +325,15 @@ public abstract class AbstractMessageLite implements MessageLite {
      * Adds the {@code values} to the {@code list}.  This is a helper method
      * used by generated code.  Users should ignore it.
      *
-     * @throws NullPointerException if any of the elements of {@code values} is
-     * null. When that happens, some elements of {@code values} may have already
-     * been added to the result {@code list}.
+     * @throws NullPointerException if {@code values} or any of the elements of
+     * {@code values} is null. When that happens, some elements of
+     * {@code values} may have already been added to the result {@code list}.
      */
     protected static <T> void addAll(final Iterable<T> values,
                                      final Collection<? super T> list) {
+      if (values == null) {
+        throw new NullPointerException();
+      }
       if (values instanceof LazyStringList) {
         // For StringOrByteStringLists, check the underlying elements to avoid
         // forcing conversions of ByteStrings to Strings.

java/src/main/java/com/google/protobuf/AbstractProtobufList.java

@@ -0,0 +1,136 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package com.google.protobuf;
+
+import com.google.protobuf.Internal.ProtobufList;
+
+import java.util.AbstractList;
+import java.util.Collection;
+
+/**
+ * An abstract implementation of {@link ProtobufList} which manages mutability semantics. All mutate
+ * methods are check if the list is mutable before proceeding. Subclasses must invoke
+ * {@link #ensureIsMutable()} manually when overriding those methods.
+ */
+abstract class AbstractProtobufList<E> extends AbstractList<E> implements ProtobufList<E> {
+
+  /**
+   * Whether or not this list is modifiable.
+   */
+  private boolean isMutable;
+  
+  /**
+   * Constructs a mutable list by default.
+   */
+  AbstractProtobufList() {
+    isMutable = true;
+  }
+
+  @Override
+  public boolean add(E e) {
+    ensureIsMutable();
+    return super.add(e);
+  }
+
+  @Override
+  public void add(int index, E element) {
+    ensureIsMutable();
+    super.add(index, element);
+  }
+
+  @Override
+  public boolean addAll(Collection<? extends E> c) {
+    ensureIsMutable();
+    return super.addAll(c);
+  }
+  
+  @Override
+  public boolean addAll(int index, Collection<? extends E> c) {
+    ensureIsMutable();
+    return super.addAll(index, c);
+  }
+
+  @Override
+  public void clear() {
+    ensureIsMutable();
+    super.clear();
+  }
+  
+  @Override
+  public boolean isModifiable() {
+    return isMutable;
+  }
+  
+  @Override
+  public final void makeImmutable() {
+    isMutable = false;
+  }
+  
+  @Override
+  public E remove(int index) {
+    ensureIsMutable();
+    return super.remove(index);
+  }
+  
+  @Override
+  public boolean remove(Object o) {
+    ensureIsMutable();
+    return super.remove(o);
+  }
+  
+  @Override
+  public boolean removeAll(Collection<?> c) {
+    ensureIsMutable();
+    return super.removeAll(c);
+  }
+  
+  @Override
+  public boolean retainAll(Collection<?> c) {
+    ensureIsMutable();
+    return super.retainAll(c);
+  }
+  
+  @Override
+  public E set(int index, E element) {
+    ensureIsMutable();
+    return super.set(index, element);
+  }
+  
+  /**
+   * Throws an {@link UnsupportedOperationException} if the list is immutable. Subclasses are
+   * responsible for invoking this method on mutate operations.
+   */
+  protected void ensureIsMutable() {
+    if (!isMutable) {
+      throw new UnsupportedOperationException();
+    }
+  }
+}

java/src/main/java/com/google/protobuf/BooleanArrayList.java

@@ -0,0 +1,244 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package com.google.protobuf;
+
+import com.google.protobuf.Internal.BooleanList;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.RandomAccess;
+
+/**
+ * An implementation of {@link BooleanList} on top of a primitive array.
+ * 
+ * @author dweis@google.com (Daniel Weis)
+ */
+final class BooleanArrayList
+    extends AbstractProtobufList<Boolean> implements BooleanList, RandomAccess {
+  
+  private static final int DEFAULT_CAPACITY = 10;
+  
+  private static final BooleanArrayList EMPTY_LIST = new BooleanArrayList();
+  static {
+    EMPTY_LIST.makeImmutable();
+  }
+  
+  public static BooleanArrayList emptyList() {
+    return EMPTY_LIST;
+  }
+  
+  /**
+   * The backing store for the list.
+   */
+  private boolean[] array;
+  
+  /**
+   * The size of the list distinct from the length of the array. That is, it is the number of
+   * elements set in the list.
+   */
+  private int size;
+
+  /**
+   * Constructs a new mutable {@code BooleanArrayList}.
+   */
+  BooleanArrayList() {
+    array = new boolean[DEFAULT_CAPACITY];
+    size = 0;
+  }
+
+  /**
+   * Constructs a new mutable {@code BooleanArrayList} containing the same elements as
+   * {@code other}.
+   */
+  BooleanArrayList(List<Boolean> other) {
+    if (other instanceof BooleanArrayList) {
+      BooleanArrayList list = (BooleanArrayList) other;
+      array = list.array.clone();
+      size = list.size;
+    } else {
+      size = other.size();
+      array = new boolean[size];
+      for (int i = 0; i < size; i++) {
+        array[i] = other.get(i);
+      }
+    }
+  }
+  
+  @Override
+  public Boolean get(int index) {
+    return getBoolean(index);
+  }
+
+  @Override
+  public boolean getBoolean(int index) {
+    ensureIndexInRange(index);
+    return array[index];
+  }
+
+  @Override
+  public int size() {
+    return size;
+  }
+
+  @Override
+  public Boolean set(int index, Boolean element) {
+    return setBoolean(index, element);
+  }
+
+  @Override
+  public boolean setBoolean(int index, boolean element) {
+    ensureIsMutable();
+    ensureIndexInRange(index);
+    boolean previousValue = array[index];
+    array[index] = element;
+    return previousValue;
+  }
+
+  @Override
+  public void add(int index, Boolean element) {
+    addBoolean(index, element);
+  }
+
+  /**
+   * Like {@link #add(Boolean)} but more efficient in that it doesn't box the element.
+   */
+  @Override
+  public void addBoolean(boolean element) {
+    addBoolean(size, element);
+  }
+
+  /**
+   * Like {@link #add(int, Boolean)} but more efficient in that it doesn't box the element.
+   */
+  private void addBoolean(int index, boolean element) {
+    ensureIsMutable();
+    if (index < 0 || index > size) {
+      throw new IndexOutOfBoundsException(makeOutOfBoundsExceptionMessage(index));
+    }
+    
+    if (size < array.length) {
+      // Shift everything over to make room
+      System.arraycopy(array, index, array, index + 1, size - index);
+    } else {
+      // Resize to 1.5x the size
+      int length = ((size * 3) / 2) + 1;
+      boolean[] newArray = new boolean[length];
+      
+      // Copy the first part directly
+      System.arraycopy(array, 0, newArray, 0, index);
+      
+      // Copy the rest shifted over by one to make room
+      System.arraycopy(array, index, newArray, index + 1, size - index);
+      array = newArray;
+    }
+
+    array[index] = element;
+    size++;
+    modCount++;
+  }
+
+  @Override
+  public boolean addAll(Collection<? extends Boolean> collection) {
+    ensureIsMutable();
+    
+    if (collection == null) {
+      throw new NullPointerException();
+    }
+    
+    // We specialize when adding another BooleanArrayList to avoid boxing elements.
+    if (!(collection instanceof BooleanArrayList)) {
+      return super.addAll(collection);
+    }
+    
+    BooleanArrayList list = (BooleanArrayList) collection;
+    if (list.size == 0) {
+      return false;
+    }
+    
+    int overflow = Integer.MAX_VALUE - size;
+    if (overflow < list.size) {
+      // We can't actually represent a list this large.
+      throw new OutOfMemoryError();
+    }
+    
+    int newSize = size + list.size;
+    if (newSize > array.length) {
+      array = Arrays.copyOf(array, newSize);
+    }
+    
+    System.arraycopy(list.array, 0, array, size, list.size);
+    size = newSize;
+    modCount++;
+    return true;
+  }
+  
+  @Override
+  public boolean remove(Object o) {
+    ensureIsMutable();
+    for (int i = 0; i < size; i++) {
+      if (o.equals(array[i])) {
+        System.arraycopy(array, i + 1, array, i, size - i);
+        size--;
+        modCount++;
+        return true;
+      }
+    }
+    return false;
+  }
+
+  @Override
+  public Boolean remove(int index) {
+    ensureIsMutable();
+    ensureIndexInRange(index);
+    boolean value = array[index];
+    System.arraycopy(array, index + 1, array, index, size - index);
+    size--;
+    modCount++;
+    return value;
+  }
+
+  /**
+   * Ensures that the provided {@code index} is within the range of {@code [0, size]}. Throws an
+   * {@link IndexOutOfBoundsException} if it is not.
+   * 
+   * @param index the index to verify is in range
+   */
+  private void ensureIndexInRange(int index) {
+    if (index < 0 || index >= size) {
+      throw new IndexOutOfBoundsException(makeOutOfBoundsExceptionMessage(index));
+    }
+  }
+
+  private String makeOutOfBoundsExceptionMessage(int index) {
+    return "Index:" + index + ", Size:" + size;
+  }
+}

java/src/main/java/com/google/protobuf/Descriptors.java

@@ -644,6 +644,30 @@ public final class Descriptors {
       return false;
     }
 
+    /** Determines if the given field number is reserved. */
+    public boolean isReservedNumber(final int number) {
+      for (final DescriptorProto.ReservedRange range :
+          proto.getReservedRangeList()) {
+        if (range.getStart() <= number && number < range.getEnd()) {
+          return true;
+        }
+      }
+      return false;
+    }
+
+    /** Determines if the given field name is reserved. */
+    public boolean isReservedName(final String name) {
+      if (name == null) {
+        throw new NullPointerException();
+      }
+      for (final String reservedName : proto.getReservedNameList()) {
+        if (reservedName.equals(name)) {
+          return true;
+        }
+      }
+      return false;
+    }
+
     /**
      * Indicates whether the message can be extended.  That is, whether it has
      * any "extensions x to y" ranges declared on it.
@@ -917,9 +941,18 @@ public final class Descriptors {
       return proto.getLabel() == FieldDescriptorProto.Label.LABEL_REPEATED;
     }
 
-    /** Does this field have the {@code [packed = true]} option? */
+    /** Does this field have the {@code [packed = true]} option or is this field
+     *  packable in proto3 and not explicitly setted to unpacked?
+     */
     public boolean isPacked() {
-      return getOptions().getPacked();
+      if (!isPackable()) {
+        return false;
+      }
+      if (getFile().getSyntax() == FileDescriptor.Syntax.PROTO2) {
+        return getOptions().getPacked();
+      } else {
+        return !getOptions().hasPacked() || getOptions().getPacked();
+      }
     }
 
     /** Can this field be packed? i.e. is it a repeated primitive field? */
@@ -2317,6 +2350,11 @@ public final class Descriptors {
 
     public int getFieldCount() { return fieldCount; }
 
+    /** Get a list of this message type's fields. */
+    public List<FieldDescriptor> getFields() {
+      return Collections.unmodifiableList(Arrays.asList(fields));
+    }
+
     public FieldDescriptor getField(int index) {
       return fields[index];
     }

java/src/main/java/com/google/protobuf/DoubleArrayList.java

@@ -0,0 +1,243 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package com.google.protobuf;
+
+import com.google.protobuf.Internal.DoubleList;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.RandomAccess;
+
+/**
+ * An implementation of {@link DoubleList} on top of a primitive array.
+ * 
+ * @author dweis@google.com (Daniel Weis)
+ */
+final class DoubleArrayList
+    extends AbstractProtobufList<Double> implements DoubleList, RandomAccess {
+  
+  private static final int DEFAULT_CAPACITY = 10;
+  
+  private static final DoubleArrayList EMPTY_LIST = new DoubleArrayList();
+  static {
+    EMPTY_LIST.makeImmutable();
+  }
+  
+  public static DoubleArrayList emptyList() {
+    return EMPTY_LIST;
+  }
+  
+  /**
+   * The backing store for the list.
+   */
+  private double[] array;
+  
+  /**
+   * The size of the list distinct from the length of the array. That is, it is the number of
+   * elements set in the list.
+   */
+  private int size;
+
+  /**
+   * Constructs a new mutable {@code DoubleArrayList}.
+   */
+  DoubleArrayList() {
+    array = new double[DEFAULT_CAPACITY];
+    size = 0;
+  }
+
+  /**
+   * Constructs a new mutable {@code DoubleArrayList} containing the same elements as {@code other}.
+   */
+  DoubleArrayList(List<Double> other) {
+    if (other instanceof DoubleArrayList) {
+      DoubleArrayList list = (DoubleArrayList) other;
+      array = list.array.clone();
+      size = list.size;
+    } else {
+      size = other.size();
+      array = new double[size];
+      for (int i = 0; i < size; i++) {
+        array[i] = other.get(i);
+      }
+    }
+  }
+  
+  @Override
+  public Double get(int index) {
+    return getDouble(index);
+  }
+
+  @Override
+  public double getDouble(int index) {
+    ensureIndexInRange(index);
+    return array[index];
+  }
+
+  @Override
+  public int size() {
+    return size;
+  }
+
+  @Override
+  public Double set(int index, Double element) {
+    return setDouble(index, element);
+  }
+
+  @Override
+  public double setDouble(int index, double element) {
+    ensureIsMutable();
+    ensureIndexInRange(index);
+    double previousValue = array[index];
+    array[index] = element;
+    return previousValue;
+  }
+
+  @Override
+  public void add(int index, Double element) {
+    addDouble(index, element);
+  }
+
+  /**
+   * Like {@link #add(Double)} but more efficient in that it doesn't box the element.
+   */
+  @Override
+  public void addDouble(double element) {
+    addDouble(size, element);
+  }
+
+  /**
+   * Like {@link #add(int, Double)} but more efficient in that it doesn't box the element.
+   */
+  private void addDouble(int index, double element) {
+    ensureIsMutable();
+    if (index < 0 || index > size) {
+      throw new IndexOutOfBoundsException(makeOutOfBoundsExceptionMessage(index));
+    }
+    
+    if (size < array.length) {
+      // Shift everything over to make room
+      System.arraycopy(array, index, array, index + 1, size - index);
+    } else {
+      // Resize to 1.5x the size
+      int length = ((size * 3) / 2) + 1;
+      double[] newArray = new double[length];
+      
+      // Copy the first part directly
+      System.arraycopy(array, 0, newArray, 0, index);
+      
+      // Copy the rest shifted over by one to make room
+      System.arraycopy(array, index, newArray, index + 1, size - index);
+      array = newArray;
+    }
+
+    array[index] = element;
+    size++;
+    modCount++;
+  }
+
+  @Override
+  public boolean addAll(Collection<? extends Double> collection) {
+    ensureIsMutable();
+    
+    if (collection == null) {
+      throw new NullPointerException();
+    }
+    
+    // We specialize when adding another DoubleArrayList to avoid boxing elements.
+    if (!(collection instanceof DoubleArrayList)) {
+      return super.addAll(collection);
+    }
+    
+    DoubleArrayList list = (DoubleArrayList) collection;
+    if (list.size == 0) {
+      return false;
+    }
+    
+    int overflow = Integer.MAX_VALUE - size;
+    if (overflow < list.size) {
+      // We can't actually represent a list this large.
+      throw new OutOfMemoryError();
+    }
+    
+    int newSize = size + list.size;
+    if (newSize > array.length) {
+      array = Arrays.copyOf(array, newSize);
+    }
+    
+    System.arraycopy(list.array, 0, array, size, list.size);
+    size = newSize;
+    modCount++;
+    return true;
+  }
+  
+  @Override
+  public boolean remove(Object o) {
+    ensureIsMutable();
+    for (int i = 0; i < size; i++) {
+      if (o.equals(array[i])) {
+        System.arraycopy(array, i + 1, array, i, size - i);
+        size--;
+        modCount++;
+        return true;
+      }
+    }
+    return false;
+  }
+
+  @Override
+  public Double remove(int index) {
+    ensureIsMutable();
+    ensureIndexInRange(index);
+    double value = array[index];
+    System.arraycopy(array, index + 1, array, index, size - index);
+    size--;
+    modCount++;
+    return value;
+  }
+
+  /**
+   * Ensures that the provided {@code index} is within the range of {@code [0, size]}. Throws an
+   * {@link IndexOutOfBoundsException} if it is not.
+   * 
+   * @param index the index to verify is in range
+   */
+  private void ensureIndexInRange(int index) {
+    if (index < 0 || index >= size) {
+      throw new IndexOutOfBoundsException(makeOutOfBoundsExceptionMessage(index));
+    }
+  }
+
+  private String makeOutOfBoundsExceptionMessage(int index) {
+    return "Index:" + index + ", Size:" + size;
+  }
+}

java/src/main/java/com/google/protobuf/FloatArrayList.java

@@ -0,0 +1,242 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package com.google.protobuf;
+
+import com.google.protobuf.Internal.FloatList;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.RandomAccess;
+
+/**
+ * An implementation of {@link FloatList} on top of a primitive array.
+ * 
+ * @author dweis@google.com (Daniel Weis)
+ */
+final class FloatArrayList extends AbstractProtobufList<Float> implements FloatList, RandomAccess {
+  
+  private static final int DEFAULT_CAPACITY = 10;
+  
+  private static final FloatArrayList EMPTY_LIST = new FloatArrayList();
+  static {
+    EMPTY_LIST.makeImmutable();
+  }
+  
+  public static FloatArrayList emptyList() {
+    return EMPTY_LIST;
+  }
+  
+  /**
+   * The backing store for the list.
+   */
+  private float[] array;
+  
+  /**
+   * The size of the list distinct from the length of the array. That is, it is the number of
+   * elements set in the list.
+   */
+  private int size;
+
+  /**
+   * Constructs a new mutable {@code FloatArrayList}.
+   */
+  FloatArrayList() {
+    array = new float[DEFAULT_CAPACITY];
+    size = 0;
+  }
+
+  /**
+   * Constructs a new mutable {@code FloatArrayList} containing the same elements as {@code other}.
+   */
+  FloatArrayList(List<Float> other) {
+    if (other instanceof FloatArrayList) {
+      FloatArrayList list = (FloatArrayList) other;
+      array = list.array.clone();
+      size = list.size;
+    } else {
+      size = other.size();
+      array = new float[size];
+      for (int i = 0; i < size; i++) {
+        array[i] = other.get(i);
+      }
+    }
+  }
+  
+  @Override
+  public Float get(int index) {
+    return getFloat(index);
+  }
+
+  @Override
+  public float getFloat(int index) {
+    ensureIndexInRange(index);
+    return array[index];
+  }
+
+  @Override
+  public int size() {
+    return size;
+  }
+
+  @Override
+  public Float set(int index, Float element) {
+    return setFloat(index, element);
+  }
+
+  @Override
+  public float setFloat(int index, float element) {
+    ensureIsMutable();
+    ensureIndexInRange(index);
+    float previousValue = array[index];
+    array[index] = element;
+    return previousValue;
+  }
+
+  @Override
+  public void add(int index, Float element) {
+    addFloat(index, element);
+  }
+
+  /**
+   * Like {@link #add(Float)} but more efficient in that it doesn't box the element.
+   */
+  @Override
+  public void addFloat(float element) {
+    addFloat(size, element);
+  }
+
+  /**
+   * Like {@link #add(int, Float)} but more efficient in that it doesn't box the element.
+   */
+  private void addFloat(int index, float element) {
+    ensureIsMutable();
+    if (index < 0 || index > size) {
+      throw new IndexOutOfBoundsException(makeOutOfBoundsExceptionMessage(index));
+    }
+    
+    if (size < array.length) {
+      // Shift everything over to make room
+      System.arraycopy(array, index, array, index + 1, size - index);
+    } else {
+      // Resize to 1.5x the size
+      int length = ((size * 3) / 2) + 1;
+      float[] newArray = new float[length];
+      
+      // Copy the first part directly
+      System.arraycopy(array, 0, newArray, 0, index);
+      
+      // Copy the rest shifted over by one to make room
+      System.arraycopy(array, index, newArray, index + 1, size - index);
+      array = newArray;
+    }
+
+    array[index] = element;
+    size++;
+    modCount++;
+  }
+
+  @Override
+  public boolean addAll(Collection<? extends Float> collection) {
+    ensureIsMutable();
+    
+    if (collection == null) {
+      throw new NullPointerException();
+    }
+    
+    // We specialize when adding another FloatArrayList to avoid boxing elements.
+    if (!(collection instanceof FloatArrayList)) {
+      return super.addAll(collection);
+    }
+    
+    FloatArrayList list = (FloatArrayList) collection;
+    if (list.size == 0) {
+      return false;
+    }
+    
+    int overflow = Integer.MAX_VALUE - size;
+    if (overflow < list.size) {
+      // We can't actually represent a list this large.
+      throw new OutOfMemoryError();
+    }
+    
+    int newSize = size + list.size;
+    if (newSize > array.length) {
+      array = Arrays.copyOf(array, newSize);
+    }
+    
+    System.arraycopy(list.array, 0, array, size, list.size);
+    size = newSize;
+    modCount++;
+    return true;
+  }
+  
+  @Override
+  public boolean remove(Object o) {
+    ensureIsMutable();
+    for (int i = 0; i < size; i++) {
+      if (o.equals(array[i])) {
+        System.arraycopy(array, i + 1, array, i, size - i);
+        size--;
+        modCount++;
+        return true;
+      }
+    }
+    return false;
+  }
+
+  @Override
+  public Float remove(int index) {
+    ensureIsMutable();
+    ensureIndexInRange(index);
+    float value = array[index];
+    System.arraycopy(array, index + 1, array, index, size - index);
+    size--;
+    modCount++;
+    return value;
+  }
+
+  /**
+   * Ensures that the provided {@code index} is within the range of {@code [0, size]}. Throws an
+   * {@link IndexOutOfBoundsException} if it is not.
+   * 
+   * @param index the index to verify is in range
+   */
+  private void ensureIndexInRange(int index) {
+    if (index < 0 || index >= size) {
+      throw new IndexOutOfBoundsException(makeOutOfBoundsExceptionMessage(index));
+    }
+  }
+
+  private String makeOutOfBoundsExceptionMessage(int index) {
+    return "Index:" + index + ", Size:" + size;
+  }
+}

java/src/main/java/com/google/protobuf/GeneratedMessageLite.java

@@ -30,6 +30,12 @@
 
 package com.google.protobuf;
 
+import com.google.protobuf.Internal.BooleanList;
+import com.google.protobuf.Internal.DoubleList;
+import com.google.protobuf.Internal.FloatList;
+import com.google.protobuf.Internal.IntList;
+import com.google.protobuf.Internal.LongList;
+import com.google.protobuf.Internal.ProtobufList;
 import com.google.protobuf.WireFormat.FieldType;
 
 import java.io.IOException;
@@ -76,7 +82,11 @@ public abstract class GeneratedMessageLite<
   private static final long serialVersionUID = 1L;
 
   /** For use by generated code only.  */
-  protected UnknownFieldSetLite unknownFields;
+  protected UnknownFieldSetLite unknownFields =
+      UnknownFieldSetLite.getDefaultInstance();
+  
+  /** For use by generated code only.  */
+  protected int memoizedSerializedSize = -1;
   
   @SuppressWarnings("unchecked") // Guaranteed by runtime.
   public final Parser<MessageType> getParserForType() {
@@ -109,10 +119,67 @@ public abstract class GeneratedMessageLite<
     return unknownFields.mergeFieldFrom(tag, input);
   }
 
-  // The default behavior. If a message has required fields in its subtree, the
-  // generated code will override.
-  public boolean isInitialized() {
-    return true;
+  public final boolean isInitialized() {
+    return dynamicMethod(MethodToInvoke.IS_INITIALIZED, Boolean.TRUE) != null;
+  }
+
+  public final BuilderType toBuilder() {
+    BuilderType builder = (BuilderType) dynamicMethod(MethodToInvoke.NEW_BUILDER);
+    builder.mergeFrom((MessageType) this);
+    return builder;
+  }
+
+  /**
+   * Defines which method path to invoke in {@link GeneratedMessageLite
+   * #dynamicMethod(MethodToInvoke, Object...)}.
+   * <p>
+   * For use by generated code only.
+   */
+  public static enum MethodToInvoke {
+    IS_INITIALIZED,
+    PARSE_PARTIAL_FROM,
+    MERGE_FROM,
+    MAKE_IMMUTABLE,
+    NEW_INSTANCE,
+    NEW_BUILDER;
+  }
+
+  /**
+   * A method that implements different types of operations described in {@link MethodToInvoke}.
+   * Theses different kinds of operations are required to implement message-level operations for
+   * builders in the runtime. This method bundles those operations to reduce the generated methods
+   * count.
+   * <ul>
+   * <li>{@code PARSE_PARTIAL_FROM} is parameterized with an {@link CodedInputStream} and
+   * {@link ExtensionRegistryLite}. It consumes the input stream, parsing the contents into the
+   * returned protocol buffer. If parsing throws an {@link InvalidProtocolBufferException}, the
+   * implementation wraps it in a RuntimeException
+   * <li>{@code NEW_INSTANCE} returns a new instance of the protocol buffer
+   * <li>{@code IS_INITIALIZED} is parameterized with a {@code Boolean} detailing whether to
+   * memoize. It returns {@code null} for false and the default instance for true. We optionally
+   * memoize to support the Builder case, where memoization is not desired.
+   * <li>{@code NEW_BUILDER} returns a {@code BuilderType} instance.
+   * <li>{@code MERGE_FROM} is parameterized with a {@code MessageType} and merges the fields from
+   * that instance into this instance.
+   * <li>{@code MAKE_IMMUTABLE} sets all internal fields to an immutable state.
+   * </ul>
+   * This method, plus the implementation of the Builder, enables the Builder class to be proguarded
+   * away entirely on Android.
+   * <p>
+   * For use by generated code only.
+   */
+  protected abstract Object dynamicMethod(
+      MethodToInvoke method,
+      Object... args);
+
+  /**
+   * Merge some unknown fields into the {@link UnknownFieldSetLite} for this
+   * message.
+   *
+   * <p>For use by generated code only.
+   */
+  protected final void mergeUnknownFields(UnknownFieldSetLite unknownFields) {
+    this.unknownFields = UnknownFieldSetLite.concat(this.unknownFields, unknownFields);
   }
 
   @SuppressWarnings("unchecked")
@@ -122,24 +189,37 @@ public abstract class GeneratedMessageLite<
           extends AbstractMessageLite.Builder<BuilderType> {
 
     private final MessageType defaultInstance;
-
-    /** For use by generated code only. */
-    protected UnknownFieldSetLite unknownFields =
-        UnknownFieldSetLite.getDefaultInstance();
+    protected MessageType instance;
+    protected boolean isBuilt;
 
     protected Builder(MessageType defaultInstance) {
       this.defaultInstance = defaultInstance;
+      this.instance = (MessageType) defaultInstance.dynamicMethod(MethodToInvoke.NEW_INSTANCE);
+      isBuilt = false;
     }
 
-    // The default behavior. If a message has required fields in its subtree,
-    // the generated code will override.
-    public boolean isInitialized() {
-      return true;
+    /**
+     * Called before any method that would mutate the builder to ensure that it correctly copies
+     * any state before the write happens to preserve immutability guarantees.
+     */
+    protected void copyOnWrite() {
+      if (isBuilt) {
+        MessageType newInstance = (MessageType) instance.dynamicMethod(MethodToInvoke.NEW_INSTANCE);
+        newInstance.dynamicMethod(MethodToInvoke.MERGE_FROM, instance);
+        instance = newInstance;
+        isBuilt = false;
+      }
+    }
+
+    //@Override (Java 1.6 override semantics, but we must support 1.5)
+    public final boolean isInitialized() {
+      return GeneratedMessageLite.isInitialized(instance, false /* shouldMemoize */);
     }
 
     //@Override (Java 1.6 override semantics, but we must support 1.5)
-    public BuilderType clear() {
-      unknownFields = UnknownFieldSetLite.getDefaultInstance();
+    public final BuilderType clear() {
+      // No need to copy on write since we're dropping the instance anyways.
+      instance = (MessageType) instance.dynamicMethod(MethodToInvoke.NEW_INSTANCE);
       return (BuilderType) this;
     }
 
@@ -151,8 +231,12 @@ public abstract class GeneratedMessageLite<
       return builder;
     }
 
-    /** All subclasses implement this. */
-    public abstract MessageType buildPartial();
+    //@Override (Java 1.6 override semantics, but we must support 1.5)
+    public MessageType buildPartial() {
+      instance.dynamicMethod(MethodToInvoke.MAKE_IMMUTABLE);
+      isBuilt = true;
+      return instance;
+    }
 
     //@Override (Java 1.6 override semantics, but we must support 1.5)
     public final MessageType build() {
@@ -162,9 +246,13 @@ public abstract class GeneratedMessageLite<
       }
       return result;
     }
-
+    
     /** All subclasses implement this. */
-    public abstract BuilderType mergeFrom(MessageType message);
+    public BuilderType mergeFrom(MessageType message) {
+      copyOnWrite();
+      instance.dynamicMethod(MethodToInvoke.MERGE_FROM, message);
+      return (BuilderType) this;
+    }
     
     public MessageType getDefaultInstanceForType() {
       return defaultInstance;
@@ -181,18 +269,6 @@ public abstract class GeneratedMessageLite<
         int tag) throws IOException {
       return unknownFields.mergeFieldFrom(tag, input);
     }
-
-    /**
-     * Merge some unknown fields into the {@link UnknownFieldSetLite} for this
-     * message.
-     *
-     * <p>For use by generated code only.
-     */
-    protected final BuilderType mergeUnknownFields(
-        final UnknownFieldSetLite unknownFields) {
-      this.unknownFields = UnknownFieldSetLite.concat(this.unknownFields, unknownFields);
-      return (BuilderType) this;
-    }
     
     public BuilderType mergeFrom(
         com.google.protobuf.CodedInputStream input,
@@ -259,19 +335,13 @@ public abstract class GeneratedMessageLite<
      */
     protected FieldSet<ExtensionDescriptor> extensions = FieldSet.newFieldSet();
 
-    // -1 => not memoized, 0 => false, 1 => true.
-    private byte memoizedIsInitialized = -1;
-
-    // The default behavior. If a message has required fields in its subtree,
-    // the generated code will override.
-    public boolean isInitialized() {
-      if (memoizedIsInitialized == -1) {
-        memoizedIsInitialized = (byte) (extensions.isInitialized() ? 1 : 0);
+    protected final void mergeExtensionFields(final MessageType other) {
+      if (extensions.isImmutable()) {
+        extensions = extensions.clone();
       }
-
-      return memoizedIsInitialized == 1;
+      extensions.mergeFrom(((ExtendableMessage) other).extensions);
     }
-
+    
     private void verifyExtensionContainingType(
         final GeneratedExtension<MessageType, ?> extension) {
       if (extension.getContainingTypeDefaultInstance() !=
@@ -420,46 +490,38 @@ public abstract class GeneratedMessageLite<
       implements ExtendableMessageOrBuilder<MessageType, BuilderType> {
     protected ExtendableBuilder(MessageType defaultInstance) {
       super(defaultInstance);
-    }
-
-    private FieldSet<ExtensionDescriptor> extensions = FieldSet.emptySet();
-    private boolean extensionsIsMutable;
-
-    // The default behavior. If a message has required fields in its subtree,
-    // the generated code will override.
-    public boolean isInitialized() {
-      return extensions.isInitialized();
+      
+      // TODO(dweis): This is kind of an unnecessary clone since we construct a
+      //     new instance in the parent constructor which makes the extensions
+      //     immutable. This extra allocation shouldn't matter in practice
+      //     though.
+      instance.extensions = instance.extensions.clone();
     }
 
     // For immutable message conversion.
     void internalSetExtensionSet(FieldSet<ExtensionDescriptor> extensions) {
-      this.extensions = extensions;
+      copyOnWrite();
+      instance.extensions = extensions;
     }
 
-    @Override
-    public BuilderType clear() {
-      extensions.clear();
-      extensionsIsMutable = false;
-      return super.clear();
+    // @Override (Java 1.6 override semantics, but we must support 1.5)
+    protected void copyOnWrite() {
+      if (!isBuilt) {
+        return;
+      }
+      
+      super.copyOnWrite();
+      instance.extensions = instance.extensions.clone();
     }
 
-    private void ensureExtensionsIsMutable() {
-      if (!extensionsIsMutable) {
-        extensions = extensions.clone();
-        extensionsIsMutable = true;
+    // @Override (Java 1.6 override semantics, but we must support 1.5)
+    public final MessageType buildPartial() {
+      if (isBuilt) {
+        return instance;
       }
-    }
 
-    /**
-     * Called by the build code path to create a copy of the extensions for
-     * building the message.
-     * <p>
-     * For use by generated code only.
-     */
-    protected final FieldSet<ExtensionDescriptor> buildExtensions() {
-      extensions.makeImmutable();
-      extensionsIsMutable = false;
-      return extensions;
+      instance.extensions.makeImmutable();
+      return super.buildPartial();
     }
 
     private void verifyExtensionContainingType(
@@ -477,22 +539,14 @@ public abstract class GeneratedMessageLite<
     //@Override (Java 1.6 override semantics, but we must support 1.5)
     public final <Type> boolean hasExtension(
         final ExtensionLite<MessageType, Type> extension) {
-      GeneratedExtension<MessageType, Type> extensionLite =
-          checkIsLite(extension);
-      
-      verifyExtensionContainingType(extensionLite);
-      return extensions.hasField(extensionLite.descriptor);
+      return instance.hasExtension(extension);
     }
 
     /** Get the number of elements in a repeated extension. */
     //@Override (Java 1.6 override semantics, but we must support 1.5)
     public final <Type> int getExtensionCount(
         final ExtensionLite<MessageType, List<Type>> extension) {
-      GeneratedExtension<MessageType, List<Type>> extensionLite =
-          checkIsLite(extension);
-      
-      verifyExtensionContainingType(extensionLite);
-      return extensions.getRepeatedFieldCount(extensionLite.descriptor);
+      return instance.getExtensionCount(extension);
     }
 
     /** Get the value of an extension. */
@@ -500,16 +554,7 @@ public abstract class GeneratedMessageLite<
     @SuppressWarnings("unchecked")
     public final <Type> Type getExtension(
         final ExtensionLite<MessageType, Type> extension) {
-      GeneratedExtension<MessageType, Type> extensionLite =
-          checkIsLite(extension);
-      
-      verifyExtensionContainingType(extensionLite);
-      final Object value = extensions.getField(extensionLite.descriptor);
-      if (value == null) {
-        return extensionLite.defaultValue;
-      } else {
-        return (Type) extensionLite.fromFieldSetType(value);
-      }
+      return instance.getExtension(extension);
     }
 
     /** Get one element of a repeated extension. */
@@ -518,12 +563,7 @@ public abstract class GeneratedMessageLite<
     public final <Type> Type getExtension(
         final ExtensionLite<MessageType, List<Type>> extension,
         final int index) {
-      GeneratedExtension<MessageType, List<Type>> extensionLite =
-          checkIsLite(extension);
-      
-      verifyExtensionContainingType(extensionLite);
-      return (Type) extensionLite.singularFromFieldSetType(
-          extensions.getRepeatedField(extensionLite.descriptor, index));
+      return instance.getExtension(extension, index);
     }
 
     // This is implemented here only to work around an apparent bug in the
@@ -542,9 +582,8 @@ public abstract class GeneratedMessageLite<
           checkIsLite(extension);
       
       verifyExtensionContainingType(extensionLite);
-      ensureExtensionsIsMutable();
-      extensions.setField(extensionLite.descriptor,
-                          extensionLite.toFieldSetType(value));
+      copyOnWrite();
+      instance.extensions.setField(extensionLite.descriptor, extensionLite.toFieldSetType(value));
       return (BuilderType) this;
     }
 
@@ -556,9 +595,9 @@ public abstract class GeneratedMessageLite<
           checkIsLite(extension);
       
       verifyExtensionContainingType(extensionLite);
-      ensureExtensionsIsMutable();
-      extensions.setRepeatedField(extensionLite.descriptor, index,
-                                  extensionLite.singularToFieldSetType(value));
+      copyOnWrite();
+      instance.extensions.setRepeatedField(
+          extensionLite.descriptor, index, extensionLite.singularToFieldSetType(value));
       return (BuilderType) this;
     }
 
@@ -570,9 +609,9 @@ public abstract class GeneratedMessageLite<
           checkIsLite(extension);
       
       verifyExtensionContainingType(extensionLite);
-      ensureExtensionsIsMutable();
-      extensions.addRepeatedField(extensionLite.descriptor,
-                                  extensionLite.singularToFieldSetType(value));
+      copyOnWrite();
+      instance.extensions.addRepeatedField(
+          extensionLite.descriptor, extensionLite.singularToFieldSetType(value));
       return (BuilderType) this;
     }
 
@@ -582,20 +621,10 @@ public abstract class GeneratedMessageLite<
       GeneratedExtension<MessageType, ?> extensionLite = checkIsLite(extension);
       
       verifyExtensionContainingType(extensionLite);
-      ensureExtensionsIsMutable();
-      extensions.clearField(extensionLite.descriptor);
+      copyOnWrite();
+      instance.extensions.clearField(extensionLite.descriptor);
       return (BuilderType) this;
     }
-
-    /** Called by subclasses to check if all extensions are initialized. */
-    protected boolean extensionsAreInitialized() {
-      return extensions.isInitialized();
-    }
-
-    protected final void mergeExtensionFields(final MessageType other) {
-      ensureExtensionsIsMutable();
-      extensions.mergeFrom(((ExtendableMessage) other).extensions);
-    }
   }
 
   //-----------------------------------------------------------------
@@ -1113,4 +1142,133 @@ public abstract class GeneratedMessageLite<
       return (BuilderType) defaultInstance.toBuilder();
     }
   }
+
+  /**
+   * A static helper method for checking if a message is initialized, optionally memoizing.
+   * <p>
+   * For use by generated code only.
+   */
+  protected static final <T extends GeneratedMessageLite<T, ?>> boolean isInitialized(
+      T message, boolean shouldMemoize) {
+    return message.dynamicMethod(MethodToInvoke.IS_INITIALIZED, shouldMemoize) != null;
+  }
+  
+  protected static final <T extends GeneratedMessageLite<T, ?>> void makeImmutable(T message) {
+    message.dynamicMethod(MethodToInvoke.MAKE_IMMUTABLE);
+  }
+  
+  /**
+   * A static helper method for parsing a partial from input using the extension registry and the
+   * instance.
+   */
+  static <T extends GeneratedMessageLite<T, ?>> T parsePartialFrom(
+      T instance, CodedInputStream input, ExtensionRegistryLite extensionRegistry)
+          throws InvalidProtocolBufferException {
+    try {
+      return (T) instance.dynamicMethod(
+          MethodToInvoke.PARSE_PARTIAL_FROM, input, extensionRegistry);
+    } catch (RuntimeException e) {
+      if (e.getCause() instanceof InvalidProtocolBufferException) {
+        throw (InvalidProtocolBufferException) e.getCause();
+      }
+      throw e;
+    }
+  }
+  
+  /**
+   * A {@link Parser} implementation that delegates to the default instance.
+   * <p>
+   * For use by generated code only.
+   */
+  protected static class DefaultInstanceBasedParser<T extends GeneratedMessageLite<T, ?>>
+      extends AbstractParser<T> {
+    
+    private T defaultInstance;
+    
+    public DefaultInstanceBasedParser(T defaultInstance) {
+      this.defaultInstance = defaultInstance;
+    }
+    
+    @Override
+    public T parsePartialFrom(CodedInputStream input, ExtensionRegistryLite extensionRegistry)
+        throws InvalidProtocolBufferException {
+      return GeneratedMessageLite.parsePartialFrom(defaultInstance, input, extensionRegistry);
+    }
+  }
+  
+  protected static IntList newIntList() {
+    return new IntArrayList();
+  }
+  
+  protected static IntList newIntList(List<Integer> toCopy) {
+    return new IntArrayList(toCopy);
+  }
+  
+  protected static IntList emptyIntList() {
+    return IntArrayList.emptyList();
+  }
+  
+  protected static LongList newLongList() {
+    return new LongArrayList();
+  }
+  
+  protected static LongList newLongList(List<Long> toCopy) {
+    return new LongArrayList(toCopy);
+  }
+  
+  protected static LongList emptyLongList() {
+    return LongArrayList.emptyList();
+  }
+  
+  protected static FloatList newFloatList() {
+    return new FloatArrayList();
+  }
+  
+  protected static FloatList newFloatList(List<Float> toCopy) {
+    return new FloatArrayList(toCopy);
+  }
+  
+  protected static FloatList emptyFloatList() {
+    return FloatArrayList.emptyList();
+  }
+  
+  protected static DoubleList newDoubleList() {
+    return new DoubleArrayList();
+  }
+  
+  protected static DoubleList newDoubleList(List<Double> toCopy) {
+    return new DoubleArrayList(toCopy);
+  }
+  
+  protected static DoubleList emptyDoubleList() {
+    return DoubleArrayList.emptyList();
+  }
+  
+  protected static BooleanList newBooleanList() {
+    return new BooleanArrayList();
+  }
+  
+  protected static BooleanList newBooleanList(List<Boolean> toCopy) {
+    return new BooleanArrayList(toCopy);
+  }
+  
+  protected static BooleanList emptyBooleanList() {
+    return BooleanArrayList.emptyList();
+  }
+  
+  protected static <E> ProtobufList<E> newProtobufList() {
+    return new ProtobufArrayList<E>();
+  }
+  
+  protected static <E> ProtobufList<E> newProtobufList(List<E> toCopy) {
+    return new ProtobufArrayList<E>(toCopy);
+  }
+  
+  protected static <E> ProtobufList<E> emptyProtobufList() {
+    return ProtobufArrayList.emptyList();
+  }
+  
+  protected static LazyStringArrayList emptyLazyStringArrayList() {
+    return LazyStringArrayList.emptyList();
+  }
 }

java/src/main/java/com/google/protobuf/IntArrayList.java

@@ -0,0 +1,242 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package com.google.protobuf;
+
+import com.google.protobuf.Internal.IntList;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.RandomAccess;
+
+/**
+ * An implementation of {@link IntList} on top of a primitive array.
+ * 
+ * @author dweis@google.com (Daniel Weis)
+ */
+final class IntArrayList extends AbstractProtobufList<Integer> implements IntList, RandomAccess {
+  
+  private static final int DEFAULT_CAPACITY = 10;
+  
+  private static final IntArrayList EMPTY_LIST = new IntArrayList();
+  static {
+    EMPTY_LIST.makeImmutable();
+  }
+  
+  public static IntArrayList emptyList() {
+    return EMPTY_LIST;
+  }
+  
+  /**
+   * The backing store for the list.
+   */
+  private int[] array;
+  
+  /**
+   * The size of the list distinct from the length of the array. That is, it is the number of
+   * elements set in the list.
+   */
+  private int size;
+
+  /**
+   * Constructs a new mutable {@code IntArrayList}.
+   */
+  IntArrayList() {
+    array = new int[DEFAULT_CAPACITY];
+    size = 0;
+  }
+
+  /**
+   * Constructs a new mutable {@code IntArrayList} containing the same elements as {@code other}.
+   */
+  IntArrayList(List<Integer> other) {
+    if (other instanceof IntArrayList) {
+      IntArrayList list = (IntArrayList) other;
+      array = list.array.clone();
+      size = list.size;
+    } else {
+      size = other.size();
+      array = new int[size];
+      for (int i = 0; i < size; i++) {
+        array[i] = other.get(i);
+      }
+    }
+  }
+  
+  @Override
+  public Integer get(int index) {
+    return getInt(index);
+  }
+
+  @Override
+  public int getInt(int index) {
+    ensureIndexInRange(index);
+    return array[index];
+  }
+
+  @Override
+  public int size() {
+    return size;
+  }
+
+  @Override
+  public Integer set(int index, Integer element) {
+    return setInt(index, element);
+  }
+
+  @Override
+  public int setInt(int index, int element) {
+    ensureIsMutable();
+    ensureIndexInRange(index);
+    int previousValue = array[index];
+    array[index] = element;
+    return previousValue;
+  }
+
+  @Override
+  public void add(int index, Integer element) {
+    addInt(index, element);
+  }
+
+  /**
+   * Like {@link #add(Integer)} but more efficient in that it doesn't box the element.
+   */
+  @Override
+  public void addInt(int element) {
+    addInt(size, element);
+  }
+
+  /**
+   * Like {@link #add(int, Integer)} but more efficient in that it doesn't box the element.
+   */
+  private void addInt(int index, int element) {
+    ensureIsMutable();
+    if (index < 0 || index > size) {
+      throw new IndexOutOfBoundsException(makeOutOfBoundsExceptionMessage(index));
+    }
+    
+    if (size < array.length) {
+      // Shift everything over to make room
+      System.arraycopy(array, index, array, index + 1, size - index);
+    } else {
+      // Resize to 1.5x the size
+      int length = ((size * 3) / 2) + 1;
+      int[] newArray = new int[length];
+      
+      // Copy the first part directly
+      System.arraycopy(array, 0, newArray, 0, index);
+      
+      // Copy the rest shifted over by one to make room
+      System.arraycopy(array, index, newArray, index + 1, size - index);
+      array = newArray;
+    }
+
+    array[index] = element;
+    size++;
+    modCount++;
+  }
+
+  @Override
+  public boolean addAll(Collection<? extends Integer> collection) {
+    ensureIsMutable();
+    
+    if (collection == null) {
+      throw new NullPointerException();
+    }
+    
+    // We specialize when adding another IntArrayList to avoid boxing elements.
+    if (!(collection instanceof IntArrayList)) {
+      return super.addAll(collection);
+    }
+    
+    IntArrayList list = (IntArrayList) collection;
+    if (list.size == 0) {
+      return false;
+    }
+    
+    int overflow = Integer.MAX_VALUE - size;
+    if (overflow < list.size) {
+      // We can't actually represent a list this large.
+      throw new OutOfMemoryError();
+    }
+    
+    int newSize = size + list.size;
+    if (newSize > array.length) {
+      array = Arrays.copyOf(array, newSize);
+    }
+    
+    System.arraycopy(list.array, 0, array, size, list.size);
+    size = newSize;
+    modCount++;
+    return true;
+  }
+  
+  @Override
+  public boolean remove(Object o) {
+    ensureIsMutable();
+    for (int i = 0; i < size; i++) {
+      if (o.equals(array[i])) {
+        System.arraycopy(array, i + 1, array, i, size - i);
+        size--;
+        modCount++;
+        return true;
+      }
+    }
+    return false;
+  }
+
+  @Override
+  public Integer remove(int index) {
+    ensureIsMutable();
+    ensureIndexInRange(index);
+    int value = array[index];
+    System.arraycopy(array, index + 1, array, index, size - index);
+    size--;
+    modCount++;
+    return value;
+  }
+
+  /**
+   * Ensures that the provided {@code index} is within the range of {@code [0, size]}. Throws an
+   * {@link IndexOutOfBoundsException} if it is not.
+   * 
+   * @param index the index to verify is in range
+   */
+  private void ensureIndexInRange(int index) {
+    if (index < 0 || index >= size) {
+      throw new IndexOutOfBoundsException(makeOutOfBoundsExceptionMessage(index));
+    }
+  }
+
+  private String makeOutOfBoundsExceptionMessage(int index) {
+    return "Index:" + index + ", Size:" + size;
+  }
+}

java/src/main/java/com/google/protobuf/Internal.java

@@ -30,6 +30,7 @@
 
 package com.google.protobuf;
 
+import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.charset.Charset;
 import java.util.AbstractList;
@@ -532,4 +533,132 @@ public class Internal {
       }
     }
   }
+
+  /**
+   * Extends {@link List} to add the capability to make the list immutable and inspect if it is
+   * modifiable.
+   */
+  public static interface ProtobufList<E> extends List<E> {
+
+    /**
+     * Makes this list immutable. All subsequent modifications will throw an
+     * {@link UnsupportedOperationException}.
+     */
+    void makeImmutable();
+
+    /**
+     * Returns whether this list can be modified via the publicly accessible {@link List} methods.
+     */
+    boolean isModifiable();
+  }
+
+  /**
+   * A {@link java.util.List} implementation that avoids boxing the elements into Integers if
+   * possible. Does not support null elements.
+   */
+  public static interface IntList extends ProtobufList<Integer> {
+
+    /**
+     * Like {@link #get(int)} but more efficient in that it doesn't box the returned value.
+     */
+    int getInt(int index);
+
+    /**
+     * Like {@link #add(Integer)} but more efficient in that it doesn't box the element.
+     */
+    void addInt(int element);
+
+    /**
+     * Like {@link #set(int, Integer)} but more efficient in that it doesn't box the element.
+     */
+    int setInt(int index, int element);
+  }
+
+  /**
+   * A {@link java.util.List} implementation that avoids boxing the elements into Booleans if
+   * possible. Does not support null elements.
+   */
+  public static interface BooleanList extends ProtobufList<Boolean> {
+
+    /**
+     * Like {@link #get(int)} but more efficient in that it doesn't box the returned value.
+     */
+    boolean getBoolean(int index);
+
+    /**
+     * Like {@link #add(Boolean)} but more efficient in that it doesn't box the element.
+     */
+    void addBoolean(boolean element);
+
+    /**
+     * Like {@link #set(int, Boolean)} but more efficient in that it doesn't box the element.
+     */
+    boolean setBoolean(int index, boolean element);
+  }
+
+  /**
+   * A {@link java.util.List} implementation that avoids boxing the elements into Longs if
+   * possible. Does not support null elements.
+   */
+  public static interface LongList extends ProtobufList<Long> {
+
+    /**
+     * Like {@link #get(int)} but more efficient in that it doesn't box the returned value.
+     */
+    long getLong(int index);
+
+    /**
+     * Like {@link #add(Long)} but more efficient in that it doesn't box the element.
+     */
+    void addLong(long element);
+
+    /**
+     * Like {@link #set(int, Long)} but more efficient in that it doesn't box the element.
+     */
+    long setLong(int index, long element);
+  }
+
+  /**
+   * A {@link java.util.List} implementation that avoids boxing the elements into Doubles if
+   * possible. Does not support null elements.
+   */
+  public static interface DoubleList extends ProtobufList<Double> {
+
+    /**
+     * Like {@link #get(int)} but more efficient in that it doesn't box the returned value.
+     */
+    double getDouble(int index);
+
+    /**
+     * Like {@link #add(Double)} but more efficient in that it doesn't box the element.
+     */
+    void addDouble(double element);
+
+    /**
+     * Like {@link #set(int, Double)} but more efficient in that it doesn't box the element.
+     */
+    double setDouble(int index, double element);
+  }
+
+  /**
+   * A {@link java.util.List} implementation that avoids boxing the elements into Floats if
+   * possible. Does not support null elements.
+   */
+  public static interface FloatList extends ProtobufList<Float> {
+
+    /**
+     * Like {@link #get(int)} but more efficient in that it doesn't box the returned value.
+     */
+    float getFloat(int index);
+
+    /**
+     * Like {@link #add(Float)} but more efficient in that it doesn't box the element.
+     */
+    void addFloat(float element);
+
+    /**
+     * Like {@link #set(int, Float)} but more efficient in that it doesn't box the element.
+     */
+    float setFloat(int index, float element);
+  }
 }

java/src/main/java/com/google/protobuf/LazyStringArrayList.java

@@ -62,11 +62,20 @@ import java.util.RandomAccess;
  *
  * @author jonp@google.com (Jon Perlow)
  */
-public class LazyStringArrayList extends AbstractList<String>
+public class LazyStringArrayList extends AbstractProtobufList<String>
     implements LazyStringList, RandomAccess {
+  
+  private static final LazyStringArrayList EMPTY_LIST = new LazyStringArrayList();
+  static {
+    EMPTY_LIST.makeImmutable();
+  }
+  
+  static LazyStringArrayList emptyList() {
+    return EMPTY_LIST;
+  }
 
-  public static final LazyStringList EMPTY =
-      new LazyStringArrayList().getUnmodifiableView();
+  // For compatibility with older runtimes.
+  public static final LazyStringList EMPTY = EMPTY_LIST;
 
   private final List<Object> list;
 
@@ -116,12 +125,26 @@ public class LazyStringArrayList extends AbstractList<String>
 
   @Override
   public String set(int index, String s) {
+    ensureIsMutable();
     Object o = list.set(index, s);
     return asString(o);
   }
 
   @Override
   public void add(int index, String element) {
+    ensureIsMutable();
+    list.add(index, element);
+    modCount++;
+  }
+  
+  private void add(int index, ByteString element) {
+    ensureIsMutable();
+    list.add(index, element);
+    modCount++;
+  }
+  
+  private void add(int index, byte[] element) {
+    ensureIsMutable();
     list.add(index, element);
     modCount++;
   }
@@ -137,6 +160,7 @@ public class LazyStringArrayList extends AbstractList<String>
 
   @Override
   public boolean addAll(int index, Collection<? extends String> c) {
+    ensureIsMutable();
     // When copying from another LazyStringList, directly copy the underlying
     // elements rather than forcing each element to be decoded to a String.
     Collection<?> collection = c instanceof LazyStringList
@@ -148,6 +172,7 @@ public class LazyStringArrayList extends AbstractList<String>
 
   // @Override
   public boolean addAllByteString(Collection<? extends ByteString> values) {
+    ensureIsMutable();
     boolean ret = list.addAll(values);
     modCount++;
     return ret;
@@ -155,6 +180,7 @@ public class LazyStringArrayList extends AbstractList<String>
 
   // @Override
   public boolean addAllByteArray(Collection<byte[]> c) {
+    ensureIsMutable();
     boolean ret = list.addAll(c);
     modCount++;
     return ret;
@@ -162,6 +188,7 @@ public class LazyStringArrayList extends AbstractList<String>
 
   @Override
   public String remove(int index) {
+    ensureIsMutable();
     Object o = list.remove(index);
     modCount++;
     return asString(o);
@@ -169,18 +196,21 @@ public class LazyStringArrayList extends AbstractList<String>
 
   @Override
   public void clear() {
+    ensureIsMutable();
     list.clear();
     modCount++;
   }
 
   // @Override
   public void add(ByteString element) {
+    ensureIsMutable();
     list.add(element);
     modCount++;
   }
   
   // @Override
   public void add(byte[] element) {
+    ensureIsMutable();
     list.add(element);
     modCount++;
   }
@@ -207,14 +237,23 @@ public class LazyStringArrayList extends AbstractList<String>
 
   // @Override
   public void set(int index, ByteString s) {
-    list.set(index, s);
+    setAndReturn(index, s);
+  }
+  
+  private Object setAndReturn(int index, ByteString s) {
+    ensureIsMutable();
+    return list.set(index, s);
   }
 
   // @Override
   public void set(int index, byte[] s) {
-    list.set(index, s);
+    setAndReturn(index, s);
+  }
+  
+  private Object setAndReturn(int index, byte[] s) {
+    ensureIsMutable();
+    return list.set(index, s);
   }
-
 
   private static String asString(Object o) {
     if (o instanceof String) {
@@ -253,6 +292,7 @@ public class LazyStringArrayList extends AbstractList<String>
 
   // @Override
   public void mergeFrom(LazyStringList other) {
+    ensureIsMutable();
     for (Object o : other.getUnderlyingElements()) {
       if (o instanceof byte[]) {
         byte[] b = (byte[]) o;
@@ -267,20 +307,15 @@ public class LazyStringArrayList extends AbstractList<String>
 
   private static class ByteArrayListView extends AbstractList<byte[]>
       implements RandomAccess {
-    private final List<Object> list;
+    private final LazyStringArrayList list;
     
-    ByteArrayListView(List<Object> list) {
+    ByteArrayListView(LazyStringArrayList list) {
       this.list = list;
     }
     
     @Override
     public byte[] get(int index) {
-      Object o = list.get(index);
-      byte[] b = asByteArray(o);
-      if (b != o) {
-        list.set(index, b);
-      }
-      return b;
+      return list.getByteArray(index);
     }
 
     @Override
@@ -290,7 +325,7 @@ public class LazyStringArrayList extends AbstractList<String>
 
     @Override
     public byte[] set(int index, byte[] s) {
-      Object o = list.set(index, s);
+      Object o = list.setAndReturn(index, s);
       modCount++;
       return asByteArray(o);
     }
@@ -311,25 +346,20 @@ public class LazyStringArrayList extends AbstractList<String>
   
   // @Override
   public List<byte[]> asByteArrayList() {
-    return new ByteArrayListView(list);
+    return new ByteArrayListView(this);
   }
 
   private static class ByteStringListView extends AbstractList<ByteString>
       implements RandomAccess {
-    private final List<Object> list;
+    private final LazyStringArrayList list;
 
-    ByteStringListView(List<Object> list) {
+    ByteStringListView(LazyStringArrayList list) {
       this.list = list;
     }
 
     @Override
     public ByteString get(int index) {
-      Object o = list.get(index);
-      ByteString b = asByteString(o);
-      if (b != o) {
-        list.set(index, b);
-      }
-      return b;
+      return list.getByteString(index);
     }
 
     @Override
@@ -339,7 +369,7 @@ public class LazyStringArrayList extends AbstractList<String>
 
     @Override
     public ByteString set(int index, ByteString s) {
-      Object o = list.set(index, s);
+      Object o = list.setAndReturn(index, s);
       modCount++;
       return asByteString(o);
     }
@@ -360,12 +390,15 @@ public class LazyStringArrayList extends AbstractList<String>
 
   // @Override
   public List<ByteString> asByteStringList() {
-    return new ByteStringListView(list);
+    return new ByteStringListView(this);
   }
 
   // @Override
   public LazyStringList getUnmodifiableView() {
-    return new UnmodifiableLazyStringList(this);
+    if (isModifiable()) {
+      return new UnmodifiableLazyStringList(this);
+    }
+    return this;
   }
 
 }

java/src/main/java/com/google/protobuf/LongArrayList.java

@@ -0,0 +1,242 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package com.google.protobuf;
+
+import com.google.protobuf.Internal.LongList;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.RandomAccess;
+
+/**
+ * An implementation of {@link LongList} on top of a primitive array.
+ * 
+ * @author dweis@google.com (Daniel Weis)
+ */
+final class LongArrayList extends AbstractProtobufList<Long> implements LongList, RandomAccess {
+  
+  private static final int DEFAULT_CAPACITY = 10;
+  
+  private static final LongArrayList EMPTY_LIST = new LongArrayList();
+  static {
+    EMPTY_LIST.makeImmutable();
+  }
+  
+  public static LongArrayList emptyList() {
+    return EMPTY_LIST;
+  }
+  
+  /**
+   * The backing store for the list.
+   */
+  private long[] array;
+  
+  /**
+   * The size of the list distinct from the length of the array. That is, it is the number of
+   * elements set in the list.
+   */
+  private int size;
+
+  /**
+   * Constructs a new mutable {@code LongArrayList}.
+   */
+  LongArrayList() {
+    array = new long[DEFAULT_CAPACITY];
+    size = 0;
+  }
+
+  /**
+   * Constructs a new mutable {@code LongArrayList} containing the same elements as {@code other}.
+   */
+  LongArrayList(List<Long> other) {
+    if (other instanceof LongArrayList) {
+      LongArrayList list = (LongArrayList) other;
+      array = list.array.clone();
+      size = list.size;
+    } else {
+      size = other.size();
+      array = new long[size];
+      for (int i = 0; i < size; i++) {
+        array[i] = other.get(i);
+      }
+    }
+  }
+  
+  @Override
+  public Long get(int index) {
+    return getLong(index);
+  }
+
+  @Override
+  public long getLong(int index) {
+    ensureIndexInRange(index);
+    return array[index];
+  }
+
+  @Override
+  public int size() {
+    return size;
+  }
+
+  @Override
+  public Long set(int index, Long element) {
+    return setLong(index, element);
+  }
+
+  @Override
+  public long setLong(int index, long element) {
+    ensureIsMutable();
+    ensureIndexInRange(index);
+    long previousValue = array[index];
+    array[index] = element;
+    return previousValue;
+  }
+
+  @Override
+  public void add(int index, Long element) {
+    addLong(index, element);
+  }
+
+  /**
+   * Like {@link #add(Long)} but more efficient in that it doesn't box the element.
+   */
+  @Override
+  public void addLong(long element) {
+    addLong(size, element);
+  }
+
+  /**
+   * Like {@link #add(int, Long)} but more efficient in that it doesn't box the element.
+   */
+  private void addLong(int index, long element) {
+    ensureIsMutable();
+    if (index < 0 || index > size) {
+      throw new IndexOutOfBoundsException(makeOutOfBoundsExceptionMessage(index));
+    }
+    
+    if (size < array.length) {
+      // Shift everything over to make room
+      System.arraycopy(array, index, array, index + 1, size - index);
+    } else {
+      // Resize to 1.5x the size
+      int length = ((size * 3) / 2) + 1;
+      long[] newArray = new long[length];
+      
+      // Copy the first part directly
+      System.arraycopy(array, 0, newArray, 0, index);
+      
+      // Copy the rest shifted over by one to make room
+      System.arraycopy(array, index, newArray, index + 1, size - index);
+      array = newArray;
+    }
+
+    array[index] = element;
+    size++;
+    modCount++;
+  }
+
+  @Override
+  public boolean addAll(Collection<? extends Long> collection) {
+    ensureIsMutable();
+    
+    if (collection == null) {
+      throw new NullPointerException();
+    }
+    
+    // We specialize when adding another LongArrayList to avoid boxing elements.
+    if (!(collection instanceof LongArrayList)) {
+      return super.addAll(collection);
+    }
+    
+    LongArrayList list = (LongArrayList) collection;
+    if (list.size == 0) {
+      return false;
+    }
+    
+    int overflow = Integer.MAX_VALUE - size;
+    if (overflow < list.size) {
+      // We can't actually represent a list this large.
+      throw new OutOfMemoryError();
+    }
+    
+    int newSize = size + list.size;
+    if (newSize > array.length) {
+      array = Arrays.copyOf(array, newSize);
+    }
+    
+    System.arraycopy(list.array, 0, array, size, list.size);
+    size = newSize;
+    modCount++;
+    return true;
+  }
+  
+  @Override
+  public boolean remove(Object o) {
+    ensureIsMutable();
+    for (int i = 0; i < size; i++) {
+      if (o.equals(array[i])) {
+        System.arraycopy(array, i + 1, array, i, size - i);
+        size--;
+        modCount++;
+        return true;
+      }
+    }
+    return false;
+  }
+
+  @Override
+  public Long remove(int index) {
+    ensureIsMutable();
+    ensureIndexInRange(index);
+    long value = array[index];
+    System.arraycopy(array, index + 1, array, index, size - index);
+    size--;
+    modCount++;
+    return value;
+  }
+
+  /**
+   * Ensures that the provided {@code index} is within the range of {@code [0, size]}. Throws an
+   * {@link IndexOutOfBoundsException} if it is not.
+   * 
+   * @param index the index to verify is in range
+   */
+  private void ensureIndexInRange(int index) {
+    if (index < 0 || index >= size) {
+      throw new IndexOutOfBoundsException(makeOutOfBoundsExceptionMessage(index));
+    }
+  }
+
+  private String makeOutOfBoundsExceptionMessage(int index) {
+    return "Index:" + index + ", Size:" + size;
+  }
+}

java/src/main/java/com/google/protobuf/MapField.java

@@ -30,9 +30,11 @@
 
 package com.google.protobuf;
 
+import com.google.protobuf.MapFieldLite.MutatabilityAwareMap;
+
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -51,7 +53,7 @@ import java.util.Map;
  * and getList() concurrently in multiple threads. If write-access is needed,
  * all access must be synchronized.
  */
-public class MapField<K, V> {
+public class MapField<K, V> implements MutabilityOracle {
   /**
    * Indicates where the data of this map field is currently stored.
    * 
@@ -72,8 +74,9 @@ public class MapField<K, V> {
    */
   private enum StorageMode {MAP, LIST, BOTH}
 
+  private volatile boolean isMutable;
   private volatile StorageMode mode;
-  private Map<K, V> mapData;
+  private MutatabilityAwareMap<K, V> mapData;
   private List<Message> listData;
   
   // Convert between a map entry Message and a key-value pair.
@@ -110,20 +113,19 @@ public class MapField<K, V> {
   private MapField(
       Converter<K, V> converter,
       StorageMode mode,
-      Map<K, V> mapData,
-      List<Message> listData) {
+      Map<K, V> mapData) {
     this.converter = converter;
+    this.isMutable = true;
     this.mode = mode;
-    this.mapData = mapData;
-    this.listData = listData;
+    this.mapData = new MutatabilityAwareMap<K, V>(this, mapData);
+    this.listData = null;
   }
     
   private MapField(
       MapEntry<K, V> defaultEntry,
       StorageMode mode,
-      Map<K, V> mapData,
-      List<Message> listData) {
-    this(new ImmutableMessageConverter<K, V>(defaultEntry), mode, mapData, listData);
+      Map<K, V> mapData) {
+    this(new ImmutableMessageConverter<K, V>(defaultEntry), mode, mapData);
   }
   
   
@@ -131,14 +133,14 @@ public class MapField<K, V> {
   public static <K, V> MapField<K, V> emptyMapField(
       MapEntry<K, V> defaultEntry) {
     return new MapField<K, V>(
-        defaultEntry, StorageMode.MAP, Collections.<K, V>emptyMap(), null);
+        defaultEntry, StorageMode.MAP, Collections.<K, V>emptyMap());
   }
   
   
   /** Creates a new mutable empty MapField. */
   public static <K, V> MapField<K, V> newMapField(MapEntry<K, V> defaultEntry) {
     return new MapField<K, V>(
-        defaultEntry, StorageMode.MAP, new HashMap<K, V>(), null);
+        defaultEntry, StorageMode.MAP, new LinkedHashMap<K, V>());
   }
   
   
@@ -151,7 +153,7 @@ public class MapField<K, V> {
     converter.convertMessageToKeyAndValue(message, map);
   }
 
-  private List<Message> convertMapToList(Map<K, V> mapData) {
+  private List<Message> convertMapToList(MutatabilityAwareMap<K, V> mapData) {
     List<Message> listData = new ArrayList<Message>();
     for (Map.Entry<K, V> entry : mapData.entrySet()) {
       listData.add(
@@ -161,12 +163,12 @@ public class MapField<K, V> {
     return listData;
   }
 
-  private Map<K, V> convertListToMap(List<Message> listData) {
-    Map<K, V> mapData = new HashMap<K, V>();
+  private MutatabilityAwareMap<K, V> convertListToMap(List<Message> listData) {
+    Map<K, V> mapData = new LinkedHashMap<K, V>();
     for (Message item : listData) {
       convertMessageToKeyAndValue(item, mapData);
     }
-    return mapData;
+    return new MutatabilityAwareMap<K, V>(this, mapData);
   }
   
   /** Returns the content of this MapField as a read-only Map. */
@@ -199,7 +201,7 @@ public class MapField<K, V> {
   }
   
   public void clear() {
-    mapData = new HashMap<K, V>();
+    mapData = new MutatabilityAwareMap<K, V>(this, new LinkedHashMap<K, V>());
     mode = StorageMode.MAP;
   }
   
@@ -221,7 +223,7 @@ public class MapField<K, V> {
   /** Returns a deep copy of this MapField. */
   public MapField<K, V> copy() {
     return new MapField<K, V>(
-        converter, StorageMode.MAP, MapFieldLite.copy(getMap()), null);
+        converter, StorageMode.MAP, MapFieldLite.copy(getMap()));
   }
   
   /** Gets the content of this MapField as a read-only List. */
@@ -256,4 +258,29 @@ public class MapField<K, V> {
   Message getMapEntryMessageDefaultInstance() {
     return converter.getMessageDefaultInstance();
   }
+  
+  /**
+   * Makes this list immutable. All subsequent modifications will throw an
+   * {@link UnsupportedOperationException}.
+   */
+  public void makeImmutable() {
+    isMutable = false;
+  }
+  
+  /**
+   * Returns whether this field can be modified.
+   */
+  public boolean isMutable() {
+    return isMutable;
+  }
+  
+  /* (non-Javadoc)
+   * @see com.google.protobuf.MutabilityOracle#ensureMutable()
+   */
+  @Override
+  public void ensureMutable() {
+    if (!isMutable()) {
+      throw new UnsupportedOperationException();
+    }
+  }
 }

java/src/main/java/com/google/protobuf/MapFieldLite.java

@@ -31,9 +31,12 @@
 package com.google.protobuf;
 
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
-import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
 import java.util.Map;
+import java.util.Set;
 
 /**
  * Internal representation of map fields in generated lite-runtime messages.
@@ -41,16 +44,21 @@ import java.util.Map;
  * This class is a protobuf implementation detail. Users shouldn't use this
  * class directly.
  */
-public class MapFieldLite<K, V> {
-  private Map<K, V> mapData;
+public class MapFieldLite<K, V> implements MutabilityOracle {
+  private MutatabilityAwareMap<K, V> mapData;
+  private boolean isMutable;
   
   private MapFieldLite(Map<K, V> mapData) {
-    this.mapData = mapData;
+    this.mapData = new MutatabilityAwareMap<K, V>(this, mapData);
+    this.isMutable = true;
   }
   
   @SuppressWarnings({"rawtypes", "unchecked"})
   private static final MapFieldLite EMPTY_MAP_FIELD =
       new MapFieldLite(Collections.emptyMap());
+  static {
+    EMPTY_MAP_FIELD.makeImmutable();
+  }
   
   /** Returns an singleton immutable empty MapFieldLite instance. */
   @SuppressWarnings({"unchecked", "cast"})
@@ -60,7 +68,7 @@ public class MapFieldLite<K, V> {
   
   /** Creates a new MapFieldLite instance. */
   public static <K, V> MapFieldLite<K, V> newMapField() {
-    return new MapFieldLite<K, V>(new HashMap<K, V>());
+    return new MapFieldLite<K, V>(new LinkedHashMap<K, V>());
   }
   
   /** Gets the content of this MapField as a read-only Map. */
@@ -168,7 +176,7 @@ public class MapFieldLite<K, V> {
    */
   @SuppressWarnings("unchecked")
   static <K, V> Map<K, V> copy(Map<K, V> map) {
-    Map<K, V> result = new HashMap<K, V>();
+    Map<K, V> result = new LinkedHashMap<K, V>();
     for (Map.Entry<K, V> entry : map.entrySet()) {
       result.put(entry.getKey(), (V) copy(entry.getValue()));
     }
@@ -179,4 +187,360 @@ public class MapFieldLite<K, V> {
   public MapFieldLite<K, V> copy() {
     return new MapFieldLite<K, V>(copy(mapData));
   }
+  
+  /**
+   * Makes this field immutable. All subsequent modifications will throw an
+   * {@link UnsupportedOperationException}.
+   */
+  public void makeImmutable() {
+    isMutable = false;
+  }
+  
+  /**
+   * Returns whether this field can be modified.
+   */
+  public boolean isMutable() {
+    return isMutable;
+  }
+  
+  @Override
+  public void ensureMutable() {
+    if (!isMutable()) {
+      throw new UnsupportedOperationException();
+    }
+  }
+
+  /**
+   * An internal map that checks for mutability before delegating.
+   */
+  static class MutatabilityAwareMap<K, V> implements Map<K, V> {    
+    private final MutabilityOracle mutabilityOracle;
+    private final Map<K, V> delegate;
+    
+    MutatabilityAwareMap(MutabilityOracle mutabilityOracle, Map<K, V> delegate) {
+      this.mutabilityOracle = mutabilityOracle;
+      this.delegate = delegate;
+    }
+
+    @Override
+    public int size() {
+      return delegate.size();
+    }
+
+    @Override
+    public boolean isEmpty() {
+      return delegate.isEmpty();
+    }
+
+    @Override
+    public boolean containsKey(Object key) {
+      return delegate.containsKey(key);
+    }
+
+    @Override
+    public boolean containsValue(Object value) {
+      return delegate.containsValue(value);
+    }
+
+    @Override
+    public V get(Object key) {
+      return delegate.get(key);
+    }
+
+    @Override
+    public V put(K key, V value) {
+      mutabilityOracle.ensureMutable();
+      return delegate.put(key, value);
+    }
+
+    @Override
+    public V remove(Object key) {
+      mutabilityOracle.ensureMutable();
+      return delegate.remove(key);
+    }
+
+    @Override
+    public void putAll(Map<? extends K, ? extends V> m) {
+      mutabilityOracle.ensureMutable();
+      delegate.putAll(m);
+    }
+
+    @Override
+    public void clear() {
+      mutabilityOracle.ensureMutable();
+      delegate.clear();
+    }
+
+    @Override
+    public Set<K> keySet() {
+      return new MutatabilityAwareSet<K>(mutabilityOracle, delegate.keySet());
+    }
+
+    @Override
+    public Collection<V> values() {
+      return new MutatabilityAwareCollection<V>(mutabilityOracle, delegate.values());
+    }
+
+    @Override
+    public Set<java.util.Map.Entry<K, V>> entrySet() {
+      return new MutatabilityAwareSet<Entry<K, V>>(mutabilityOracle, delegate.entrySet());
+    }
+
+    @Override
+    public boolean equals(Object o) {
+      return delegate.equals(o);
+    }
+
+    @Override
+    public int hashCode() {
+      return delegate.hashCode();
+    }
+
+    @Override
+    public String toString() {
+      return delegate.toString();
+    }
+  }
+
+  /**
+   * An internal collection that checks for mutability before delegating.
+   */
+  private static class MutatabilityAwareCollection<E> implements Collection<E> {
+    private final MutabilityOracle mutabilityOracle;
+    private final Collection<E> delegate;
+    
+    MutatabilityAwareCollection(MutabilityOracle mutabilityOracle, Collection<E> delegate) {
+      this.mutabilityOracle = mutabilityOracle;
+      this.delegate = delegate;
+    }
+
+    @Override
+    public int size() {
+      return delegate.size();
+    }
+
+    @Override
+    public boolean isEmpty() {
+      return delegate.isEmpty();
+    }
+
+    @Override
+    public boolean contains(Object o) {
+      return delegate.contains(o);
+    }
+
+    @Override
+    public Iterator<E> iterator() {
+      return new MutatabilityAwareIterator<E>(mutabilityOracle, delegate.iterator());
+    }
+
+    @Override
+    public Object[] toArray() {
+      return delegate.toArray();
+    }
+
+    @Override
+    public <T> T[] toArray(T[] a) {
+      return delegate.toArray(a);
+    }
+
+    @Override
+    public boolean add(E e) {
+      // Unsupported operation in the delegate.
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public boolean remove(Object o) {
+      mutabilityOracle.ensureMutable();
+      return delegate.remove(o);
+    }
+
+    @Override
+    public boolean containsAll(Collection<?> c) {
+      return delegate.containsAll(c);
+    }
+
+    @Override
+    public boolean addAll(Collection<? extends E> c) {
+      // Unsupported operation in the delegate.
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public boolean removeAll(Collection<?> c) {
+      mutabilityOracle.ensureMutable();
+      return delegate.removeAll(c);
+    }
+
+    @Override
+    public boolean retainAll(Collection<?> c) {
+      mutabilityOracle.ensureMutable();
+      return delegate.retainAll(c);
+    }
+
+    @Override
+    public void clear() {
+      mutabilityOracle.ensureMutable();
+      delegate.clear();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+      return delegate.equals(o);
+    }
+
+    @Override
+    public int hashCode() {
+      return delegate.hashCode();
+    }
+    
+    @Override
+    public String toString() {
+      return delegate.toString();
+    }
+  }
+
+  /**
+   * An internal set that checks for mutability before delegating.
+   */
+  private static class MutatabilityAwareSet<E> implements Set<E> {
+    private final MutabilityOracle mutabilityOracle;
+    private final Set<E> delegate;
+    
+    MutatabilityAwareSet(MutabilityOracle mutabilityOracle, Set<E> delegate) {
+      this.mutabilityOracle = mutabilityOracle;
+      this.delegate = delegate;
+    }
+
+    @Override
+    public int size() {
+      return delegate.size();
+    }
+
+    @Override
+    public boolean isEmpty() {
+      return delegate.isEmpty();
+    }
+
+    @Override
+    public boolean contains(Object o) {
+      return delegate.contains(o);
+    }
+
+    @Override
+    public Iterator<E> iterator() {
+      return new MutatabilityAwareIterator<E>(mutabilityOracle, delegate.iterator());
+    }
+
+    @Override
+    public Object[] toArray() {
+      return delegate.toArray();
+    }
+
+    @Override
+    public <T> T[] toArray(T[] a) {
+      return delegate.toArray(a);
+    }
+
+    @Override
+    public boolean add(E e) {
+      mutabilityOracle.ensureMutable();
+      return delegate.add(e);
+    }
+
+    @Override
+    public boolean remove(Object o) {
+      mutabilityOracle.ensureMutable();
+      return delegate.remove(o);
+    }
+
+    @Override
+    public boolean containsAll(Collection<?> c) {
+      return delegate.containsAll(c);
+    }
+
+    @Override
+    public boolean addAll(Collection<? extends E> c) {
+      mutabilityOracle.ensureMutable();
+      return delegate.addAll(c);
+    }
+
+    @Override
+    public boolean retainAll(Collection<?> c) {
+      mutabilityOracle.ensureMutable();
+      return delegate.retainAll(c);
+    }
+
+    @Override
+    public boolean removeAll(Collection<?> c) {
+      mutabilityOracle.ensureMutable();
+      return delegate.removeAll(c);
+    }
+
+    @Override
+    public void clear() {
+      mutabilityOracle.ensureMutable();
+      delegate.clear();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+      return delegate.equals(o);
+    }
+
+    @Override
+    public int hashCode() {
+      return delegate.hashCode();
+    }
+
+    @Override
+    public String toString() {
+      return delegate.toString();
+    }
+  }
+  
+  /**
+   * An internal iterator that checks for mutability before delegating.
+   */
+  private static class MutatabilityAwareIterator<E> implements Iterator<E> {
+    private final MutabilityOracle mutabilityOracle;
+    private final Iterator<E> delegate;
+    
+    MutatabilityAwareIterator(MutabilityOracle mutabilityOracle, Iterator<E> delegate) {
+      this.mutabilityOracle = mutabilityOracle;
+      this.delegate = delegate;
+    }
+
+    @Override
+    public boolean hasNext() {
+      return delegate.hasNext();
+    }
+
+    @Override
+    public E next() {
+      return delegate.next();
+    }
+
+    @Override
+    public void remove() {
+      mutabilityOracle.ensureMutable();
+      delegate.remove();
+    }
+    
+    @Override
+    public boolean equals(Object obj) {
+      return delegate.equals(obj);
+    }
+    
+    @Override
+    public int hashCode() {
+      return delegate.hashCode();
+    }
+
+    @Override
+    public String toString() {
+      return delegate.toString();
+    }
+  }
 }

java/src/main/java/com/google/protobuf/MutabilityOracle.java

@@ -0,0 +1,48 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package com.google.protobuf;
+
+/**
+ * Verifies that an object is mutable, throwing if not.
+ */
+interface MutabilityOracle {
+  static final MutabilityOracle IMMUTABLE = new MutabilityOracle() {
+    @Override
+    public void ensureMutable() {
+      throw new UnsupportedOperationException();
+    }
+  };
+
+  /**
+   * Throws an {@link UnsupportedOperationException} if not mutable.
+   */
+  void ensureMutable();
+}

java/src/main/java/com/google/protobuf/ProtobufArrayList.java

@@ -0,0 +1,95 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package com.google.protobuf;
+
+import com.google.protobuf.Internal.ProtobufList;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Implements {@link ProtobufList} for non-primitive and {@link String} types.
+ */
+class ProtobufArrayList<E> extends AbstractProtobufList<E> {
+
+  private static final ProtobufArrayList<Object> EMPTY_LIST = new ProtobufArrayList<Object>();
+  static {
+    EMPTY_LIST.makeImmutable();
+  }
+  
+  @SuppressWarnings("unchecked") // Guaranteed safe by runtime.
+  public static <E> ProtobufArrayList<E> emptyList() {
+    return (ProtobufArrayList<E>) EMPTY_LIST;
+  }
+  
+  private final List<E> list;
+  
+  ProtobufArrayList() {
+    list = new ArrayList<E>();
+  }
+  
+  ProtobufArrayList(List<E> toCopy) {
+    list = new ArrayList<E>(toCopy);
+  }
+  
+  @Override
+  public void add(int index, E element) {
+    ensureIsMutable();
+    list.add(index, element);
+    modCount++;
+  }
+
+  @Override
+  public E get(int index) {
+    return list.get(index);
+  }
+  
+  @Override
+  public E remove(int index) {
+    ensureIsMutable();
+    E toReturn = list.remove(index);
+    modCount++;
+    return toReturn;
+  }
+  
+  @Override
+  public E set(int index, E element) {
+    ensureIsMutable();
+    E toReturn = list.set(index, element);
+    modCount++;
+    return toReturn;
+  }
+
+  @Override
+  public int size() {
+    return list.size();
+  }
+}

java/src/main/java/com/google/protobuf/TextFormat.java

@@ -1725,7 +1725,7 @@ public final class TextFormat {
    * {@link #escapeBytes(ByteString)}.  Two-digit hex escapes (starting with
    * "\x") are also recognized.
    */
-  static ByteString unescapeBytes(final CharSequence charString)
+  public static ByteString unescapeBytes(final CharSequence charString)
       throws InvalidEscapeSequenceException {
     // First convert the Java character sequence to UTF-8 bytes.
     ByteString input = ByteString.copyFromUtf8(charString.toString());
@@ -1808,7 +1808,7 @@ public final class TextFormat {
    * Thrown by {@link TextFormat#unescapeBytes} and
    * {@link TextFormat#unescapeText} when an invalid escape sequence is seen.
    */
-  static class InvalidEscapeSequenceException extends IOException {
+  public static class InvalidEscapeSequenceException extends IOException {
     private static final long serialVersionUID = -8164033650142593304L;
 
     InvalidEscapeSequenceException(final String description) {

java/src/test/java/com/google/protobuf/BooleanArrayListTest.java

@@ -0,0 +1,473 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package com.google.protobuf;
+
+import static java.util.Arrays.asList;
+
+import junit.framework.TestCase;
+
+import java.util.Collections;
+import java.util.ConcurrentModificationException;
+import java.util.Iterator;
+
+/**
+ * Tests for {@link BooleanArrayList}.
+ * 
+ * @author dweis@google.com (Daniel Weis)
+ */
+public class BooleanArrayListTest extends TestCase {
+  
+  private static final BooleanArrayList UNARY_LIST = newImmutableBooleanArrayList(true);
+  private static final BooleanArrayList TERTIARY_LIST =
+      newImmutableBooleanArrayList(true, true, false);
+  
+  private BooleanArrayList list;
+  
+  @Override
+  protected void setUp() throws Exception {
+    list = new BooleanArrayList();
+  }
+  
+  public void testEmptyListReturnsSameInstance() {
+    assertSame(BooleanArrayList.emptyList(), BooleanArrayList.emptyList());
+  }
+  
+  public void testEmptyListIsImmutable() {
+    assertImmutable(BooleanArrayList.emptyList());
+  }
+  
+  public void testMakeImmutable() {
+    list.addBoolean(true);
+    list.addBoolean(false);
+    list.addBoolean(true);
+    list.addBoolean(true);
+    list.makeImmutable();
+    assertImmutable(list);
+  }
+  
+  public void testCopyConstructor() {
+    BooleanArrayList copy = new BooleanArrayList(TERTIARY_LIST);
+    assertEquals(TERTIARY_LIST, copy);
+
+    copy = new BooleanArrayList(BooleanArrayList.emptyList());
+    assertEquals(BooleanArrayList.emptyList(), copy);
+    
+    copy = new BooleanArrayList(asList(false, false, true));
+    assertEquals(asList(false, false, true), copy);
+
+    copy = new BooleanArrayList(Collections.<Boolean>emptyList());
+    assertEquals(BooleanArrayList.emptyList(), copy);
+  }
+  
+  public void testModificationWithIteration() {
+    list.addAll(asList(true, false, false, true));
+    Iterator<Boolean> iterator = list.iterator();
+    assertEquals(4, list.size());
+    assertEquals(true, (boolean) list.get(0));
+    assertEquals(true, (boolean) iterator.next());
+    list.set(0, true);
+    assertEquals(false, (boolean) iterator.next());
+    
+    list.remove(0);
+    try {
+      iterator.next();
+      fail();
+    } catch (ConcurrentModificationException e) {
+      // expected
+    }
+    
+    iterator = list.iterator();
+    list.add(0, false);
+    try {
+      iterator.next();
+      fail();
+    } catch (ConcurrentModificationException e) {
+      // expected
+    }
+  }
+  
+  public void testGet() {
+    assertEquals(true, (boolean) TERTIARY_LIST.get(0));
+    assertEquals(true, (boolean) TERTIARY_LIST.get(1));
+    assertEquals(false, (boolean) TERTIARY_LIST.get(2));
+    
+    try {
+      TERTIARY_LIST.get(-1);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+    
+    try {
+      TERTIARY_LIST.get(3);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+  }
+  
+  public void testGetInt() {
+    assertEquals(true, TERTIARY_LIST.getBoolean(0));
+    assertEquals(true, TERTIARY_LIST.getBoolean(1));
+    assertEquals(false, TERTIARY_LIST.getBoolean(2));
+    
+    try {
+      TERTIARY_LIST.get(-1);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+    
+    try {
+      TERTIARY_LIST.get(3);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+  }
+  
+  public void testSize() {
+    assertEquals(0, BooleanArrayList.emptyList().size());
+    assertEquals(1, UNARY_LIST.size());
+    assertEquals(3, TERTIARY_LIST.size());
+
+    list.addBoolean(true);
+    list.addBoolean(false);
+    list.addBoolean(false);
+    list.addBoolean(false);
+    assertEquals(4, list.size());
+    
+    list.remove(0);
+    assertEquals(3, list.size());
+    
+    list.add(true);
+    assertEquals(4, list.size());
+  }
+  
+  public void testSet() {
+    list.addBoolean(false);
+    list.addBoolean(false);
+    
+    assertEquals(false, (boolean) list.set(0, true));
+    assertEquals(true, list.getBoolean(0));
+
+    assertEquals(false, (boolean) list.set(1, false));
+    assertEquals(false, list.getBoolean(1));
+    
+    try {
+      list.set(-1, true);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+
+    try {
+      list.set(2, false);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+  }
+  
+  public void testSetInt() {
+    list.addBoolean(true);
+    list.addBoolean(true);
+    
+    assertEquals(true, list.setBoolean(0, false));
+    assertEquals(false, list.getBoolean(0));
+
+    assertEquals(true, list.setBoolean(1, false));
+    assertEquals(false, list.getBoolean(1));
+    
+    try {
+      list.setBoolean(-1, false);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+
+    try {
+      list.setBoolean(2, true);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+  }
+  
+  public void testAdd() {
+    assertEquals(0, list.size());
+
+    assertTrue(list.add(true));
+    assertEquals(asList(true), list);
+
+    assertTrue(list.add(false));
+    list.add(0, false);
+    assertEquals(asList(false, true, false), list);
+    
+    list.add(0, false);
+    list.add(0, true);
+    // Force a resize by getting up to 11 elements.
+    for (int i = 0; i < 6; i++) {
+      list.add(true);
+    }
+    assertEquals(asList(true, false, false, true, false, true, true, true, true, true, true), list);
+    
+    try {
+      list.add(-1, false);
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+    
+    try {
+      list.add(4, true);
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+  }
+  
+  public void testAddInt() {
+    assertEquals(0, list.size());
+
+    list.addBoolean(true);
+    assertEquals(asList(true), list);
+
+    list.addBoolean(false);
+    assertEquals(asList(true, false), list);
+  }
+  
+  public void testAddAll() {
+    assertEquals(0, list.size());
+
+    assertTrue(list.addAll(Collections.singleton(false)));
+    assertEquals(1, list.size());
+    assertEquals(false, (boolean) list.get(0));
+    assertEquals(false, list.getBoolean(0));
+    
+    assertTrue(list.addAll(asList(true, false, false, false, true)));
+    assertEquals(asList(false, true, false, false, false, true), list);
+    
+    assertTrue(list.addAll(TERTIARY_LIST));
+    assertEquals(asList(false, true, false, false, false, true, true, true, false), list);
+
+    assertFalse(list.addAll(Collections.<Boolean>emptyList()));
+    assertFalse(list.addAll(BooleanArrayList.emptyList()));
+  }
+  
+  public void testRemove() {
+    list.addAll(TERTIARY_LIST);
+    assertEquals(true, (boolean) list.remove(0));
+    assertEquals(asList(true, false), list);
+
+    assertTrue(list.remove(Boolean.TRUE));
+    assertEquals(asList(false), list);
+
+    assertFalse(list.remove(Boolean.TRUE));
+    assertEquals(asList(false), list);
+
+    assertEquals(false, (boolean) list.remove(0));
+    assertEquals(asList(), list);
+    
+    try {
+      list.remove(-1);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+    
+    try {
+      list.remove(0);
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+  }
+  
+  private void assertImmutable(BooleanArrayList list) {
+    if (list.contains(1)) {
+      throw new RuntimeException("Cannot test the immutability of lists that contain 1.");
+    }
+    
+    try {
+      list.add(false);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.add(0, true);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(Collections.<Boolean>emptyList());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(Collections.singletonList(false));
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(new BooleanArrayList());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(UNARY_LIST);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(0, Collections.singleton(true));
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(0, UNARY_LIST);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(0, Collections.<Boolean>emptyList());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addBoolean(true);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.clear();
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+
+    try {
+      list.remove(1);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.remove(new Object());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.removeAll(Collections.<Boolean>emptyList());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.removeAll(Collections.singleton(1));
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.removeAll(UNARY_LIST);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.retainAll(Collections.<Boolean>emptyList());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.retainAll(Collections.singleton(1));
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.retainAll(UNARY_LIST);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.set(0, true);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.setBoolean(0, false);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+  }
+  
+  private static BooleanArrayList newImmutableBooleanArrayList(boolean... elements) {
+    BooleanArrayList list = new BooleanArrayList();
+    for (boolean element : elements) {
+      list.addBoolean(element);
+    }
+    list.makeImmutable();
+    return list;
+  }
+}

java/src/test/java/com/google/protobuf/DescriptorsTest.java

@@ -56,6 +56,7 @@ import protobuf_unittest.UnittestProto.TestAllTypes;
 import protobuf_unittest.UnittestProto.TestExtremeDefaultValues;
 import protobuf_unittest.UnittestProto.TestMultipleExtensionRanges;
 import protobuf_unittest.UnittestProto.TestRequired;
+import protobuf_unittest.UnittestProto.TestReservedFields;
 import protobuf_unittest.UnittestProto.TestService;
 
 import junit.framework.TestCase;
@@ -687,6 +688,9 @@ public class DescriptorsTest extends TestCase {
 
     assertEquals(4, oneofDescriptor.getFieldCount());
     assertSame(oneofDescriptor.getField(1), field);
+
+    assertEquals(4, oneofDescriptor.getFields().size());
+    assertEquals(oneofDescriptor.getFields().get(1), field);
   }
 
   public void testMessageDescriptorExtensions() throws Exception {
@@ -702,6 +706,19 @@ public class DescriptorsTest extends TestCase {
     assertTrue(TestMultipleExtensionRanges.getDescriptor().isExtensionNumber(4143));
   }
 
+  public void testReservedFields() {
+    Descriptor d = TestReservedFields.getDescriptor();
+    assertTrue(d.isReservedNumber(2));
+    assertFalse(d.isReservedNumber(8));
+    assertTrue(d.isReservedNumber(9));
+    assertTrue(d.isReservedNumber(10));
+    assertTrue(d.isReservedNumber(11));
+    assertFalse(d.isReservedNumber(12));
+    assertFalse(d.isReservedName("foo"));
+    assertTrue(d.isReservedName("bar"));
+    assertTrue(d.isReservedName("baz"));
+  }
+
   public void testToString() {
     assertEquals("protobuf_unittest.TestAllTypes.optional_uint64",
         UnittestProto.TestAllTypes.getDescriptor().findFieldByNumber(

java/src/test/java/com/google/protobuf/DoubleArrayListTest.java

@@ -0,0 +1,473 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package com.google.protobuf;
+
+import static java.util.Arrays.asList;
+
+import junit.framework.TestCase;
+
+import java.util.Collections;
+import java.util.ConcurrentModificationException;
+import java.util.Iterator;
+
+/**
+ * Tests for {@link DoubleArrayList}.
+ * 
+ * @author dweis@google.com (Daniel Weis)
+ */
+public class DoubleArrayListTest extends TestCase {
+  
+  private static final DoubleArrayList UNARY_LIST = newImmutableDoubleArrayList(1);
+  private static final DoubleArrayList TERTIARY_LIST =
+      newImmutableDoubleArrayList(1, 2, 3);
+  
+  private DoubleArrayList list;
+  
+  @Override
+  protected void setUp() throws Exception {
+    list = new DoubleArrayList();
+  }
+  
+  public void testEmptyListReturnsSameInstance() {
+    assertSame(DoubleArrayList.emptyList(), DoubleArrayList.emptyList());
+  }
+  
+  public void testEmptyListIsImmutable() {
+    assertImmutable(DoubleArrayList.emptyList());
+  }
+  
+  public void testMakeImmutable() {
+    list.addDouble(2);
+    list.addDouble(4);
+    list.addDouble(6);
+    list.addDouble(8);
+    list.makeImmutable();
+    assertImmutable(list);
+  }
+  
+  public void testCopyConstructor() {
+    DoubleArrayList copy = new DoubleArrayList(TERTIARY_LIST);
+    assertEquals(TERTIARY_LIST, copy);
+
+    copy = new DoubleArrayList(DoubleArrayList.emptyList());
+    assertEquals(DoubleArrayList.emptyList(), copy);
+    
+    copy = new DoubleArrayList(asList(1D, 2D, 3D));
+    assertEquals(asList(1D, 2D, 3D), copy);
+
+    copy = new DoubleArrayList(Collections.<Double>emptyList());
+    assertEquals(DoubleArrayList.emptyList(), copy);
+  }
+  
+  public void testModificationWithIteration() {
+    list.addAll(asList(1D, 2D, 3D, 4D));
+    Iterator<Double> iterator = list.iterator();
+    assertEquals(4, list.size());
+    assertEquals(1D, (double) list.get(0));
+    assertEquals(1D, (double) iterator.next());
+    list.set(0, 1D);
+    assertEquals(2D, (double) iterator.next());
+    
+    list.remove(0);
+    try {
+      iterator.next();
+      fail();
+    } catch (ConcurrentModificationException e) {
+      // expected
+    }
+    
+    iterator = list.iterator();
+    list.add(0, 0D);
+    try {
+      iterator.next();
+      fail();
+    } catch (ConcurrentModificationException e) {
+      // expected
+    }
+  }
+  
+  public void testGet() {
+    assertEquals(1D, (double) TERTIARY_LIST.get(0));
+    assertEquals(2D, (double) TERTIARY_LIST.get(1));
+    assertEquals(3D, (double) TERTIARY_LIST.get(2));
+    
+    try {
+      TERTIARY_LIST.get(-1);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+    
+    try {
+      TERTIARY_LIST.get(3);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+  }
+  
+  public void testGetInt() {
+    assertEquals(1D, TERTIARY_LIST.getDouble(0));
+    assertEquals(2D, TERTIARY_LIST.getDouble(1));
+    assertEquals(3D, TERTIARY_LIST.getDouble(2));
+    
+    try {
+      TERTIARY_LIST.get(-1);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+    
+    try {
+      TERTIARY_LIST.get(3);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+  }
+  
+  public void testSize() {
+    assertEquals(0, DoubleArrayList.emptyList().size());
+    assertEquals(1, UNARY_LIST.size());
+    assertEquals(3, TERTIARY_LIST.size());
+
+    list.addDouble(2);
+    list.addDouble(4);
+    list.addDouble(6);
+    list.addDouble(8);
+    assertEquals(4, list.size());
+    
+    list.remove(0);
+    assertEquals(3, list.size());
+    
+    list.add(16D);
+    assertEquals(4, list.size());
+  }
+  
+  public void testSet() {
+    list.addDouble(2);
+    list.addDouble(4);
+    
+    assertEquals(2D, (double) list.set(0, 0D));
+    assertEquals(0D, list.getDouble(0));
+
+    assertEquals(4D, (double) list.set(1, 0D));
+    assertEquals(0D, list.getDouble(1));
+    
+    try {
+      list.set(-1, 0D);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+
+    try {
+      list.set(2, 0D);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+  }
+  
+  public void testSetInt() {
+    list.addDouble(2);
+    list.addDouble(4);
+    
+    assertEquals(2D, list.setDouble(0, 0));
+    assertEquals(0D, list.getDouble(0));
+
+    assertEquals(4D, list.setDouble(1, 0));
+    assertEquals(0D, list.getDouble(1));
+    
+    try {
+      list.setDouble(-1, 0);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+
+    try {
+      list.setDouble(2, 0);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+  }
+  
+  public void testAdd() {
+    assertEquals(0, list.size());
+
+    assertTrue(list.add(2D));
+    assertEquals(asList(2D), list);
+
+    assertTrue(list.add(3D));
+    list.add(0, 4D);
+    assertEquals(asList(4D, 2D, 3D), list);
+    
+    list.add(0, 1D);
+    list.add(0, 0D);
+    // Force a resize by getting up to 11 elements.
+    for (int i = 0; i < 6; i++) {
+      list.add(Double.valueOf(5 + i));
+    }
+    assertEquals(asList(0D, 1D, 4D, 2D, 3D, 5D, 6D, 7D, 8D, 9D, 10D), list);
+    
+    try {
+      list.add(-1, 5D);
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+    
+    try {
+      list.add(4, 5D);
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+  }
+  
+  public void testAddInt() {
+    assertEquals(0, list.size());
+
+    list.addDouble(2);
+    assertEquals(asList(2D), list);
+
+    list.addDouble(3);
+    assertEquals(asList(2D, 3D), list);
+  }
+  
+  public void testAddAll() {
+    assertEquals(0, list.size());
+
+    assertTrue(list.addAll(Collections.singleton(1D)));
+    assertEquals(1, list.size());
+    assertEquals(1D, (double) list.get(0));
+    assertEquals(1D, list.getDouble(0));
+    
+    assertTrue(list.addAll(asList(2D, 3D, 4D, 5D, 6D)));
+    assertEquals(asList(1D, 2D, 3D, 4D, 5D, 6D), list);
+    
+    assertTrue(list.addAll(TERTIARY_LIST));
+    assertEquals(asList(1D, 2D, 3D, 4D, 5D, 6D, 1D, 2D, 3D), list);
+
+    assertFalse(list.addAll(Collections.<Double>emptyList()));
+    assertFalse(list.addAll(DoubleArrayList.emptyList()));
+  }
+  
+  public void testRemove() {
+    list.addAll(TERTIARY_LIST);
+    assertEquals(1D, (double) list.remove(0));
+    assertEquals(asList(2D, 3D), list);
+
+    assertTrue(list.remove(Double.valueOf(3)));
+    assertEquals(asList(2D), list);
+
+    assertFalse(list.remove(Double.valueOf(3)));
+    assertEquals(asList(2D), list);
+
+    assertEquals(2D, (double) list.remove(0));
+    assertEquals(asList(), list);
+    
+    try {
+      list.remove(-1);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+    
+    try {
+      list.remove(0);
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+  }
+  
+  private void assertImmutable(DoubleArrayList list) {
+    if (list.contains(1)) {
+      throw new RuntimeException("Cannot test the immutability of lists that contain 1.");
+    }
+    
+    try {
+      list.add(1D);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.add(0, 1D);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(Collections.<Double>emptyList());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(Collections.singletonList(1D));
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(new DoubleArrayList());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(UNARY_LIST);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(0, Collections.singleton(1D));
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(0, UNARY_LIST);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(0, Collections.<Double>emptyList());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addDouble(0);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.clear();
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+
+    try {
+      list.remove(1);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.remove(new Object());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.removeAll(Collections.<Double>emptyList());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.removeAll(Collections.singleton(1));
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.removeAll(UNARY_LIST);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+
+    try {
+      list.retainAll(Collections.<Double>emptyList());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.retainAll(Collections.singleton(1));
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.retainAll(UNARY_LIST);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.set(0, 0D);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.setDouble(0, 0);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+  }
+  
+  private static DoubleArrayList newImmutableDoubleArrayList(double... elements) {
+    DoubleArrayList list = new DoubleArrayList();
+    for (double element : elements) {
+      list.addDouble(element);
+    }
+    list.makeImmutable();
+    return list;
+  }
+}

java/src/test/java/com/google/protobuf/FloatArrayListTest.java

@@ -0,0 +1,473 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package com.google.protobuf;
+
+import static java.util.Arrays.asList;
+
+import junit.framework.TestCase;
+
+import java.util.Collections;
+import java.util.ConcurrentModificationException;
+import java.util.Iterator;
+
+/**
+ * Tests for {@link FloatArrayList}.
+ * 
+ * @author dweis@google.com (Daniel Weis)
+ */
+public class FloatArrayListTest extends TestCase {
+  
+  private static final FloatArrayList UNARY_LIST = newImmutableFloatArrayList(1);
+  private static final FloatArrayList TERTIARY_LIST =
+      newImmutableFloatArrayList(1, 2, 3);
+  
+  private FloatArrayList list;
+  
+  @Override
+  protected void setUp() throws Exception {
+    list = new FloatArrayList();
+  }
+  
+  public void testEmptyListReturnsSameInstance() {
+    assertSame(FloatArrayList.emptyList(), FloatArrayList.emptyList());
+  }
+  
+  public void testEmptyListIsImmutable() {
+    assertImmutable(FloatArrayList.emptyList());
+  }
+  
+  public void testMakeImmutable() {
+    list.addFloat(2);
+    list.addFloat(4);
+    list.addFloat(6);
+    list.addFloat(8);
+    list.makeImmutable();
+    assertImmutable(list);
+  }
+  
+  public void testCopyConstructor() {
+    FloatArrayList copy = new FloatArrayList(TERTIARY_LIST);
+    assertEquals(TERTIARY_LIST, copy);
+
+    copy = new FloatArrayList(FloatArrayList.emptyList());
+    assertEquals(FloatArrayList.emptyList(), copy);
+    
+    copy = new FloatArrayList(asList(1F, 2F, 3F));
+    assertEquals(asList(1F, 2F, 3F), copy);
+
+    copy = new FloatArrayList(Collections.<Float>emptyList());
+    assertEquals(FloatArrayList.emptyList(), copy);
+  }
+  
+  public void testModificationWithIteration() {
+    list.addAll(asList(1F, 2F, 3F, 4F));
+    Iterator<Float> iterator = list.iterator();
+    assertEquals(4, list.size());
+    assertEquals(1F, (float) list.get(0));
+    assertEquals(1F, (float) iterator.next());
+    list.set(0, 1F);
+    assertEquals(2F, (float) iterator.next());
+    
+    list.remove(0);
+    try {
+      iterator.next();
+      fail();
+    } catch (ConcurrentModificationException e) {
+      // expected
+    }
+    
+    iterator = list.iterator();
+    list.add(0, 0F);
+    try {
+      iterator.next();
+      fail();
+    } catch (ConcurrentModificationException e) {
+      // expected
+    }
+  }
+  
+  public void testGet() {
+    assertEquals(1F, (float) TERTIARY_LIST.get(0));
+    assertEquals(2F, (float) TERTIARY_LIST.get(1));
+    assertEquals(3F, (float) TERTIARY_LIST.get(2));
+    
+    try {
+      TERTIARY_LIST.get(-1);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+    
+    try {
+      TERTIARY_LIST.get(3);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+  }
+  
+  public void testGetFloat() {
+    assertEquals(1F, TERTIARY_LIST.getFloat(0));
+    assertEquals(2F, TERTIARY_LIST.getFloat(1));
+    assertEquals(3F, TERTIARY_LIST.getFloat(2));
+    
+    try {
+      TERTIARY_LIST.get(-1);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+    
+    try {
+      TERTIARY_LIST.get(3);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+  }
+  
+  public void testSize() {
+    assertEquals(0, FloatArrayList.emptyList().size());
+    assertEquals(1, UNARY_LIST.size());
+    assertEquals(3, TERTIARY_LIST.size());
+
+    list.addFloat(2);
+    list.addFloat(4);
+    list.addFloat(6);
+    list.addFloat(8);
+    assertEquals(4, list.size());
+    
+    list.remove(0);
+    assertEquals(3, list.size());
+    
+    list.add(16F);
+    assertEquals(4, list.size());
+  }
+  
+  public void testSet() {
+    list.addFloat(2);
+    list.addFloat(4);
+    
+    assertEquals(2F, (float) list.set(0, 0F));
+    assertEquals(0F, list.getFloat(0));
+
+    assertEquals(4F, (float) list.set(1, 0F));
+    assertEquals(0F, list.getFloat(1));
+    
+    try {
+      list.set(-1, 0F);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+
+    try {
+      list.set(2, 0F);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+  }
+  
+  public void testSetFloat() {
+    list.addFloat(2);
+    list.addFloat(4);
+    
+    assertEquals(2F, list.setFloat(0, 0));
+    assertEquals(0F, list.getFloat(0));
+
+    assertEquals(4F, list.setFloat(1, 0));
+    assertEquals(0F, list.getFloat(1));
+    
+    try {
+      list.setFloat(-1, 0);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+
+    try {
+      list.setFloat(2, 0);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+  }
+  
+  public void testAdd() {
+    assertEquals(0, list.size());
+
+    assertTrue(list.add(2F));
+    assertEquals(asList(2F), list);
+
+    assertTrue(list.add(3F));
+    list.add(0, 4F);
+    assertEquals(asList(4F, 2F, 3F), list);
+    
+    list.add(0, 1F);
+    list.add(0, 0F);
+    // Force a resize by getting up to 11 elements.
+    for (int i = 0; i < 6; i++) {
+      list.add(Float.valueOf(5 + i));
+    }
+    assertEquals(asList(0F, 1F, 4F, 2F, 3F, 5F, 6F, 7F, 8F, 9F, 10F), list);
+    
+    try {
+      list.add(-1, 5F);
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+    
+    try {
+      list.add(4, 5F);
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+  }
+  
+  public void testAddFloat() {
+    assertEquals(0, list.size());
+
+    list.addFloat(2);
+    assertEquals(asList(2F), list);
+
+    list.addFloat(3);
+    assertEquals(asList(2F, 3F), list);
+  }
+  
+  public void testAddAll() {
+    assertEquals(0, list.size());
+
+    assertTrue(list.addAll(Collections.singleton(1F)));
+    assertEquals(1, list.size());
+    assertEquals(1F, (float) list.get(0));
+    assertEquals(1F, list.getFloat(0));
+    
+    assertTrue(list.addAll(asList(2F, 3F, 4F, 5F, 6F)));
+    assertEquals(asList(1F, 2F, 3F, 4F, 5F, 6F), list);
+    
+    assertTrue(list.addAll(TERTIARY_LIST));
+    assertEquals(asList(1F, 2F, 3F, 4F, 5F, 6F, 1F, 2F, 3F), list);
+
+    assertFalse(list.addAll(Collections.<Float>emptyList()));
+    assertFalse(list.addAll(FloatArrayList.emptyList()));
+  }
+  
+  public void testRemove() {
+    list.addAll(TERTIARY_LIST);
+    assertEquals(1F, (float) list.remove(0));
+    assertEquals(asList(2F, 3F), list);
+
+    assertTrue(list.remove(Float.valueOf(3)));
+    assertEquals(asList(2F), list);
+
+    assertFalse(list.remove(Float.valueOf(3)));
+    assertEquals(asList(2F), list);
+
+    assertEquals(2F, (float) list.remove(0));
+    assertEquals(asList(), list);
+    
+    try {
+      list.remove(-1);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+    
+    try {
+      list.remove(0);
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+  }
+  
+  private void assertImmutable(FloatArrayList list) {
+    if (list.contains(1)) {
+      throw new RuntimeException("Cannot test the immutability of lists that contain 1.");
+    }
+    
+    try {
+      list.add(1F);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.add(0, 1F);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(Collections.<Float>emptyList());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(Collections.singletonList(1F));
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(new FloatArrayList());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(UNARY_LIST);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(0, Collections.singleton(1F));
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(0, UNARY_LIST);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(0, Collections.<Float>emptyList());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addFloat(0);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.clear();
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+
+    try {
+      list.remove(1);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.remove(new Object());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.removeAll(Collections.<Float>emptyList());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.removeAll(Collections.singleton(1));
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.removeAll(UNARY_LIST);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.retainAll(Collections.<Float>emptyList());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.retainAll(Collections.singleton(1));
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.retainAll(UNARY_LIST);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.set(0, 0F);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.setFloat(0, 0);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+  }
+  
+  private static FloatArrayList newImmutableFloatArrayList(int... elements) {
+    FloatArrayList list = new FloatArrayList();
+    for (int element : elements) {
+      list.addFloat(element);
+    }
+    list.makeImmutable();
+    return list;
+  }
+}

java/src/test/java/com/google/protobuf/IntArrayListTest.java

@@ -0,0 +1,473 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package com.google.protobuf;
+
+import static java.util.Arrays.asList;
+
+import junit.framework.TestCase;
+
+import java.util.Collections;
+import java.util.ConcurrentModificationException;
+import java.util.Iterator;
+
+/**
+ * Tests for {@link IntArrayList}.
+ * 
+ * @author dweis@google.com (Daniel Weis)
+ */
+public class IntArrayListTest extends TestCase {
+  
+  private static final IntArrayList UNARY_LIST = newImmutableIntArrayList(1);
+  private static final IntArrayList TERTIARY_LIST =
+      newImmutableIntArrayList(1, 2, 3);
+  
+  private IntArrayList list;
+  
+  @Override
+  protected void setUp() throws Exception {
+    list = new IntArrayList();
+  }
+  
+  public void testEmptyListReturnsSameInstance() {
+    assertSame(IntArrayList.emptyList(), IntArrayList.emptyList());
+  }
+  
+  public void testEmptyListIsImmutable() {
+    assertImmutable(IntArrayList.emptyList());
+  }
+  
+  public void testMakeImmutable() {
+    list.addInt(2);
+    list.addInt(4);
+    list.addInt(6);
+    list.addInt(8);
+    list.makeImmutable();
+    assertImmutable(list);
+  }
+  
+  public void testCopyConstructor() {
+    IntArrayList copy = new IntArrayList(TERTIARY_LIST);
+    assertEquals(TERTIARY_LIST, copy);
+
+    copy = new IntArrayList(IntArrayList.emptyList());
+    assertEquals(IntArrayList.emptyList(), copy);
+    
+    copy = new IntArrayList(asList(1, 2, 3));
+    assertEquals(asList(1, 2, 3), copy);
+
+    copy = new IntArrayList(Collections.<Integer>emptyList());
+    assertEquals(IntArrayList.emptyList(), copy);
+  }
+
+  public void testModificationWithIteration() {
+    list.addAll(asList(1, 2, 3, 4));
+    Iterator<Integer> iterator = list.iterator();
+    assertEquals(4, list.size());
+    assertEquals(1, (int) list.get(0));
+    assertEquals(1, (int) iterator.next());
+    list.set(0, 1);
+    assertEquals(2, (int) iterator.next());
+    
+    list.remove(0);
+    try {
+      iterator.next();
+      fail();
+    } catch (ConcurrentModificationException e) {
+      // expected
+    }
+
+    iterator = list.iterator();
+    list.add(0, 0);
+    try {
+      iterator.next();
+      fail();
+    } catch (ConcurrentModificationException e) {
+      // expected
+    }
+  }
+  
+  public void testGet() {
+    assertEquals(1, (int) TERTIARY_LIST.get(0));
+    assertEquals(2, (int) TERTIARY_LIST.get(1));
+    assertEquals(3, (int) TERTIARY_LIST.get(2));
+    
+    try {
+      TERTIARY_LIST.get(-1);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+    
+    try {
+      TERTIARY_LIST.get(3);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+  }
+  
+  public void testGetInt() {
+    assertEquals(1, TERTIARY_LIST.getInt(0));
+    assertEquals(2, TERTIARY_LIST.getInt(1));
+    assertEquals(3, TERTIARY_LIST.getInt(2));
+    
+    try {
+      TERTIARY_LIST.get(-1);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+    
+    try {
+      TERTIARY_LIST.get(3);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+  }
+  
+  public void testSize() {
+    assertEquals(0, IntArrayList.emptyList().size());
+    assertEquals(1, UNARY_LIST.size());
+    assertEquals(3, TERTIARY_LIST.size());
+
+    list.addInt(2);
+    list.addInt(4);
+    list.addInt(6);
+    list.addInt(8);
+    assertEquals(4, list.size());
+    
+    list.remove(0);
+    assertEquals(3, list.size());
+    
+    list.add(16);
+    assertEquals(4, list.size());
+  }
+  
+  public void testSet() {
+    list.addInt(2);
+    list.addInt(4);
+    
+    assertEquals(2, (int) list.set(0, 0));
+    assertEquals(0, list.getInt(0));
+
+    assertEquals(4, (int) list.set(1, 0));
+    assertEquals(0, list.getInt(1));
+    
+    try {
+      list.set(-1, 0);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+
+    try {
+      list.set(2, 0);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+  }
+  
+  public void testSetInt() {
+    list.addInt(2);
+    list.addInt(4);
+    
+    assertEquals(2, list.setInt(0, 0));
+    assertEquals(0, list.getInt(0));
+
+    assertEquals(4, list.setInt(1, 0));
+    assertEquals(0, list.getInt(1));
+    
+    try {
+      list.setInt(-1, 0);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+
+    try {
+      list.setInt(2, 0);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+  }
+  
+  public void testAdd() {
+    assertEquals(0, list.size());
+
+    assertTrue(list.add(2));
+    assertEquals(asList(2), list);
+
+    assertTrue(list.add(3));
+    list.add(0, 4);
+    assertEquals(asList(4, 2, 3), list);
+    
+    list.add(0, 1);
+    list.add(0, 0);
+    // Force a resize by getting up to 11 elements.
+    for (int i = 0; i < 6; i++) {
+      list.add(5 + i);
+    }
+    assertEquals(asList(0, 1, 4, 2, 3, 5, 6, 7, 8, 9, 10), list);
+    
+    try {
+      list.add(-1, 5);
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+    
+    try {
+      list.add(4, 5);
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+  }
+  
+  public void testAddInt() {
+    assertEquals(0, list.size());
+
+    list.addInt(2);
+    assertEquals(asList(2), list);
+
+    list.addInt(3);
+    assertEquals(asList(2, 3), list);
+  }
+  
+  public void testAddAll() {
+    assertEquals(0, list.size());
+
+    assertTrue(list.addAll(Collections.singleton(1)));
+    assertEquals(1, list.size());
+    assertEquals(1, (int) list.get(0));
+    assertEquals(1, list.getInt(0));
+    
+    assertTrue(list.addAll(asList(2, 3, 4, 5, 6)));
+    assertEquals(asList(1, 2, 3, 4, 5, 6), list);
+    
+    assertTrue(list.addAll(TERTIARY_LIST));
+    assertEquals(asList(1, 2, 3, 4, 5, 6, 1, 2, 3), list);
+
+    assertFalse(list.addAll(Collections.<Integer>emptyList()));
+    assertFalse(list.addAll(IntArrayList.emptyList()));
+  }
+  
+  public void testRemove() {
+    list.addAll(TERTIARY_LIST);
+    assertEquals(1, (int) list.remove(0));
+    assertEquals(asList(2, 3), list);
+
+    assertTrue(list.remove(Integer.valueOf(3)));
+    assertEquals(asList(2), list);
+
+    assertFalse(list.remove(Integer.valueOf(3)));
+    assertEquals(asList(2), list);
+
+    assertEquals(2, (int) list.remove(0));
+    assertEquals(asList(), list);
+    
+    try {
+      list.remove(-1);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+    
+    try {
+      list.remove(0);
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+  }
+  
+  private void assertImmutable(IntArrayList list) {
+    if (list.contains(1)) {
+      throw new RuntimeException("Cannot test the immutability of lists that contain 1.");
+    }
+    
+    try {
+      list.add(1);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.add(0, 1);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(Collections.<Integer>emptyList());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(Collections.singletonList(1));
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(new IntArrayList());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(UNARY_LIST);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(0, Collections.singleton(1));
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(0, UNARY_LIST);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(0, Collections.<Integer>emptyList());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addInt(0);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.clear();
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+
+    try {
+      list.remove(1);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.remove(new Object());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.removeAll(Collections.<Integer>emptyList());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.removeAll(Collections.singleton(1));
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.removeAll(UNARY_LIST);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.retainAll(Collections.<Integer>emptyList());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.retainAll(Collections.singleton(1));
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.retainAll(UNARY_LIST);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.set(0, 0);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.setInt(0, 0);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+  }
+  
+  private static IntArrayList newImmutableIntArrayList(int... elements) {
+    IntArrayList list = new IntArrayList();
+    for (int element : elements) {
+      list.addInt(element);
+    }
+    list.makeImmutable();
+    return list;
+  }
+}

java/src/test/java/com/google/protobuf/LazyStringArrayListTest.java

@@ -30,9 +30,13 @@
 
 package com.google.protobuf;
 
+import static java.util.Arrays.asList;
+
 import junit.framework.TestCase;
 
 import java.util.ArrayList;
+import java.util.ConcurrentModificationException;
+import java.util.Iterator;
 import java.util.List;
 
 /**
@@ -171,4 +175,188 @@ public class LazyStringArrayListTest extends TestCase {
     assertSame(BYTE_STRING_B, list2.getByteString(1));
     assertSame(BYTE_STRING_C, list2.getByteString(2));
   }
+  
+  public void testModificationWithIteration() {
+    LazyStringArrayList list = new LazyStringArrayList();
+    list.addAll(asList(STRING_A, STRING_B, STRING_C));
+    Iterator<String> iterator = list.iterator();
+    assertEquals(3, list.size());
+    assertEquals(STRING_A, list.get(0));
+    assertEquals(STRING_A, iterator.next());
+    
+    // Does not structurally modify.
+    iterator = list.iterator();
+    list.set(0, STRING_B);
+    iterator.next();
+    
+    list.remove(0);
+    try {
+      iterator.next();
+      fail();
+    } catch (ConcurrentModificationException e) {
+      // expected
+    }
+    
+    iterator = list.iterator();
+    list.add(0, STRING_C);
+    try {
+      iterator.next();
+      fail();
+    } catch (ConcurrentModificationException e) {
+      // expected
+    }
+  }
+  
+  public void testMakeImmutable() {
+    LazyStringArrayList list = new LazyStringArrayList();
+    list.add(STRING_A);
+    list.add(STRING_B);
+    list.add(STRING_C);
+    list.makeImmutable();
+    assertGenericListImmutable(list, STRING_A);
+    
+    // LazyStringArrayList has extra methods not covered in the generic
+    // assertion.
+    
+    try {
+      list.add(BYTE_STRING_A.toByteArray());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.add(BYTE_STRING_A);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAllByteArray(asList(BYTE_STRING_A.toByteArray()));
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAllByteString(asList(BYTE_STRING_A));
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.mergeFrom(new LazyStringArrayList());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.set(0, BYTE_STRING_A.toByteArray());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.set(0, BYTE_STRING_A);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+  }
+  
+  public void testImmutabilityPropagation() {
+    LazyStringArrayList list = new LazyStringArrayList();
+    list.add(STRING_A);
+    list.makeImmutable();
+
+    assertGenericListImmutable(list.asByteStringList(), BYTE_STRING_A);
+    
+    // Arrays use reference equality so need to retrieve the underlying value
+    // to properly test deep immutability.
+    List<byte[]> byteArrayList = list.asByteArrayList();
+    assertGenericListImmutable(byteArrayList, byteArrayList.get(0));
+  }
+  
+  private static <T> void assertGenericListImmutable(List<T> list, T value) {
+    try {
+      list.add(value);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.add(0, value);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(asList(value));
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(0, asList(value));
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+
+    try {
+      list.clear();
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.remove(0);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.remove(value);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.removeAll(asList(value));
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.retainAll(asList());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.retainAll(asList());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.set(0, value);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+  }
 }

java/src/test/java/com/google/protobuf/LiteTest.java

@@ -30,9 +30,18 @@
 
 package com.google.protobuf;
 
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singletonList;
+
 import com.google.protobuf.UnittestLite;
-import com.google.protobuf.UnittestLite.TestAllTypesLite;
+import com.google.protobuf.UnittestLite.ForeignEnumLite;
+import com.google.protobuf.UnittestLite.ForeignMessageLite;
 import com.google.protobuf.UnittestLite.TestAllExtensionsLite;
+import com.google.protobuf.UnittestLite.TestAllTypesLite;
+import com.google.protobuf.UnittestLite.TestAllTypesLite.NestedMessage;
+import com.google.protobuf.UnittestLite.TestAllTypesLite.OneofFieldCase;
+import com.google.protobuf.UnittestLite.TestAllTypesLite.OptionalGroup;
+import com.google.protobuf.UnittestLite.TestAllTypesLite.RepeatedGroup;
 import com.google.protobuf.UnittestLite.TestNestedExtensionLite;
 
 import junit.framework.TestCase;
@@ -149,13 +158,1302 @@ public class LiteTest extends TestCase {
   public void testClone() {
     TestAllTypesLite.Builder expected = TestAllTypesLite.newBuilder()
         .setOptionalInt32(123);
-   assertEquals(
-       expected.getOptionalInt32(), expected.clone().getOptionalInt32());
+    assertEquals(
+        expected.getOptionalInt32(), expected.clone().getOptionalInt32());
    
-   TestAllExtensionsLite.Builder expected2 = TestAllExtensionsLite.newBuilder()
-       .setExtension(UnittestLite.optionalInt32ExtensionLite, 123);
-   assertEquals(
-       expected2.getExtension(UnittestLite.optionalInt32ExtensionLite),
-       expected2.clone().getExtension(UnittestLite.optionalInt32ExtensionLite));
+    TestAllExtensionsLite.Builder expected2 = TestAllExtensionsLite.newBuilder()
+        .setExtension(UnittestLite.optionalInt32ExtensionLite, 123);
+    assertEquals(
+        expected2.getExtension(UnittestLite.optionalInt32ExtensionLite),
+        expected2.clone().getExtension(UnittestLite.optionalInt32ExtensionLite));
+  }
+  
+  public void testAddAll() {
+    try {
+      TestAllTypesLite.newBuilder()
+          .addAllRepeatedBytes(null);
+      fail();
+    } catch (NullPointerException e) {
+      // expected.
+    }
+  }
+  
+  public void testSanityCopyOnWrite() throws InvalidProtocolBufferException {
+    // Since builders are implemented as a thin wrapper around a message
+    // instance, we attempt to verify that we can't cause the builder to modify
+    // a produced message.
+
+    TestAllTypesLite.Builder builder = TestAllTypesLite.newBuilder();
+    TestAllTypesLite message = builder.build();
+    TestAllTypesLite messageAfterBuild;
+    builder.setOptionalBool(true);
+    assertEquals(false, message.getOptionalBool());
+    assertEquals(true, builder.getOptionalBool());
+    messageAfterBuild = builder.build();
+    assertEquals(true, messageAfterBuild.getOptionalBool());
+    assertEquals(false, message.getOptionalBool());
+    builder.clearOptionalBool();
+    assertEquals(false, builder.getOptionalBool());
+    assertEquals(true, messageAfterBuild.getOptionalBool());
+
+    message = builder.build();
+    builder.setOptionalBytes(ByteString.copyFromUtf8("hi"));
+    assertEquals(ByteString.EMPTY, message.getOptionalBytes());
+    assertEquals(ByteString.copyFromUtf8("hi"), builder.getOptionalBytes());
+    messageAfterBuild = builder.build();
+    assertEquals(
+        ByteString.copyFromUtf8("hi"), messageAfterBuild.getOptionalBytes());
+    assertEquals(ByteString.EMPTY, message.getOptionalBytes());
+    builder.clearOptionalBytes();
+    assertEquals(ByteString.EMPTY, builder.getOptionalBytes());
+    assertEquals(
+        ByteString.copyFromUtf8("hi"), messageAfterBuild.getOptionalBytes());
+
+    message = builder.build();
+    builder.setOptionalCord("hi");
+    assertEquals("", message.getOptionalCord());
+    assertEquals("hi", builder.getOptionalCord());
+    messageAfterBuild = builder.build();
+    assertEquals("hi", messageAfterBuild.getOptionalCord());
+    assertEquals("", message.getOptionalCord());
+    builder.clearOptionalCord();
+    assertEquals("", builder.getOptionalCord());
+    assertEquals("hi", messageAfterBuild.getOptionalCord());
+
+    message = builder.build();
+    builder.setOptionalCordBytes(ByteString.copyFromUtf8("no"));
+    assertEquals(ByteString.EMPTY, message.getOptionalCordBytes());
+    assertEquals(ByteString.copyFromUtf8("no"), builder.getOptionalCordBytes());
+    messageAfterBuild = builder.build();
+    assertEquals(
+        ByteString.copyFromUtf8("no"),
+        messageAfterBuild.getOptionalCordBytes());
+    assertEquals(ByteString.EMPTY, message.getOptionalCordBytes());
+    builder.clearOptionalCord();
+    assertEquals(ByteString.EMPTY, builder.getOptionalCordBytes());
+    assertEquals(
+        ByteString.copyFromUtf8("no"),
+        messageAfterBuild.getOptionalCordBytes());
+    
+    message = builder.build();
+    builder.setOptionalDouble(1);
+    assertEquals(0D, message.getOptionalDouble());
+    assertEquals(1D, builder.getOptionalDouble());
+    messageAfterBuild = builder.build();
+    assertEquals(1D, messageAfterBuild.getOptionalDouble());
+    assertEquals(0D, message.getOptionalDouble());
+    builder.clearOptionalDouble();
+    assertEquals(0D, builder.getOptionalDouble());
+    assertEquals(1D, messageAfterBuild.getOptionalDouble());
+    
+    message = builder.build();
+    builder.setOptionalFixed32(1);
+    assertEquals(0, message.getOptionalFixed32());
+    assertEquals(1, builder.getOptionalFixed32());
+    messageAfterBuild = builder.build();
+    assertEquals(1, messageAfterBuild.getOptionalFixed32());
+    assertEquals(0, message.getOptionalFixed32());
+    builder.clearOptionalFixed32();
+    assertEquals(0, builder.getOptionalFixed32());
+    assertEquals(1, messageAfterBuild.getOptionalFixed32());
+    
+    message = builder.build();
+    builder.setOptionalFixed64(1);
+    assertEquals(0L, message.getOptionalFixed64());
+    assertEquals(1L, builder.getOptionalFixed64());
+    messageAfterBuild = builder.build();
+    assertEquals(1L, messageAfterBuild.getOptionalFixed64());
+    assertEquals(0L, message.getOptionalFixed64());
+    builder.clearOptionalFixed64();
+    assertEquals(0L, builder.getOptionalFixed64());
+    assertEquals(1L, messageAfterBuild.getOptionalFixed64());
+
+    message = builder.build();
+    builder.setOptionalFloat(1);
+    assertEquals(0F, message.getOptionalFloat());
+    assertEquals(1F, builder.getOptionalFloat());
+    messageAfterBuild = builder.build();
+    assertEquals(1F, messageAfterBuild.getOptionalFloat());
+    assertEquals(0F, message.getOptionalFloat());
+    builder.clearOptionalFloat();
+    assertEquals(0F, builder.getOptionalFloat());
+    assertEquals(1F, messageAfterBuild.getOptionalFloat());
+
+    message = builder.build();
+    builder.setOptionalForeignEnum(ForeignEnumLite.FOREIGN_LITE_BAR);
+    assertEquals(
+        ForeignEnumLite.FOREIGN_LITE_FOO, message.getOptionalForeignEnum());
+    assertEquals(
+        ForeignEnumLite.FOREIGN_LITE_BAR, builder.getOptionalForeignEnum());
+    messageAfterBuild = builder.build();
+    assertEquals(
+        ForeignEnumLite.FOREIGN_LITE_BAR,
+        messageAfterBuild.getOptionalForeignEnum());
+    assertEquals(
+        ForeignEnumLite.FOREIGN_LITE_FOO, message.getOptionalForeignEnum());
+    builder.clearOptionalForeignEnum();
+    assertEquals(
+        ForeignEnumLite.FOREIGN_LITE_FOO, builder.getOptionalForeignEnum());
+    assertEquals(
+        ForeignEnumLite.FOREIGN_LITE_BAR,
+        messageAfterBuild.getOptionalForeignEnum());
+
+    message = builder.build();
+    ForeignMessageLite foreignMessage = ForeignMessageLite.newBuilder()
+        .setC(1)
+        .build();
+    builder.setOptionalForeignMessage(foreignMessage);
+    assertEquals(
+        ForeignMessageLite.getDefaultInstance(),
+        message.getOptionalForeignMessage());
+    assertEquals(foreignMessage, builder.getOptionalForeignMessage());
+    messageAfterBuild = builder.build();
+    assertEquals(foreignMessage, messageAfterBuild.getOptionalForeignMessage());
+    assertEquals(
+        ForeignMessageLite.getDefaultInstance(),
+        message.getOptionalForeignMessage());
+    builder.clearOptionalForeignMessage();
+    assertEquals(
+        ForeignMessageLite.getDefaultInstance(),
+        builder.getOptionalForeignMessage());
+    assertEquals(foreignMessage, messageAfterBuild.getOptionalForeignMessage());
+
+    message = builder.build();
+    ForeignMessageLite.Builder foreignMessageBuilder =
+        ForeignMessageLite.newBuilder()
+            .setC(3);
+    builder.setOptionalForeignMessage(foreignMessageBuilder);
+    assertEquals(
+        ForeignMessageLite.getDefaultInstance(),
+        message.getOptionalForeignMessage());
+    // LITE_RUNTIME doesn't implement equals so we compare on a property and
+    // ensure the property isn't set on foreignMessage.
+    assertEquals(3, builder.getOptionalForeignMessage().getC());
+    messageAfterBuild = builder.build();
+    assertEquals(3, messageAfterBuild.getOptionalForeignMessage().getC());
+    assertEquals(
+        ForeignMessageLite.getDefaultInstance(),
+        message.getOptionalForeignMessage());
+    builder.clearOptionalForeignMessage();
+    assertEquals(
+        ForeignMessageLite.getDefaultInstance(),
+        builder.getOptionalForeignMessage());
+    assertEquals(3, messageAfterBuild.getOptionalForeignMessage().getC());
+
+    message = builder.build();
+    OptionalGroup optionalGroup = OptionalGroup.newBuilder()
+        .setA(1)
+        .build();
+    builder.setOptionalGroup(optionalGroup);
+    assertEquals(
+        OptionalGroup.getDefaultInstance(), message.getOptionalGroup());
+    assertEquals(optionalGroup, builder.getOptionalGroup());
+    messageAfterBuild = builder.build();
+    assertEquals(optionalGroup, messageAfterBuild.getOptionalGroup());
+    assertEquals(
+        OptionalGroup.getDefaultInstance(), message.getOptionalGroup());
+    builder.clearOptionalGroup();
+    assertEquals(
+        OptionalGroup.getDefaultInstance(), builder.getOptionalGroup());
+    assertEquals(optionalGroup, messageAfterBuild.getOptionalGroup());
+
+    message = builder.build();
+    OptionalGroup.Builder optionalGroupBuilder = OptionalGroup.newBuilder()
+        .setA(3);
+    builder.setOptionalGroup(optionalGroupBuilder);
+    assertEquals(
+        OptionalGroup.getDefaultInstance(), message.getOptionalGroup());
+    // LITE_RUNTIME doesn't implement equals so we compare on a property and
+    // ensure the property isn't set on optionalGroup.
+    assertEquals(3, builder.getOptionalGroup().getA());
+    messageAfterBuild = builder.build();
+    assertEquals(3, messageAfterBuild.getOptionalGroup().getA());
+    assertEquals(
+        OptionalGroup.getDefaultInstance(), message.getOptionalGroup());
+    builder.clearOptionalGroup();
+    assertEquals(
+        OptionalGroup.getDefaultInstance(), builder.getOptionalGroup());
+    assertEquals(3, messageAfterBuild.getOptionalGroup().getA());
+
+    message = builder.build();
+    builder.setOptionalInt32(1);
+    assertEquals(0, message.getOptionalInt32());
+    assertEquals(1, builder.getOptionalInt32());
+    messageAfterBuild = builder.build();
+    assertEquals(1, messageAfterBuild.getOptionalInt32());
+    assertEquals(0, message.getOptionalInt32());
+    builder.clearOptionalInt32();
+    assertEquals(0, builder.getOptionalInt32());
+    assertEquals(1, messageAfterBuild.getOptionalInt32());
+
+    message = builder.build();
+    builder.setOptionalInt64(1);
+    assertEquals(0L, message.getOptionalInt64());
+    assertEquals(1L, builder.getOptionalInt64());
+    messageAfterBuild = builder.build();
+    assertEquals(1L, messageAfterBuild.getOptionalInt64());
+    assertEquals(0L, message.getOptionalInt64());
+    builder.clearOptionalInt64();
+    assertEquals(0L, builder.getOptionalInt64());
+    assertEquals(1L, messageAfterBuild.getOptionalInt64());
+    
+    message = builder.build();
+    NestedMessage nestedMessage = NestedMessage.newBuilder()
+        .setBb(1)
+        .build();
+    builder.setOptionalLazyMessage(nestedMessage);
+    assertEquals(
+        NestedMessage.getDefaultInstance(),
+        message.getOptionalLazyMessage());
+    assertEquals(nestedMessage, builder.getOptionalLazyMessage());
+    messageAfterBuild = builder.build();
+    assertEquals(nestedMessage, messageAfterBuild.getOptionalLazyMessage());
+    assertEquals(
+        NestedMessage.getDefaultInstance(),
+        message.getOptionalLazyMessage());
+    builder.clearOptionalLazyMessage();
+    assertEquals(
+        NestedMessage.getDefaultInstance(), builder.getOptionalLazyMessage());
+    assertEquals(nestedMessage, messageAfterBuild.getOptionalLazyMessage());
+
+    message = builder.build();
+    NestedMessage.Builder nestedMessageBuilder =
+        NestedMessage.newBuilder()
+            .setBb(3);
+    builder.setOptionalLazyMessage(nestedMessageBuilder);
+    assertEquals(
+        NestedMessage.getDefaultInstance(),
+        message.getOptionalLazyMessage());
+    // LITE_RUNTIME doesn't implement equals so we compare on a property.
+    assertEquals(3, builder.getOptionalLazyMessage().getBb());
+    messageAfterBuild = builder.build();
+    assertEquals(3, messageAfterBuild.getOptionalLazyMessage().getBb());
+    assertEquals(
+        NestedMessage.getDefaultInstance(),
+        message.getOptionalLazyMessage());
+    builder.clearOptionalLazyMessage();
+    assertEquals(
+        NestedMessage.getDefaultInstance(), builder.getOptionalLazyMessage());
+    assertEquals(3, messageAfterBuild.getOptionalLazyMessage().getBb());
+
+    message = builder.build();
+    builder.setOptionalSfixed32(1);
+    assertEquals(0, message.getOptionalSfixed32());
+    assertEquals(1, builder.getOptionalSfixed32());
+    messageAfterBuild = builder.build();
+    assertEquals(1, messageAfterBuild.getOptionalSfixed32());
+    assertEquals(0, message.getOptionalSfixed32());
+    builder.clearOptionalSfixed32();
+    assertEquals(0, builder.getOptionalSfixed32());
+    assertEquals(1, messageAfterBuild.getOptionalSfixed32());
+
+    message = builder.build();
+    builder.setOptionalSfixed64(1);
+    assertEquals(0L, message.getOptionalSfixed64());
+    assertEquals(1L, builder.getOptionalSfixed64());
+    messageAfterBuild = builder.build();
+    assertEquals(1L, messageAfterBuild.getOptionalSfixed64());
+    assertEquals(0L, message.getOptionalSfixed64());
+    builder.clearOptionalSfixed64();
+    assertEquals(0L, builder.getOptionalSfixed64());
+    assertEquals(1L, messageAfterBuild.getOptionalSfixed64());
+
+    message = builder.build();
+    builder.setOptionalSint32(1);
+    assertEquals(0, message.getOptionalSint32());
+    assertEquals(1, builder.getOptionalSint32());
+    messageAfterBuild = builder.build();
+    assertEquals(1, messageAfterBuild.getOptionalSint32());
+    builder.clearOptionalSint32();
+    assertEquals(0, builder.getOptionalSint32());
+    assertEquals(1, messageAfterBuild.getOptionalSint32());
+
+    message = builder.build();
+    builder.setOptionalSint64(1);
+    assertEquals(0L, message.getOptionalSint64());
+    assertEquals(1L, builder.getOptionalSint64());
+    messageAfterBuild = builder.build();
+    assertEquals(1L, messageAfterBuild.getOptionalSint64());
+    assertEquals(0L, message.getOptionalSint64());
+    builder.clearOptionalSint64();
+    assertEquals(0L, builder.getOptionalSint64());
+    assertEquals(1L, messageAfterBuild.getOptionalSint64());
+
+    message = builder.build();
+    builder.setOptionalString("hi");
+    assertEquals("", message.getOptionalString());
+    assertEquals("hi", builder.getOptionalString());
+    messageAfterBuild = builder.build();
+    assertEquals("hi", messageAfterBuild.getOptionalString());
+    assertEquals("", message.getOptionalString());
+    builder.clearOptionalString();
+    assertEquals("", builder.getOptionalString());
+    assertEquals("hi", messageAfterBuild.getOptionalString());
+
+    message = builder.build();
+    builder.setOptionalStringBytes(ByteString.copyFromUtf8("no"));
+    assertEquals(ByteString.EMPTY, message.getOptionalStringBytes());
+    assertEquals(
+        ByteString.copyFromUtf8("no"), builder.getOptionalStringBytes());
+    messageAfterBuild = builder.build();
+    assertEquals(
+        ByteString.copyFromUtf8("no"),
+        messageAfterBuild.getOptionalStringBytes());
+    assertEquals(ByteString.EMPTY, message.getOptionalStringBytes());
+    builder.clearOptionalString();
+    assertEquals(ByteString.EMPTY, builder.getOptionalStringBytes());
+    assertEquals(
+        ByteString.copyFromUtf8("no"),
+        messageAfterBuild.getOptionalStringBytes());
+    
+    message = builder.build();
+    builder.setOptionalStringPiece("hi");
+    assertEquals("", message.getOptionalStringPiece());
+    assertEquals("hi", builder.getOptionalStringPiece());
+    messageAfterBuild = builder.build();
+    assertEquals("hi", messageAfterBuild.getOptionalStringPiece());
+    assertEquals("", message.getOptionalStringPiece());
+    builder.clearOptionalStringPiece();
+    assertEquals("", builder.getOptionalStringPiece());
+    assertEquals("hi", messageAfterBuild.getOptionalStringPiece());
+
+    message = builder.build();
+    builder.setOptionalStringPieceBytes(ByteString.copyFromUtf8("no"));
+    assertEquals(ByteString.EMPTY, message.getOptionalStringPieceBytes());
+    assertEquals(
+        ByteString.copyFromUtf8("no"), builder.getOptionalStringPieceBytes());
+    messageAfterBuild = builder.build();
+    assertEquals(
+        ByteString.copyFromUtf8("no"),
+        messageAfterBuild.getOptionalStringPieceBytes());
+    assertEquals(ByteString.EMPTY, message.getOptionalStringPieceBytes());
+    builder.clearOptionalStringPiece();
+    assertEquals(ByteString.EMPTY, builder.getOptionalStringPieceBytes());
+    assertEquals(
+        ByteString.copyFromUtf8("no"),
+        messageAfterBuild.getOptionalStringPieceBytes());
+
+    message = builder.build();
+    builder.setOptionalUint32(1);
+    assertEquals(0, message.getOptionalUint32());
+    assertEquals(1, builder.getOptionalUint32());
+    messageAfterBuild = builder.build();
+    assertEquals(1, messageAfterBuild.getOptionalUint32());
+    assertEquals(0, message.getOptionalUint32());
+    builder.clearOptionalUint32();
+    assertEquals(0, builder.getOptionalUint32());
+    assertEquals(1, messageAfterBuild.getOptionalUint32());
+
+    message = builder.build();
+    builder.setOptionalUint64(1);
+    assertEquals(0L, message.getOptionalUint64());
+    assertEquals(1L, builder.getOptionalUint64());
+    messageAfterBuild = builder.build();
+    assertEquals(1L, messageAfterBuild.getOptionalUint64());
+    assertEquals(0L, message.getOptionalUint64());
+    builder.clearOptionalUint64();
+    assertEquals(0L, builder.getOptionalUint64());
+    assertEquals(1L, messageAfterBuild.getOptionalUint64());
+
+    message = builder.build();
+    builder.addAllRepeatedBool(singletonList(true));
+    assertEquals(emptyList(), message.getRepeatedBoolList());
+    assertEquals(singletonList(true), builder.getRepeatedBoolList());
+    assertEquals(emptyList(), message.getRepeatedBoolList());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedBool();
+    assertEquals(emptyList(), builder.getRepeatedBoolList());
+    assertEquals(singletonList(true), messageAfterBuild.getRepeatedBoolList());
+
+    message = builder.build();
+    builder.addAllRepeatedBytes(singletonList(ByteString.copyFromUtf8("hi")));
+    assertEquals(emptyList(), message.getRepeatedBytesList());
+    assertEquals(
+        singletonList(ByteString.copyFromUtf8("hi")),
+        builder.getRepeatedBytesList());
+    assertEquals(emptyList(), message.getRepeatedBytesList());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedBytes();
+    assertEquals(emptyList(), builder.getRepeatedBytesList());
+    assertEquals(
+        singletonList(ByteString.copyFromUtf8("hi")),
+        messageAfterBuild.getRepeatedBytesList());
+
+    message = builder.build();
+    builder.addAllRepeatedCord(singletonList("hi"));
+    assertEquals(emptyList(), message.getRepeatedCordList());
+    assertEquals(singletonList("hi"), builder.getRepeatedCordList());
+    assertEquals(emptyList(), message.getRepeatedCordList());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedCord();
+    assertEquals(emptyList(), builder.getRepeatedCordList());
+    assertEquals(singletonList("hi"), messageAfterBuild.getRepeatedCordList());
+
+    message = builder.build();
+    builder.addAllRepeatedDouble(singletonList(1D));
+    assertEquals(emptyList(), message.getRepeatedDoubleList());
+    assertEquals(singletonList(1D), builder.getRepeatedDoubleList());
+    assertEquals(emptyList(), message.getRepeatedDoubleList());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedDouble();
+    assertEquals(emptyList(), builder.getRepeatedDoubleList());
+    assertEquals(singletonList(1D), messageAfterBuild.getRepeatedDoubleList());
+
+    message = builder.build();
+    builder.addAllRepeatedFixed32(singletonList(1));
+    assertEquals(emptyList(), message.getRepeatedFixed32List());
+    assertEquals(singletonList(1), builder.getRepeatedFixed32List());
+    assertEquals(emptyList(), message.getRepeatedFixed32List());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedFixed32();
+    assertEquals(emptyList(), builder.getRepeatedFixed32List());
+    assertEquals(singletonList(1), messageAfterBuild.getRepeatedFixed32List());
+
+    message = builder.build();
+    builder.addAllRepeatedFixed64(singletonList(1L));
+    assertEquals(emptyList(), message.getRepeatedFixed64List());
+    assertEquals(singletonList(1L), builder.getRepeatedFixed64List());
+    assertEquals(emptyList(), message.getRepeatedFixed64List());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedFixed64();
+    assertEquals(emptyList(), builder.getRepeatedFixed64List());
+    assertEquals(singletonList(1L), messageAfterBuild.getRepeatedFixed64List());
+
+    message = builder.build();
+    builder.addAllRepeatedFloat(singletonList(1F));
+    assertEquals(emptyList(), message.getRepeatedFloatList());
+    assertEquals(singletonList(1F), builder.getRepeatedFloatList());
+    assertEquals(emptyList(), message.getRepeatedFloatList());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedFloat();
+    assertEquals(emptyList(), builder.getRepeatedFloatList());
+    assertEquals(singletonList(1F), messageAfterBuild.getRepeatedFloatList());
+
+    message = builder.build();
+    builder.addAllRepeatedForeignEnum(
+        singletonList(ForeignEnumLite.FOREIGN_LITE_BAR));
+    assertEquals(emptyList(), message.getRepeatedForeignEnumList());
+    assertEquals(
+        singletonList(ForeignEnumLite.FOREIGN_LITE_BAR),
+        builder.getRepeatedForeignEnumList());
+    assertEquals(emptyList(), message.getRepeatedForeignEnumList());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedForeignEnum();
+    assertEquals(emptyList(), builder.getRepeatedForeignEnumList());
+    assertEquals(
+        singletonList(ForeignEnumLite.FOREIGN_LITE_BAR),
+        messageAfterBuild.getRepeatedForeignEnumList());
+
+    message = builder.build();
+    builder.addAllRepeatedForeignMessage(singletonList(foreignMessage));
+    assertEquals(emptyList(), message.getRepeatedForeignMessageList());
+    assertEquals(
+        singletonList(foreignMessage), builder.getRepeatedForeignMessageList());
+    assertEquals(emptyList(), message.getRepeatedForeignMessageList());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedForeignMessage();
+    assertEquals(emptyList(), builder.getRepeatedForeignMessageList());
+    assertEquals(
+        singletonList(foreignMessage),
+        messageAfterBuild.getRepeatedForeignMessageList());
+
+    message = builder.build();
+    builder.addAllRepeatedGroup(
+        singletonList(RepeatedGroup.getDefaultInstance()));
+    assertEquals(emptyList(), message.getRepeatedGroupList());
+    assertEquals(
+        singletonList(RepeatedGroup.getDefaultInstance()),
+        builder.getRepeatedGroupList());
+    assertEquals(emptyList(), message.getRepeatedGroupList());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedGroup();
+    assertEquals(emptyList(), builder.getRepeatedGroupList());
+    assertEquals(
+        singletonList(RepeatedGroup.getDefaultInstance()),
+        messageAfterBuild.getRepeatedGroupList());
+
+    message = builder.build();
+    builder.addAllRepeatedInt32(singletonList(1));
+    assertEquals(emptyList(), message.getRepeatedInt32List());
+    assertEquals(singletonList(1), builder.getRepeatedInt32List());
+    assertEquals(emptyList(), message.getRepeatedInt32List());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedInt32();
+    assertEquals(emptyList(), builder.getRepeatedInt32List());
+    assertEquals(singletonList(1), messageAfterBuild.getRepeatedInt32List());
+
+    message = builder.build();
+    builder.addAllRepeatedInt64(singletonList(1L));
+    assertEquals(emptyList(), message.getRepeatedInt64List());
+    assertEquals(singletonList(1L), builder.getRepeatedInt64List());
+    assertEquals(emptyList(), message.getRepeatedInt64List());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedInt64();
+    assertEquals(emptyList(), builder.getRepeatedInt64List());
+    assertEquals(singletonList(1L), messageAfterBuild.getRepeatedInt64List());
+
+    message = builder.build();
+    builder.addAllRepeatedLazyMessage(singletonList(nestedMessage));
+    assertEquals(emptyList(), message.getRepeatedLazyMessageList());
+    assertEquals(
+        singletonList(nestedMessage), builder.getRepeatedLazyMessageList());
+    assertEquals(emptyList(), message.getRepeatedLazyMessageList());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedLazyMessage();
+    assertEquals(emptyList(), builder.getRepeatedLazyMessageList());
+    assertEquals(
+        singletonList(nestedMessage),
+        messageAfterBuild.getRepeatedLazyMessageList());
+
+    message = builder.build();
+    builder.addAllRepeatedSfixed32(singletonList(1));
+    assertEquals(emptyList(), message.getRepeatedSfixed32List());
+    assertEquals(singletonList(1), builder.getRepeatedSfixed32List());
+    assertEquals(emptyList(), message.getRepeatedSfixed32List());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedSfixed32();
+    assertEquals(emptyList(), builder.getRepeatedSfixed32List());
+    assertEquals(singletonList(1), messageAfterBuild.getRepeatedSfixed32List());
+
+    message = builder.build();
+    builder.addAllRepeatedSfixed64(singletonList(1L));
+    assertEquals(emptyList(), message.getRepeatedSfixed64List());
+    assertEquals(singletonList(1L), builder.getRepeatedSfixed64List());
+    assertEquals(emptyList(), message.getRepeatedSfixed64List());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedSfixed64();
+    assertEquals(emptyList(), builder.getRepeatedSfixed64List());
+    assertEquals(
+        singletonList(1L), messageAfterBuild.getRepeatedSfixed64List());
+
+    message = builder.build();
+    builder.addAllRepeatedSint32(singletonList(1));
+    assertEquals(emptyList(), message.getRepeatedSint32List());
+    assertEquals(singletonList(1), builder.getRepeatedSint32List());
+    assertEquals(emptyList(), message.getRepeatedSint32List());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedSint32();
+    assertEquals(emptyList(), builder.getRepeatedSint32List());
+    assertEquals(singletonList(1), messageAfterBuild.getRepeatedSint32List());
+
+    message = builder.build();
+    builder.addAllRepeatedSint64(singletonList(1L));
+    assertEquals(emptyList(), message.getRepeatedSint64List());
+    assertEquals(singletonList(1L), builder.getRepeatedSint64List());
+    assertEquals(emptyList(), message.getRepeatedSint64List());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedSint64();
+    assertEquals(emptyList(), builder.getRepeatedSint64List());
+    assertEquals(singletonList(1L), messageAfterBuild.getRepeatedSint64List());
+
+    message = builder.build();
+    builder.addAllRepeatedString(singletonList("hi"));
+    assertEquals(emptyList(), message.getRepeatedStringList());
+    assertEquals(singletonList("hi"), builder.getRepeatedStringList());
+    assertEquals(emptyList(), message.getRepeatedStringList());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedString();
+    assertEquals(emptyList(), builder.getRepeatedStringList());
+    assertEquals(
+        singletonList("hi"), messageAfterBuild.getRepeatedStringList());
+
+    message = builder.build();
+    builder.addAllRepeatedStringPiece(singletonList("hi"));
+    assertEquals(emptyList(), message.getRepeatedStringPieceList());
+    assertEquals(singletonList("hi"), builder.getRepeatedStringPieceList());
+    assertEquals(emptyList(), message.getRepeatedStringPieceList());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedStringPiece();
+    assertEquals(emptyList(), builder.getRepeatedStringPieceList());
+    assertEquals(
+        singletonList("hi"), messageAfterBuild.getRepeatedStringPieceList());
+
+    message = builder.build();
+    builder.addAllRepeatedUint32(singletonList(1));
+    assertEquals(emptyList(), message.getRepeatedUint32List());
+    assertEquals(singletonList(1), builder.getRepeatedUint32List());
+    assertEquals(emptyList(), message.getRepeatedUint32List());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedUint32();
+    assertEquals(emptyList(), builder.getRepeatedUint32List());
+    assertEquals(singletonList(1), messageAfterBuild.getRepeatedUint32List());
+
+    message = builder.build();
+    builder.addAllRepeatedUint64(singletonList(1L));
+    assertEquals(emptyList(), message.getRepeatedUint64List());
+    assertEquals(singletonList(1L), builder.getRepeatedUint64List());
+    assertEquals(emptyList(), message.getRepeatedUint64List());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedUint64();
+    assertEquals(emptyList(), builder.getRepeatedUint64List());
+    assertEquals(singletonList(1L), messageAfterBuild.getRepeatedUint64List());
+
+    message = builder.build();
+    builder.addRepeatedBool(true);
+    assertEquals(emptyList(), message.getRepeatedBoolList());
+    assertEquals(singletonList(true), builder.getRepeatedBoolList());
+    assertEquals(emptyList(), message.getRepeatedBoolList());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedBool();
+    assertEquals(emptyList(), builder.getRepeatedBoolList());
+    assertEquals(singletonList(true), messageAfterBuild.getRepeatedBoolList());
+
+    message = builder.build();
+    builder.addRepeatedBytes(ByteString.copyFromUtf8("hi"));
+    assertEquals(emptyList(), message.getRepeatedBytesList());
+    assertEquals(
+        singletonList(ByteString.copyFromUtf8("hi")),
+        builder.getRepeatedBytesList());
+    assertEquals(emptyList(), message.getRepeatedBytesList());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedBytes();
+    assertEquals(emptyList(), builder.getRepeatedBytesList());
+    assertEquals(
+        singletonList(ByteString.copyFromUtf8("hi")),
+        messageAfterBuild.getRepeatedBytesList());
+
+    message = builder.build();
+    builder.addRepeatedCord("hi");
+    assertEquals(emptyList(), message.getRepeatedCordList());
+    assertEquals(singletonList("hi"), builder.getRepeatedCordList());
+    assertEquals(emptyList(), message.getRepeatedCordList());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedCord();
+    assertEquals(emptyList(), builder.getRepeatedCordList());
+    assertEquals(singletonList("hi"), messageAfterBuild.getRepeatedCordList());
+
+    message = builder.build();
+    builder.addRepeatedDouble(1D);
+    assertEquals(emptyList(), message.getRepeatedDoubleList());
+    assertEquals(singletonList(1D), builder.getRepeatedDoubleList());
+    assertEquals(emptyList(), message.getRepeatedDoubleList());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedDouble();
+    assertEquals(emptyList(), builder.getRepeatedDoubleList());
+    assertEquals(singletonList(1D), messageAfterBuild.getRepeatedDoubleList());
+
+    message = builder.build();
+    builder.addRepeatedFixed32(1);
+    assertEquals(emptyList(), message.getRepeatedFixed32List());
+    assertEquals(singletonList(1), builder.getRepeatedFixed32List());
+    assertEquals(emptyList(), message.getRepeatedFixed32List());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedFixed32();
+    assertEquals(emptyList(), builder.getRepeatedFixed32List());
+    assertEquals(singletonList(1), messageAfterBuild.getRepeatedFixed32List());
+
+    message = builder.build();
+    builder.addRepeatedFixed64(1L);
+    assertEquals(emptyList(), message.getRepeatedFixed64List());
+    assertEquals(singletonList(1L), builder.getRepeatedFixed64List());
+    assertEquals(emptyList(), message.getRepeatedFixed64List());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedFixed64();
+    assertEquals(emptyList(), builder.getRepeatedFixed64List());
+    assertEquals(singletonList(1L), messageAfterBuild.getRepeatedFixed64List());
+
+    message = builder.build();
+    builder.addRepeatedFloat(1F);
+    assertEquals(emptyList(), message.getRepeatedFloatList());
+    assertEquals(singletonList(1F), builder.getRepeatedFloatList());
+    assertEquals(emptyList(), message.getRepeatedFloatList());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedFloat();
+    assertEquals(emptyList(), builder.getRepeatedFloatList());
+    assertEquals(singletonList(1F), messageAfterBuild.getRepeatedFloatList());
+
+    message = builder.build();
+    builder.addRepeatedForeignEnum(ForeignEnumLite.FOREIGN_LITE_BAR);
+    assertEquals(emptyList(), message.getRepeatedForeignEnumList());
+    assertEquals(
+        singletonList(ForeignEnumLite.FOREIGN_LITE_BAR),
+        builder.getRepeatedForeignEnumList());
+    assertEquals(emptyList(), message.getRepeatedForeignEnumList());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedForeignEnum();
+    assertEquals(emptyList(), builder.getRepeatedForeignEnumList());
+    assertEquals(
+        singletonList(ForeignEnumLite.FOREIGN_LITE_BAR),
+        messageAfterBuild.getRepeatedForeignEnumList());
+
+    message = builder.build();
+    builder.addRepeatedForeignMessage(foreignMessage);
+    assertEquals(emptyList(), message.getRepeatedForeignMessageList());
+    assertEquals(
+        singletonList(foreignMessage), builder.getRepeatedForeignMessageList());
+    assertEquals(emptyList(), message.getRepeatedForeignMessageList());
+    messageAfterBuild = builder.build();
+    builder.removeRepeatedForeignMessage(0);
+    assertEquals(emptyList(), builder.getRepeatedForeignMessageList());
+    assertEquals(
+        singletonList(foreignMessage),
+        messageAfterBuild.getRepeatedForeignMessageList());
+
+    message = builder.build();
+    builder.addRepeatedGroup(RepeatedGroup.getDefaultInstance());
+    assertEquals(emptyList(), message.getRepeatedGroupList());
+    assertEquals(
+        singletonList(RepeatedGroup.getDefaultInstance()),
+        builder.getRepeatedGroupList());
+    assertEquals(emptyList(), message.getRepeatedGroupList());
+    messageAfterBuild = builder.build();
+    builder.removeRepeatedGroup(0);
+    assertEquals(emptyList(), builder.getRepeatedGroupList());
+    assertEquals(
+        singletonList(RepeatedGroup.getDefaultInstance()),
+        messageAfterBuild.getRepeatedGroupList());
+
+    message = builder.build();
+    builder.addRepeatedInt32(1);
+    assertEquals(emptyList(), message.getRepeatedInt32List());
+    assertEquals(singletonList(1), builder.getRepeatedInt32List());
+    assertEquals(emptyList(), message.getRepeatedInt32List());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedInt32();
+    assertEquals(emptyList(), builder.getRepeatedInt32List());
+    assertEquals(singletonList(1), messageAfterBuild.getRepeatedInt32List());
+
+    message = builder.build();
+    builder.addRepeatedInt64(1L);
+    assertEquals(emptyList(), message.getRepeatedInt64List());
+    assertEquals(singletonList(1L), builder.getRepeatedInt64List());
+    assertEquals(emptyList(), message.getRepeatedInt64List());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedInt64();
+    assertEquals(emptyList(), builder.getRepeatedInt64List());
+    assertEquals(singletonList(1L), messageAfterBuild.getRepeatedInt64List());
+
+    message = builder.build();
+    builder.addRepeatedLazyMessage(nestedMessage);
+    assertEquals(emptyList(), message.getRepeatedLazyMessageList());
+    assertEquals(
+        singletonList(nestedMessage), builder.getRepeatedLazyMessageList());
+    assertEquals(emptyList(), message.getRepeatedLazyMessageList());
+    messageAfterBuild = builder.build();
+    builder.removeRepeatedLazyMessage(0);
+    assertEquals(emptyList(), builder.getRepeatedLazyMessageList());
+    assertEquals(
+        singletonList(nestedMessage),
+        messageAfterBuild.getRepeatedLazyMessageList());
+
+    message = builder.build();
+    builder.addRepeatedSfixed32(1);
+    assertEquals(emptyList(), message.getRepeatedSfixed32List());
+    assertEquals(singletonList(1), builder.getRepeatedSfixed32List());
+    assertEquals(emptyList(), message.getRepeatedSfixed32List());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedSfixed32();
+    assertEquals(emptyList(), builder.getRepeatedSfixed32List());
+    assertEquals(singletonList(1), messageAfterBuild.getRepeatedSfixed32List());
+
+    message = builder.build();
+    builder.addRepeatedSfixed64(1L);
+    assertEquals(emptyList(), message.getRepeatedSfixed64List());
+    assertEquals(singletonList(1L), builder.getRepeatedSfixed64List());
+    assertEquals(emptyList(), message.getRepeatedSfixed64List());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedSfixed64();
+    assertEquals(emptyList(), builder.getRepeatedSfixed64List());
+    assertEquals(
+        singletonList(1L), messageAfterBuild.getRepeatedSfixed64List());
+
+    message = builder.build();
+    builder.addRepeatedSint32(1);
+    assertEquals(emptyList(), message.getRepeatedSint32List());
+    assertEquals(singletonList(1), builder.getRepeatedSint32List());
+    assertEquals(emptyList(), message.getRepeatedSint32List());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedSint32();
+    assertEquals(emptyList(), builder.getRepeatedSint32List());
+    assertEquals(singletonList(1), messageAfterBuild.getRepeatedSint32List());
+
+    message = builder.build();
+    builder.addRepeatedSint64(1L);
+    assertEquals(emptyList(), message.getRepeatedSint64List());
+    assertEquals(singletonList(1L), builder.getRepeatedSint64List());
+    assertEquals(emptyList(), message.getRepeatedSint64List());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedSint64();
+    assertEquals(emptyList(), builder.getRepeatedSint64List());
+    assertEquals(singletonList(1L), messageAfterBuild.getRepeatedSint64List());
+
+    message = builder.build();
+    builder.addRepeatedString("hi");
+    assertEquals(emptyList(), message.getRepeatedStringList());
+    assertEquals(singletonList("hi"), builder.getRepeatedStringList());
+    assertEquals(emptyList(), message.getRepeatedStringList());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedString();
+    assertEquals(emptyList(), builder.getRepeatedStringList());
+    assertEquals(
+        singletonList("hi"), messageAfterBuild.getRepeatedStringList());
+
+    message = builder.build();
+    builder.addRepeatedStringPiece("hi");
+    assertEquals(emptyList(), message.getRepeatedStringPieceList());
+    assertEquals(singletonList("hi"), builder.getRepeatedStringPieceList());
+    assertEquals(emptyList(), message.getRepeatedStringPieceList());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedStringPiece();
+    assertEquals(emptyList(), builder.getRepeatedStringPieceList());
+    assertEquals(
+        singletonList("hi"), messageAfterBuild.getRepeatedStringPieceList());
+
+    message = builder.build();
+    builder.addRepeatedUint32(1);
+    assertEquals(emptyList(), message.getRepeatedUint32List());
+    assertEquals(singletonList(1), builder.getRepeatedUint32List());
+    assertEquals(emptyList(), message.getRepeatedUint32List());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedUint32();
+    assertEquals(emptyList(), builder.getRepeatedUint32List());
+    assertEquals(singletonList(1), messageAfterBuild.getRepeatedUint32List());
+
+    message = builder.build();
+    builder.addRepeatedUint64(1L);
+    assertEquals(emptyList(), message.getRepeatedUint64List());
+    assertEquals(singletonList(1L), builder.getRepeatedUint64List());
+    assertEquals(emptyList(), message.getRepeatedUint64List());
+    messageAfterBuild = builder.build();
+    builder.clearRepeatedUint64();
+    assertEquals(emptyList(), builder.getRepeatedUint64List());
+    assertEquals(singletonList(1L), messageAfterBuild.getRepeatedUint64List());
+    
+    message = builder.build();
+    builder.addRepeatedBool(true);
+    messageAfterBuild = builder.build();
+    assertEquals(0, message.getRepeatedBoolCount());
+    builder.setRepeatedBool(0, false);
+    assertEquals(true, messageAfterBuild.getRepeatedBool(0));
+    assertEquals(false, builder.getRepeatedBool(0));
+    builder.clearRepeatedBool();
+    
+    message = builder.build();
+    builder.addRepeatedBytes(ByteString.copyFromUtf8("hi"));
+    messageAfterBuild = builder.build();
+    assertEquals(0, message.getRepeatedBytesCount());
+    builder.setRepeatedBytes(0, ByteString.EMPTY);
+    assertEquals(
+        ByteString.copyFromUtf8("hi"), messageAfterBuild.getRepeatedBytes(0));
+    assertEquals(ByteString.EMPTY, builder.getRepeatedBytes(0));
+    builder.clearRepeatedBytes();
+
+    message = builder.build();
+    builder.addRepeatedCord("hi");
+    messageAfterBuild = builder.build();
+    assertEquals(0, message.getRepeatedCordCount());
+    builder.setRepeatedCord(0, "");
+    assertEquals("hi", messageAfterBuild.getRepeatedCord(0));
+    assertEquals("", builder.getRepeatedCord(0));
+    builder.clearRepeatedCord();
+    message = builder.build();
+
+    builder.addRepeatedCordBytes(ByteString.copyFromUtf8("hi"));
+    messageAfterBuild = builder.build();
+    assertEquals(0, message.getRepeatedCordCount());
+    builder.setRepeatedCord(0, "");
+    assertEquals(
+        ByteString.copyFromUtf8("hi"), messageAfterBuild.getRepeatedCordBytes(0));
+    assertEquals(ByteString.EMPTY, builder.getRepeatedCordBytes(0));
+    builder.clearRepeatedCord();
+
+    message = builder.build();
+    builder.addRepeatedDouble(1D);
+    messageAfterBuild = builder.build();
+    assertEquals(0, message.getRepeatedDoubleCount());
+    builder.setRepeatedDouble(0, 0D);
+    assertEquals(1D, messageAfterBuild.getRepeatedDouble(0));
+    assertEquals(0D, builder.getRepeatedDouble(0));
+    builder.clearRepeatedDouble();
+
+    message = builder.build();
+    builder.addRepeatedFixed32(1);
+    messageAfterBuild = builder.build();
+    assertEquals(0, message.getRepeatedFixed32Count());
+    builder.setRepeatedFixed32(0, 0);
+    assertEquals(1, messageAfterBuild.getRepeatedFixed32(0));
+    assertEquals(0, builder.getRepeatedFixed32(0));
+    builder.clearRepeatedFixed32();
+
+    message = builder.build();
+    builder.addRepeatedFixed64(1L);
+    messageAfterBuild = builder.build();
+    assertEquals(0, message.getRepeatedFixed64Count());
+    builder.setRepeatedFixed64(0, 0L);
+    assertEquals(1L, messageAfterBuild.getRepeatedFixed64(0));
+    assertEquals(0L, builder.getRepeatedFixed64(0));
+    builder.clearRepeatedFixed64();
+
+    message = builder.build();
+    builder.addRepeatedFloat(1F);
+    messageAfterBuild = builder.build();
+    assertEquals(0, message.getRepeatedFloatCount());
+    builder.setRepeatedFloat(0, 0F);
+    assertEquals(1F, messageAfterBuild.getRepeatedFloat(0));
+    assertEquals(0F, builder.getRepeatedFloat(0));
+    builder.clearRepeatedFloat();
+
+    message = builder.build();
+    builder.addRepeatedForeignEnum(ForeignEnumLite.FOREIGN_LITE_BAR);
+    messageAfterBuild = builder.build();
+    assertEquals(0, message.getRepeatedForeignEnumCount());
+    builder.setRepeatedForeignEnum(0, ForeignEnumLite.FOREIGN_LITE_FOO);
+    assertEquals(
+        ForeignEnumLite.FOREIGN_LITE_BAR,
+        messageAfterBuild.getRepeatedForeignEnum(0));
+    assertEquals(
+        ForeignEnumLite.FOREIGN_LITE_FOO, builder.getRepeatedForeignEnum(0));
+    builder.clearRepeatedForeignEnum();
+
+    message = builder.build();
+    builder.addRepeatedForeignMessage(foreignMessage);
+    messageAfterBuild = builder.build();
+    assertEquals(0, message.getRepeatedForeignMessageCount());
+    builder.setRepeatedForeignMessage(
+        0, ForeignMessageLite.getDefaultInstance());
+    assertEquals(
+        foreignMessage, messageAfterBuild.getRepeatedForeignMessage(0));
+    assertEquals(
+        ForeignMessageLite.getDefaultInstance(),
+        builder.getRepeatedForeignMessage(0));
+    builder.clearRepeatedForeignMessage();
+    
+    message = builder.build();
+    builder.addRepeatedForeignMessage(foreignMessageBuilder);
+    messageAfterBuild = builder.build();
+    assertEquals(0, message.getRepeatedForeignMessageCount());
+    builder.setRepeatedForeignMessage(
+        0, ForeignMessageLite.getDefaultInstance());
+    // LITE_RUNTIME doesn't implement equals so we compare on a property.
+    assertEquals(3, messageAfterBuild.getRepeatedForeignMessage(0).getC());
+    assertEquals(
+        ForeignMessageLite.getDefaultInstance(),
+        builder.getRepeatedForeignMessage(0));
+    builder.clearRepeatedForeignMessage();
+
+    message = builder.build();
+    builder.addRepeatedForeignMessage(0, foreignMessage);
+    messageAfterBuild = builder.build();
+    assertEquals(0, message.getRepeatedForeignMessageCount());
+    builder.setRepeatedForeignMessage(0, foreignMessageBuilder);
+    assertEquals(
+        foreignMessage, messageAfterBuild.getRepeatedForeignMessage(0));
+    // LITE_RUNTIME doesn't implement equals so we compare on a property.
+    assertEquals(3, builder.getRepeatedForeignMessage(0).getC());
+    builder.clearRepeatedForeignMessage();
+
+    message = builder.build();
+    RepeatedGroup repeatedGroup = RepeatedGroup.newBuilder()
+        .setA(1)
+        .build();
+    builder.addRepeatedGroup(repeatedGroup);
+    messageAfterBuild = builder.build();
+    assertEquals(0, message.getRepeatedGroupCount());
+    builder.setRepeatedGroup(0, RepeatedGroup.getDefaultInstance());
+    assertEquals(repeatedGroup, messageAfterBuild.getRepeatedGroup(0));
+    assertEquals(
+        RepeatedGroup.getDefaultInstance(), builder.getRepeatedGroup(0));
+    builder.clearRepeatedGroup();
+    
+    message = builder.build();
+    builder.addRepeatedGroup(0, repeatedGroup);
+    messageAfterBuild = builder.build();
+    assertEquals(0, message.getRepeatedGroupCount());
+    builder.setRepeatedGroup(0, RepeatedGroup.getDefaultInstance());
+    assertEquals(repeatedGroup, messageAfterBuild.getRepeatedGroup(0));
+    assertEquals(
+        RepeatedGroup.getDefaultInstance(), builder.getRepeatedGroup(0));
+    builder.clearRepeatedGroup();
+    
+    message = builder.build();
+    RepeatedGroup.Builder repeatedGroupBuilder = RepeatedGroup.newBuilder()
+        .setA(3);
+    builder.addRepeatedGroup(repeatedGroupBuilder);
+    messageAfterBuild = builder.build();
+    assertEquals(0, message.getRepeatedGroupCount());
+    builder.setRepeatedGroup(0, RepeatedGroup.getDefaultInstance());
+    // LITE_RUNTIME doesn't implement equals so we compare on a property and
+    // ensure the property isn't set on repeatedGroup.
+    assertEquals(3, messageAfterBuild.getRepeatedGroup(0).getA());
+    assertEquals(
+        RepeatedGroup.getDefaultInstance(), builder.getRepeatedGroup(0));
+    builder.clearRepeatedGroup();
+    
+    message = builder.build();
+    builder.addRepeatedGroup(0, repeatedGroupBuilder);
+    messageAfterBuild = builder.build();
+    assertEquals(0, message.getRepeatedGroupCount());
+    builder.setRepeatedGroup(0, RepeatedGroup.getDefaultInstance());
+    // LITE_RUNTIME doesn't implement equals so we compare on a property and
+    // ensure the property isn't set on repeatedGroup.
+    assertEquals(3, messageAfterBuild.getRepeatedGroup(0).getA());
+    assertEquals(
+        RepeatedGroup.getDefaultInstance(), builder.getRepeatedGroup(0));
+    builder.clearRepeatedGroup();
+
+    message = builder.build();
+    builder.addRepeatedInt32(1);
+    messageAfterBuild = builder.build();
+    assertEquals(0, message.getRepeatedInt32Count());
+    builder.setRepeatedInt32(0, 0);
+    assertEquals(1, messageAfterBuild.getRepeatedInt32(0));
+    assertEquals(0, builder.getRepeatedInt32(0));
+    builder.clearRepeatedInt32();
+
+    message = builder.build();
+    builder.addRepeatedInt64(1L);
+    messageAfterBuild = builder.build();
+    assertEquals(0L, message.getRepeatedInt64Count());
+    builder.setRepeatedInt64(0, 0L);
+    assertEquals(1L, messageAfterBuild.getRepeatedInt64(0));
+    assertEquals(0L, builder.getRepeatedInt64(0));
+    builder.clearRepeatedInt64();
+    
+    message = builder.build();
+    builder.addRepeatedLazyMessage(nestedMessage);
+    messageAfterBuild = builder.build();
+    assertEquals(0, message.getRepeatedLazyMessageCount());
+    builder.setRepeatedLazyMessage(0, NestedMessage.getDefaultInstance());
+    assertEquals(nestedMessage, messageAfterBuild.getRepeatedLazyMessage(0));
+    assertEquals(
+        NestedMessage.getDefaultInstance(), builder.getRepeatedLazyMessage(0));
+    builder.clearRepeatedLazyMessage();
+    
+    message = builder.build();
+    builder.addRepeatedLazyMessage(0, nestedMessage);
+    messageAfterBuild = builder.build();
+    assertEquals(0, message.getRepeatedLazyMessageCount());
+    builder.setRepeatedLazyMessage(0, NestedMessage.getDefaultInstance());
+    assertEquals(nestedMessage, messageAfterBuild.getRepeatedLazyMessage(0));
+    assertEquals(
+        NestedMessage.getDefaultInstance(), builder.getRepeatedLazyMessage(0));
+    builder.clearRepeatedLazyMessage();
+    
+    message = builder.build();
+    builder.addRepeatedLazyMessage(nestedMessageBuilder);
+    messageAfterBuild = builder.build();
+    assertEquals(0, message.getRepeatedLazyMessageCount());
+    builder.setRepeatedLazyMessage(0, NestedMessage.getDefaultInstance());
+    // LITE_RUNTIME doesn't implement equals so we compare on a property and
+    // ensure the property isn't set on repeatedGroup.
+    assertEquals(3, messageAfterBuild.getRepeatedLazyMessage(0).getBb());
+    assertEquals(
+        NestedMessage.getDefaultInstance(), builder.getRepeatedLazyMessage(0));
+    builder.clearRepeatedLazyMessage();
+    
+    message = builder.build();
+    builder.addRepeatedLazyMessage(0, nestedMessageBuilder);
+    messageAfterBuild = builder.build();
+    assertEquals(0, message.getRepeatedLazyMessageCount());
+    builder.setRepeatedLazyMessage(0, NestedMessage.getDefaultInstance());
+    // LITE_RUNTIME doesn't implement equals so we compare on a property and
+    // ensure the property isn't set on repeatedGroup.
+    assertEquals(3, messageAfterBuild.getRepeatedLazyMessage(0).getBb());
+    assertEquals(
+        NestedMessage.getDefaultInstance(), builder.getRepeatedLazyMessage(0));
+    builder.clearRepeatedLazyMessage();
+
+    message = builder.build();
+    builder.addRepeatedSfixed32(1);
+    messageAfterBuild = builder.build();
+    assertEquals(0, message.getRepeatedSfixed32Count());
+    builder.setRepeatedSfixed32(0, 0);
+    assertEquals(1, messageAfterBuild.getRepeatedSfixed32(0));
+    assertEquals(0, builder.getRepeatedSfixed32(0));
+    builder.clearRepeatedSfixed32();
+
+    message = builder.build();
+    builder.addRepeatedSfixed64(1L);
+    messageAfterBuild = builder.build();
+    assertEquals(0L, message.getRepeatedSfixed64Count());
+    builder.setRepeatedSfixed64(0, 0L);
+    assertEquals(1L, messageAfterBuild.getRepeatedSfixed64(0));
+    assertEquals(0L, builder.getRepeatedSfixed64(0));
+    builder.clearRepeatedSfixed64();
+
+    message = builder.build();
+    builder.addRepeatedSint32(1);
+    messageAfterBuild = builder.build();
+    assertEquals(0, message.getRepeatedSint32Count());
+    builder.setRepeatedSint32(0, 0);
+    assertEquals(1, messageAfterBuild.getRepeatedSint32(0));
+    assertEquals(0, builder.getRepeatedSint32(0));
+    builder.clearRepeatedSint32();
+
+    message = builder.build();
+    builder.addRepeatedSint64(1L);
+    messageAfterBuild = builder.build();
+    assertEquals(0L, message.getRepeatedSint64Count());
+    builder.setRepeatedSint64(0, 0L);
+    assertEquals(1L, messageAfterBuild.getRepeatedSint64(0));
+    assertEquals(0L, builder.getRepeatedSint64(0));
+    builder.clearRepeatedSint64();
+
+    message = builder.build();
+    builder.addRepeatedString("hi");
+    messageAfterBuild = builder.build();
+    assertEquals(0L, message.getRepeatedStringCount());
+    builder.setRepeatedString(0, "");
+    assertEquals("hi", messageAfterBuild.getRepeatedString(0));
+    assertEquals("", builder.getRepeatedString(0));
+    builder.clearRepeatedString();
+
+    message = builder.build();
+    builder.addRepeatedStringBytes(ByteString.copyFromUtf8("hi"));
+    messageAfterBuild = builder.build();
+    assertEquals(0L, message.getRepeatedStringCount());
+    builder.setRepeatedString(0, "");
+    assertEquals(
+        ByteString.copyFromUtf8("hi"),
+        messageAfterBuild.getRepeatedStringBytes(0));
+    assertEquals(ByteString.EMPTY, builder.getRepeatedStringBytes(0));
+    builder.clearRepeatedString();
+
+    message = builder.build();
+    builder.addRepeatedStringPiece("hi");
+    messageAfterBuild = builder.build();
+    assertEquals(0L, message.getRepeatedStringPieceCount());
+    builder.setRepeatedStringPiece(0, "");
+    assertEquals("hi", messageAfterBuild.getRepeatedStringPiece(0));
+    assertEquals("", builder.getRepeatedStringPiece(0));
+    builder.clearRepeatedStringPiece();
+
+    message = builder.build();
+    builder.addRepeatedStringPieceBytes(ByteString.copyFromUtf8("hi"));
+    messageAfterBuild = builder.build();
+    assertEquals(0L, message.getRepeatedStringPieceCount());
+    builder.setRepeatedStringPiece(0, "");
+    assertEquals(
+        ByteString.copyFromUtf8("hi"),
+        messageAfterBuild.getRepeatedStringPieceBytes(0));
+    assertEquals(ByteString.EMPTY, builder.getRepeatedStringPieceBytes(0));
+    builder.clearRepeatedStringPiece();
+
+    message = builder.build();
+    builder.addRepeatedUint32(1);
+    messageAfterBuild = builder.build();
+    assertEquals(0, message.getRepeatedUint32Count());
+    builder.setRepeatedUint32(0, 0);
+    assertEquals(1, messageAfterBuild.getRepeatedUint32(0));
+    assertEquals(0, builder.getRepeatedUint32(0));
+    builder.clearRepeatedUint32();
+
+    message = builder.build();
+    builder.addRepeatedUint64(1L);
+    messageAfterBuild = builder.build();
+    assertEquals(0L, message.getRepeatedUint64Count());
+    builder.setRepeatedUint64(0, 0L);
+    assertEquals(1L, messageAfterBuild.getRepeatedUint64(0));
+    assertEquals(0L, builder.getRepeatedUint64(0));
+    builder.clearRepeatedUint64();
+
+    message = builder.build();
+    assertEquals(0, message.getSerializedSize());
+    builder.mergeFrom(TestAllTypesLite.newBuilder()
+        .setOptionalBool(true)
+        .build());
+    assertEquals(0, message.getSerializedSize());
+    assertEquals(true, builder.build().getOptionalBool());
+    builder.clearOptionalBool();
+
+    message = builder.build();
+    assertEquals(0, message.getSerializedSize());
+    builder.mergeFrom(TestAllTypesLite.newBuilder()
+        .setOptionalBool(true)
+        .build());
+    assertEquals(0, message.getSerializedSize());
+    assertEquals(true, builder.build().getOptionalBool());
+    builder.clear();
+    assertEquals(0, builder.build().getSerializedSize());
+
+    message = builder.build();
+    assertEquals(0, message.getSerializedSize());
+    builder.mergeOptionalForeignMessage(foreignMessage);
+    assertEquals(0, message.getSerializedSize());
+    assertEquals(
+        foreignMessage.getC(),
+        builder.build().getOptionalForeignMessage().getC());
+    builder.clearOptionalForeignMessage();
+
+    message = builder.build();
+    assertEquals(0, message.getSerializedSize());
+    builder.mergeOptionalLazyMessage(nestedMessage);
+    assertEquals(0, message.getSerializedSize());
+    assertEquals(
+        nestedMessage.getBb(),
+        builder.build().getOptionalLazyMessage().getBb());
+    builder.clearOptionalLazyMessage();
+    
+    message = builder.build();
+    builder.setOneofString("hi");
+    assertEquals(
+        OneofFieldCase.ONEOFFIELD_NOT_SET, message.getOneofFieldCase());
+    assertEquals(OneofFieldCase.ONEOF_STRING, builder.getOneofFieldCase());
+    assertEquals("hi", builder.getOneofString());
+    messageAfterBuild = builder.build();
+    assertEquals(
+        OneofFieldCase.ONEOF_STRING, messageAfterBuild.getOneofFieldCase());
+    assertEquals("hi", messageAfterBuild.getOneofString());
+    builder.setOneofUint32(1);
+    assertEquals(
+        OneofFieldCase.ONEOF_STRING, messageAfterBuild.getOneofFieldCase());
+    assertEquals("hi", messageAfterBuild.getOneofString());
+    assertEquals(OneofFieldCase.ONEOF_UINT32, builder.getOneofFieldCase());
+    assertEquals(1, builder.getOneofUint32());
+    
+    TestAllExtensionsLite.Builder extendableMessageBuilder =
+        TestAllExtensionsLite.newBuilder();
+    TestAllExtensionsLite extendableMessage = extendableMessageBuilder.build();
+    extendableMessageBuilder.setExtension(
+        UnittestLite.optionalInt32ExtensionLite, 1);
+    assertFalse(extendableMessage.hasExtension(
+        UnittestLite.optionalInt32ExtensionLite));
+    extendableMessage = extendableMessageBuilder.build();
+    assertEquals(
+        1, (int) extendableMessageBuilder.getExtension(
+            UnittestLite.optionalInt32ExtensionLite));
+    assertEquals(
+        1, (int) extendableMessage.getExtension(
+            UnittestLite.optionalInt32ExtensionLite));
+    extendableMessageBuilder.setExtension(
+        UnittestLite.optionalInt32ExtensionLite, 3);
+    assertEquals(
+        3, (int) extendableMessageBuilder.getExtension(
+            UnittestLite.optionalInt32ExtensionLite));
+    assertEquals(
+        1, (int) extendableMessage.getExtension(
+            UnittestLite.optionalInt32ExtensionLite));
+    extendableMessage = extendableMessageBuilder.build();
+    assertEquals(
+        3, (int) extendableMessageBuilder.getExtension(
+            UnittestLite.optionalInt32ExtensionLite));
+    assertEquals(
+        3, (int) extendableMessage.getExtension(
+            UnittestLite.optionalInt32ExtensionLite));
+    
+    // No extension registry, so it should be in unknown fields.
+    extendableMessage =
+        TestAllExtensionsLite.parseFrom(extendableMessage.toByteArray());
+    assertFalse(extendableMessage.hasExtension(
+        UnittestLite.optionalInt32ExtensionLite));
+    
+    extendableMessageBuilder = extendableMessage.toBuilder();
+    extendableMessageBuilder.mergeFrom(TestAllExtensionsLite.newBuilder()
+        .setExtension(UnittestLite.optionalFixed32ExtensionLite, 11)
+        .build());
+    
+    extendableMessage = extendableMessageBuilder.build();
+    ExtensionRegistryLite registry = ExtensionRegistryLite.newInstance();
+    UnittestLite.registerAllExtensions(registry);
+    extendableMessage = TestAllExtensionsLite.parseFrom(
+        extendableMessage.toByteArray(), registry);
+    
+    // The unknown field was preserved.
+    assertEquals(
+        3, (int) extendableMessage.getExtension(
+            UnittestLite.optionalInt32ExtensionLite));
+    assertEquals(
+        11, (int) extendableMessage.getExtension(
+            UnittestLite.optionalFixed32ExtensionLite));
   }
 }

java/src/test/java/com/google/protobuf/LiteralByteStringTest.java

@@ -307,7 +307,8 @@ public class LiteralByteStringTest extends TestCase {
 
   public void testToString_returnsCanonicalEmptyString() throws UnsupportedEncodingException{
     assertSame(classUnderTest + " must be the same string references",
-        ByteString.EMPTY.toString(Internal.UTF_8), new LiteralByteString(new byte[]{}).toString(Internal.UTF_8));
+        ByteString.EMPTY.toString(Internal.UTF_8),
+        new LiteralByteString(new byte[]{}).toString(Internal.UTF_8));
   }
 
   public void testToString_raisesException() throws UnsupportedEncodingException{

java/src/test/java/com/google/protobuf/LongArrayListTest.java

@@ -0,0 +1,473 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package com.google.protobuf;
+
+import static java.util.Arrays.asList;
+
+import junit.framework.TestCase;
+
+import java.util.Collections;
+import java.util.ConcurrentModificationException;
+import java.util.Iterator;
+
+/**
+ * Tests for {@link LongArrayList}.
+ * 
+ * @author dweis@google.com (Daniel Weis)
+ */
+public class LongArrayListTest extends TestCase {
+  
+  private static final LongArrayList UNARY_LIST = newImmutableLongArrayList(1);
+  private static final LongArrayList TERTIARY_LIST =
+      newImmutableLongArrayList(1, 2, 3);
+  
+  private LongArrayList list;
+  
+  @Override
+  protected void setUp() throws Exception {
+    list = new LongArrayList();
+  }
+  
+  public void testEmptyListReturnsSameInstance() {
+    assertSame(LongArrayList.emptyList(), LongArrayList.emptyList());
+  }
+  
+  public void testEmptyListIsImmutable() {
+    assertImmutable(LongArrayList.emptyList());
+  }
+  
+  public void testMakeImmutable() {
+    list.addLong(2);
+    list.addLong(4);
+    list.addLong(6);
+    list.addLong(8);
+    list.makeImmutable();
+    assertImmutable(list);
+  }
+  
+  public void testCopyConstructor() {
+    LongArrayList copy = new LongArrayList(TERTIARY_LIST);
+    assertEquals(TERTIARY_LIST, copy);
+
+    copy = new LongArrayList(LongArrayList.emptyList());
+    assertEquals(LongArrayList.emptyList(), copy);
+    
+    copy = new LongArrayList(asList(1L, 2L, 3L));
+    assertEquals(asList(1L, 2L, 3L), copy);
+
+    copy = new LongArrayList(Collections.<Long>emptyList());
+    assertEquals(LongArrayList.emptyList(), copy);
+  }
+  
+  public void testModificationWithIteration() {
+    list.addAll(asList(1L, 2L, 3L, 4L));
+    Iterator<Long> iterator = list.iterator();
+    assertEquals(4, list.size());
+    assertEquals(1, (long) list.get(0));
+    assertEquals(1, (long) iterator.next());
+    list.set(0, 1L);
+    assertEquals(2, (long) iterator.next());
+    
+    list.remove(0);
+    try {
+      iterator.next();
+      fail();
+    } catch (ConcurrentModificationException e) {
+      // expected
+    }
+    
+    iterator = list.iterator();
+    list.add(0, 0L);
+    try {
+      iterator.next();
+      fail();
+    } catch (ConcurrentModificationException e) {
+      // expected
+    }
+  }
+  
+  public void testGet() {
+    assertEquals(1, (long) TERTIARY_LIST.get(0));
+    assertEquals(2, (long) TERTIARY_LIST.get(1));
+    assertEquals(3, (long) TERTIARY_LIST.get(2));
+    
+    try {
+      TERTIARY_LIST.get(-1);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+    
+    try {
+      TERTIARY_LIST.get(3);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+  }
+  
+  public void testGetLong() {
+    assertEquals(1, TERTIARY_LIST.getLong(0));
+    assertEquals(2, TERTIARY_LIST.getLong(1));
+    assertEquals(3, TERTIARY_LIST.getLong(2));
+    
+    try {
+      TERTIARY_LIST.get(-1);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+    
+    try {
+      TERTIARY_LIST.get(3);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+  }
+  
+  public void testSize() {
+    assertEquals(0, LongArrayList.emptyList().size());
+    assertEquals(1, UNARY_LIST.size());
+    assertEquals(3, TERTIARY_LIST.size());
+
+    list.addLong(2);
+    list.addLong(4);
+    list.addLong(6);
+    list.addLong(8);
+    assertEquals(4, list.size());
+    
+    list.remove(0);
+    assertEquals(3, list.size());
+    
+    list.add(16L);
+    assertEquals(4, list.size());
+  }
+  
+  public void testSet() {
+    list.addLong(2);
+    list.addLong(4);
+    
+    assertEquals(2, (long) list.set(0, 0L));
+    assertEquals(0, list.getLong(0));
+
+    assertEquals(4, (long) list.set(1, 0L));
+    assertEquals(0, list.getLong(1));
+    
+    try {
+      list.set(-1, 0L);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+
+    try {
+      list.set(2, 0L);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+  }
+  
+  public void testSetLong() {
+    list.addLong(2);
+    list.addLong(4);
+    
+    assertEquals(2, list.setLong(0, 0));
+    assertEquals(0, list.getLong(0));
+
+    assertEquals(4, list.setLong(1, 0));
+    assertEquals(0, list.getLong(1));
+    
+    try {
+      list.setLong(-1, 0);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+
+    try {
+      list.setLong(2, 0);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+  }
+  
+  public void testAdd() {
+    assertEquals(0, list.size());
+
+    assertTrue(list.add(2L));
+    assertEquals(asList(2L), list);
+
+    assertTrue(list.add(3L));
+    list.add(0, 4L);
+    assertEquals(asList(4L, 2L, 3L), list);
+    
+    list.add(0, 1L);
+    list.add(0, 0L);
+    // Force a resize by getting up to 11 elements.
+    for (int i = 0; i < 6; i++) {
+      list.add(Long.valueOf(5 + i));
+    }
+    assertEquals(asList(0L, 1L, 4L, 2L, 3L, 5L, 6L, 7L, 8L, 9L, 10L), list);
+    
+    try {
+      list.add(-1, 5L);
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+    
+    try {
+      list.add(4, 5L);
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+  }
+  
+  public void testAddLong() {
+    assertEquals(0, list.size());
+
+    list.addLong(2);
+    assertEquals(asList(2L), list);
+
+    list.addLong(3);
+    assertEquals(asList(2L, 3L), list);
+  }
+  
+  public void testAddAll() {
+    assertEquals(0, list.size());
+
+    assertTrue(list.addAll(Collections.singleton(1L)));
+    assertEquals(1, list.size());
+    assertEquals(1, (long) list.get(0));
+    assertEquals(1, list.getLong(0));
+    
+    assertTrue(list.addAll(asList(2L, 3L, 4L, 5L, 6L)));
+    assertEquals(asList(1L, 2L, 3L, 4L, 5L, 6L), list);
+    
+    assertTrue(list.addAll(TERTIARY_LIST));
+    assertEquals(asList(1L, 2L, 3L, 4L, 5L, 6L, 1L, 2L, 3L), list);
+
+    assertFalse(list.addAll(Collections.<Long>emptyList()));
+    assertFalse(list.addAll(LongArrayList.emptyList()));
+  }
+  
+  public void testRemove() {
+    list.addAll(TERTIARY_LIST);
+    assertEquals(1, (long) list.remove(0));
+    assertEquals(asList(2L, 3L), list);
+
+    assertTrue(list.remove(3L));
+    assertEquals(asList(2L), list);
+
+    assertFalse(list.remove(3L));
+    assertEquals(asList(2L), list);
+
+    assertEquals(2, (long) list.remove(0));
+    assertEquals(asList(), list);
+    
+    try {
+      list.remove(-1);
+      fail();
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+    
+    try {
+      list.remove(0);
+    } catch (IndexOutOfBoundsException e) {
+      // expected
+    }
+  }
+  
+  private void assertImmutable(LongArrayList list) {
+    if (list.contains(1)) {
+      throw new RuntimeException("Cannot test the immutability of lists that contain 1.");
+    }
+    
+    try {
+      list.add(1L);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.add(0, 1L);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(Collections.<Long>emptyList());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(Collections.singletonList(1L));
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(new LongArrayList());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(UNARY_LIST);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(0, Collections.singleton(1L));
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(0, UNARY_LIST);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(0, Collections.<Long>emptyList());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addLong(0);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.clear();
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+
+    try {
+      list.remove(1);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.remove(new Object());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.removeAll(Collections.<Long>emptyList());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.removeAll(Collections.singleton(1));
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.removeAll(UNARY_LIST);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.retainAll(Collections.<Long>emptyList());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.retainAll(Collections.singleton(1));
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.retainAll(UNARY_LIST);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.set(0, 0L);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.setLong(0, 0);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+  }
+  
+  private static LongArrayList newImmutableLongArrayList(long... elements) {
+    LongArrayList list = new LongArrayList();
+    for (long element : elements) {
+      list.addLong(element);
+    }
+    list.makeImmutable();
+    return list;
+  }
+}

java/src/test/java/com/google/protobuf/MapForProto2LiteTest.java

@@ -36,6 +36,11 @@ import map_lite_test.MapForProto2TestProto.TestUnknownEnumValue;
 
 import junit.framework.TestCase;
 
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
 /**
  * Unit tests for map fields.
  */
@@ -170,6 +175,140 @@ public class MapForProto2LiteTest extends TestCase {
     assertEquals(0, message.getStringToInt32Field().size());
   }
 
+  public void testSanityCopyOnWrite() throws InvalidProtocolBufferException {
+    // Since builders are implemented as a thin wrapper around a message
+    // instance, we attempt to verify that we can't cause the builder to modify
+    // a produced message.
+    
+    TestMap.Builder builder = TestMap.newBuilder();
+    TestMap message = builder.build();
+    Map<Integer, Integer> intMap = builder.getMutableInt32ToInt32Field();
+    intMap.put(1, 2);
+    assertTrue(message.getInt32ToInt32Field().isEmpty());
+    message = builder.build();
+    try {
+      intMap.put(2, 3);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    assertEquals(newMap(1, 2), message.getInt32ToInt32Field());
+    assertEquals(newMap(1, 2), builder.getInt32ToInt32Field());
+    builder.getMutableInt32ToInt32Field().put(2, 3);
+    assertEquals(newMap(1, 2), message.getInt32ToInt32Field());
+    assertEquals(newMap(1, 2, 2, 3), builder.getInt32ToInt32Field());
+  }
+  
+  public void testMutableMapLifecycle() {
+    TestMap.Builder builder = TestMap.newBuilder();
+    Map<Integer, Integer> intMap = builder.getMutableInt32ToInt32Field();
+    intMap.put(1, 2);
+    assertEquals(newMap(1, 2), builder.build().getInt32ToInt32Field());
+    try {
+      intMap.put(2, 3);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    assertEquals(newMap(1, 2), builder.getInt32ToInt32Field());
+    builder.getMutableInt32ToInt32Field().put(2, 3);
+    assertEquals(newMap(1, 2, 2, 3), builder.getInt32ToInt32Field());
+
+    Map<Integer, TestMap.EnumValue> enumMap = builder.getMutableInt32ToEnumField();
+    enumMap.put(1, TestMap.EnumValue.BAR);
+    assertEquals(newMap(1, TestMap.EnumValue.BAR), builder.build().getInt32ToEnumField());
+    try {
+      enumMap.put(2, TestMap.EnumValue.FOO);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    assertEquals(newMap(1, TestMap.EnumValue.BAR), builder.getInt32ToEnumField());
+    builder.getMutableInt32ToEnumField().put(2, TestMap.EnumValue.FOO);
+    assertEquals(
+        newMap(1, TestMap.EnumValue.BAR, 2, TestMap.EnumValue.FOO),
+        builder.getInt32ToEnumField());
+    
+    Map<Integer, String> stringMap = builder.getMutableInt32ToStringField();
+    stringMap.put(1, "1");
+    assertEquals(newMap(1, "1"), builder.build().getInt32ToStringField());
+    try {
+      stringMap.put(2, "2");
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    assertEquals(newMap(1, "1"), builder.getInt32ToStringField());
+    builder.getMutableInt32ToStringField().put(2, "2");
+    assertEquals(
+        newMap(1, "1", 2, "2"),
+        builder.getInt32ToStringField());
+    
+    Map<Integer, TestMap.MessageValue> messageMap = builder.getMutableInt32ToMessageField();
+    messageMap.put(1, TestMap.MessageValue.getDefaultInstance());
+    assertEquals(newMap(1, TestMap.MessageValue.getDefaultInstance()),
+        builder.build().getInt32ToMessageField());
+    try {
+      messageMap.put(2, TestMap.MessageValue.getDefaultInstance());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    assertEquals(newMap(1, TestMap.MessageValue.getDefaultInstance()),
+        builder.getInt32ToMessageField());
+    builder.getMutableInt32ToMessageField().put(2, TestMap.MessageValue.getDefaultInstance());
+    assertEquals(
+        newMap(1, TestMap.MessageValue.getDefaultInstance(),
+            2, TestMap.MessageValue.getDefaultInstance()),
+        builder.getInt32ToMessageField());
+  }
+
+  public void testMutableMapLifecycle_collections() {
+    TestMap.Builder builder = TestMap.newBuilder();
+    Map<Integer, Integer> intMap = builder.getMutableInt32ToInt32Field();
+    intMap.put(1, 2);
+    assertEquals(newMap(1, 2), builder.build().getInt32ToInt32Field());
+    try {
+      intMap.remove(2);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    try {
+      intMap.entrySet().remove(new Object());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    try {
+      intMap.entrySet().iterator().remove();
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    try {
+      intMap.keySet().remove(new Object());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    try {
+      intMap.values().remove(new Object());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    try {
+      intMap.values().iterator().remove();
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    assertEquals(newMap(1, 2), intMap);
+    assertEquals(newMap(1, 2), builder.getInt32ToInt32Field());
+    assertEquals(newMap(1, 2), builder.build().getInt32ToInt32Field());
+  }
+
   public void testGettersAndSetters() throws Exception {
     TestMap.Builder builder = TestMap.newBuilder();
     TestMap message = builder.build();
@@ -274,4 +413,26 @@ public class MapForProto2LiteTest extends TestCase {
     assertEquals(54321, messageWithUnknownEnums.getInt32ToInt32Field().get(2).intValue());
   }
   
+
+  public void testIterationOrder() throws Exception {
+    TestMap.Builder builder = TestMap.newBuilder();
+    setMapValues(builder);
+    TestMap message = builder.build();
+
+    assertEquals(Arrays.asList("1", "2", "3"),
+        new ArrayList<String>(message.getStringToInt32Field().keySet()));
+  }
+  
+  private static <K, V> Map<K, V> newMap(K key1, V value1) {
+    Map<K, V> map = new HashMap<K, V>();
+    map.put(key1, value1);
+    return map;
+  }
+  
+  private static <K, V> Map<K, V> newMap(K key1, V value1, K key2, V value2) {
+    Map<K, V> map = new HashMap<K, V>();
+    map.put(key1, value1);
+    map.put(key2, value2);
+    return map;
+  }
 }

java/src/test/java/com/google/protobuf/MapForProto2Test.java

@@ -40,6 +40,7 @@ import map_test.MapForProto2TestProto.TestUnknownEnumValue;
 import junit.framework.TestCase;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -177,6 +178,116 @@ public class MapForProto2Test extends TestCase {
     assertEquals(0, message.getInt32ToMessageField().size());
     assertEquals(0, message.getStringToInt32Field().size());
   }
+  
+  public void testMutableMapLifecycle() {
+    TestMap.Builder builder = TestMap.newBuilder();
+    Map<Integer, Integer> intMap = builder.getMutableInt32ToInt32Field();
+    intMap.put(1, 2);
+    assertEquals(newMap(1, 2), builder.build().getInt32ToInt32Field());
+    try {
+      intMap.put(2, 3);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    assertEquals(newMap(1, 2), builder.getInt32ToInt32Field());
+    builder.getMutableInt32ToInt32Field().put(2, 3);
+    assertEquals(newMap(1, 2, 2, 3), builder.getInt32ToInt32Field());
+
+    Map<Integer, TestMap.EnumValue> enumMap = builder.getMutableInt32ToEnumField();
+    enumMap.put(1, TestMap.EnumValue.BAR);
+    assertEquals(newMap(1, TestMap.EnumValue.BAR), builder.build().getInt32ToEnumField());
+    try {
+      enumMap.put(2, TestMap.EnumValue.FOO);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    assertEquals(newMap(1, TestMap.EnumValue.BAR), builder.getInt32ToEnumField());
+    builder.getMutableInt32ToEnumField().put(2, TestMap.EnumValue.FOO);
+    assertEquals(
+        newMap(1, TestMap.EnumValue.BAR, 2, TestMap.EnumValue.FOO),
+        builder.getInt32ToEnumField());
+    
+    Map<Integer, String> stringMap = builder.getMutableInt32ToStringField();
+    stringMap.put(1, "1");
+    assertEquals(newMap(1, "1"), builder.build().getInt32ToStringField());
+    try {
+      stringMap.put(2, "2");
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    assertEquals(newMap(1, "1"), builder.getInt32ToStringField());
+    builder.getMutableInt32ToStringField().put(2, "2");
+    assertEquals(
+        newMap(1, "1", 2, "2"),
+        builder.getInt32ToStringField());
+    
+    Map<Integer, TestMap.MessageValue> messageMap = builder.getMutableInt32ToMessageField();
+    messageMap.put(1, TestMap.MessageValue.getDefaultInstance());
+    assertEquals(newMap(1, TestMap.MessageValue.getDefaultInstance()),
+        builder.build().getInt32ToMessageField());
+    try {
+      messageMap.put(2, TestMap.MessageValue.getDefaultInstance());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    assertEquals(newMap(1, TestMap.MessageValue.getDefaultInstance()),
+        builder.getInt32ToMessageField());
+    builder.getMutableInt32ToMessageField().put(2, TestMap.MessageValue.getDefaultInstance());
+    assertEquals(
+        newMap(1, TestMap.MessageValue.getDefaultInstance(),
+            2, TestMap.MessageValue.getDefaultInstance()),
+        builder.getInt32ToMessageField());
+  }
+
+  public void testMutableMapLifecycle_collections() {
+    TestMap.Builder builder = TestMap.newBuilder();
+    Map<Integer, Integer> intMap = builder.getMutableInt32ToInt32Field();
+    intMap.put(1, 2);
+    assertEquals(newMap(1, 2), builder.build().getInt32ToInt32Field());
+    try {
+      intMap.remove(2);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    try {
+      intMap.entrySet().remove(new Object());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    try {
+      intMap.entrySet().iterator().remove();
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    try {
+      intMap.keySet().remove(new Object());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    try {
+      intMap.values().remove(new Object());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    try {
+      intMap.values().iterator().remove();
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    assertEquals(newMap(1, 2), intMap);
+    assertEquals(newMap(1, 2), builder.getInt32ToInt32Field());
+    assertEquals(newMap(1, 2), builder.build().getInt32ToInt32Field());
+  }
 
   public void testGettersAndSetters() throws Exception {
     TestMap.Builder builder = TestMap.newBuilder();
@@ -513,4 +624,41 @@ public class MapForProto2Test extends TestCase {
     assertEquals(2, message.getRecursiveMapField().get(1).getValue());
     assertEquals(4, message.getRecursiveMapField().get(3).getValue());
   }
+
+  public void testIterationOrder() throws Exception {
+    TestMap.Builder builder = TestMap.newBuilder();
+    setMapValues(builder);
+    TestMap message = builder.build();
+
+    assertEquals(Arrays.asList("1", "2", "3"),
+        new ArrayList<String>(message.getStringToInt32Field().keySet()));
+  }
+
+  // Regression test for b/20494788
+  public void testMapInitializationOrder() throws Exception {
+    assertEquals("RedactAllTypes", map_test.RedactAllTypes
+        .getDefaultInstance().getDescriptorForType().getName());
+
+    map_test.Message1.Builder builder =
+        map_test.Message1.newBuilder();
+    builder.getMutableMapField().put("key", true);
+    map_test.Message1 message = builder.build();
+    Message mapEntry = (Message) message.getRepeatedField(
+        message.getDescriptorForType().findFieldByName("map_field"), 0);
+    assertEquals(2, mapEntry.getAllFields().size());
+  }
+  
+  private static <K, V> Map<K, V> newMap(K key1, V value1) {
+    Map<K, V> map = new HashMap<K, V>();
+    map.put(key1, value1);
+    return map;
+  }
+  
+  private static <K, V> Map<K, V> newMap(K key1, V value1, K key2, V value2) {
+    Map<K, V> map = new HashMap<K, V>();
+    map.put(key1, value1);
+    map.put(key2, value2);
+    return map;
+  }
 }
+

java/src/test/java/com/google/protobuf/MapTest.java

@@ -41,6 +41,7 @@ import map_test.MapTestProto.TestOnChangeEventPropagation;
 import junit.framework.TestCase;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -178,7 +179,117 @@ public class MapTest extends TestCase {
     assertEquals(0, message.getInt32ToMessageField().size());
     assertEquals(0, message.getStringToInt32Field().size());
   }
+  
+  public void testMutableMapLifecycle() {
+    TestMap.Builder builder = TestMap.newBuilder();
+    Map<Integer, Integer> intMap = builder.getMutableInt32ToInt32Field();
+    intMap.put(1, 2);
+    assertEquals(newMap(1, 2), builder.build().getInt32ToInt32Field());
+    try {
+      intMap.put(2, 3);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    assertEquals(newMap(1, 2), builder.getInt32ToInt32Field());
+    builder.getMutableInt32ToInt32Field().put(2, 3);
+    assertEquals(newMap(1, 2, 2, 3), builder.getInt32ToInt32Field());
+
+    Map<Integer, TestMap.EnumValue> enumMap = builder.getMutableInt32ToEnumField();
+    enumMap.put(1, TestMap.EnumValue.BAR);
+    assertEquals(newMap(1, TestMap.EnumValue.BAR), builder.build().getInt32ToEnumField());
+    try {
+      enumMap.put(2, TestMap.EnumValue.FOO);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    assertEquals(newMap(1, TestMap.EnumValue.BAR), builder.getInt32ToEnumField());
+    builder.getMutableInt32ToEnumField().put(2, TestMap.EnumValue.FOO);
+    assertEquals(
+        newMap(1, TestMap.EnumValue.BAR, 2, TestMap.EnumValue.FOO),
+        builder.getInt32ToEnumField());
+    
+    Map<Integer, String> stringMap = builder.getMutableInt32ToStringField();
+    stringMap.put(1, "1");
+    assertEquals(newMap(1, "1"), builder.build().getInt32ToStringField());
+    try {
+      stringMap.put(2, "2");
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    assertEquals(newMap(1, "1"), builder.getInt32ToStringField());
+    builder.getMutableInt32ToStringField().put(2, "2");
+    assertEquals(
+        newMap(1, "1", 2, "2"),
+        builder.getInt32ToStringField());
+    
+    Map<Integer, TestMap.MessageValue> messageMap = builder.getMutableInt32ToMessageField();
+    messageMap.put(1, TestMap.MessageValue.getDefaultInstance());
+    assertEquals(newMap(1, TestMap.MessageValue.getDefaultInstance()),
+        builder.build().getInt32ToMessageField());
+    try {
+      messageMap.put(2, TestMap.MessageValue.getDefaultInstance());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    assertEquals(newMap(1, TestMap.MessageValue.getDefaultInstance()),
+        builder.getInt32ToMessageField());
+    builder.getMutableInt32ToMessageField().put(2, TestMap.MessageValue.getDefaultInstance());
+    assertEquals(
+        newMap(1, TestMap.MessageValue.getDefaultInstance(),
+            2, TestMap.MessageValue.getDefaultInstance()),
+        builder.getInt32ToMessageField());
+  }
 
+  public void testMutableMapLifecycle_collections() {
+    TestMap.Builder builder = TestMap.newBuilder();
+    Map<Integer, Integer> intMap = builder.getMutableInt32ToInt32Field();
+    intMap.put(1, 2);
+    assertEquals(newMap(1, 2), builder.build().getInt32ToInt32Field());
+    try {
+      intMap.remove(2);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    try {
+      intMap.entrySet().remove(new Object());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    try {
+      intMap.entrySet().iterator().remove();
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    try {
+      intMap.keySet().remove(new Object());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    try {
+      intMap.values().remove(new Object());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    try {
+      intMap.values().iterator().remove();
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    assertEquals(newMap(1, 2), intMap);
+    assertEquals(newMap(1, 2), builder.getInt32ToInt32Field());
+    assertEquals(newMap(1, 2), builder.build().getInt32ToInt32Field());
+  }
+  
   public void testGettersAndSetters() throws Exception {
     TestMap.Builder builder = TestMap.newBuilder();
     TestMap message = builder.build();
@@ -611,4 +722,26 @@ public class MapTest extends TestCase {
       assertEquals(data.get(entry.getKey()) + 1, entry.getValue().intValue());
     }
   }
+
+  public void testIterationOrder() throws Exception {
+    TestMap.Builder builder = TestMap.newBuilder();
+    setMapValues(builder);
+    TestMap message = builder.build();
+
+    assertEquals(Arrays.asList("1", "2", "3"),
+        new ArrayList<String>(message.getStringToInt32Field().keySet()));
+  }
+  
+  private static <K, V> Map<K, V> newMap(K key1, V value1) {
+    Map<K, V> map = new HashMap<K, V>();
+    map.put(key1, value1);
+    return map;
+  }
+  
+  private static <K, V> Map<K, V> newMap(K key1, V value1, K key2, V value2) {
+    Map<K, V> map = new HashMap<K, V>();
+    map.put(key1, value1);
+    map.put(key2, value2);
+    return map;
+  }
 }

java/src/test/java/com/google/protobuf/ProtobufArrayListTest.java

@@ -0,0 +1,303 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package com.google.protobuf;
+
+import static java.util.Arrays.asList;
+
+import junit.framework.TestCase;
+
+import java.util.Collections;
+import java.util.ConcurrentModificationException;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * Tests for {@link ProtobufArrayList}.
+ */
+public class ProtobufArrayListTest extends TestCase {
+  
+  private static final ProtobufArrayList<Integer> UNARY_LIST = newImmutableProtoArrayList(1);
+  private static final ProtobufArrayList<Integer> TERTIARY_LIST =
+      newImmutableProtoArrayList(1, 2, 3);
+  
+  private ProtobufArrayList<Integer> list;
+  
+  @Override
+  protected void setUp() throws Exception {
+    list = new ProtobufArrayList<Integer>();
+  }
+  
+  public void testEmptyListReturnsSameInstance() {
+    assertSame(ProtobufArrayList.emptyList(), ProtobufArrayList.emptyList());
+  }
+  
+  public void testEmptyListIsImmutable() {
+    assertImmutable(ProtobufArrayList.<Integer>emptyList());
+  }
+  
+  public void testCopyConstructor() {
+    ProtobufArrayList<Integer> copy = new ProtobufArrayList<Integer>(TERTIARY_LIST);
+    assertEquals(TERTIARY_LIST, copy);
+
+    copy = new ProtobufArrayList<Integer>(IntArrayList.emptyList());
+    assertEquals(ProtobufArrayList.emptyList(), copy);
+    
+    copy = new ProtobufArrayList<Integer>(asList(1, 2, 3));
+    assertEquals(asList(1, 2, 3), copy);
+
+    copy = new ProtobufArrayList<Integer>(Collections.<Integer>emptyList());
+    assertEquals(ProtobufArrayList.emptyList(), copy);
+  }
+  
+  public void testModificationWithIteration() {
+    list.addAll(asList(1, 2, 3, 4));
+    Iterator<Integer> iterator = list.iterator();
+    assertEquals(4, list.size());
+    assertEquals(1, (int) list.get(0));
+    assertEquals(1, (int) iterator.next());
+    
+    list.remove(0);
+    try {
+      iterator.next();
+      fail();
+    } catch (ConcurrentModificationException e) {
+      // expected
+    }
+    
+    iterator = list.iterator();
+    list.set(0, 1);
+    try {
+      iterator.next();
+      fail();
+    } catch (ConcurrentModificationException e) {
+      // expected
+    }
+    
+    iterator = list.iterator();
+    list.add(0, 0);
+    try {
+      iterator.next();
+      fail();
+    } catch (ConcurrentModificationException e) {
+      // expected
+    }
+  }
+  
+  public void testMakeImmutable() {
+    list.add(2);
+    list.add(4);
+    list.add(6);
+    list.add(8);
+    list.makeImmutable();
+    assertImmutable(list);
+  }
+  
+  public void testRemove() {
+    list.add(2);
+    list.add(4);
+    list.add(6);
+
+    list.remove(1);
+    assertEquals(asList(2, 6), list);
+
+    list.remove(1);
+    assertEquals(asList(2), list);
+
+    list.remove(0);
+    assertEquals(asList(), list);
+  }
+  
+  public void testGet() {
+    list.add(2);
+    list.add(6);
+    
+    assertEquals(2, (int) list.get(0));
+    assertEquals(6, (int) list.get(1));
+  }
+  
+  public void testSet() {
+    list.add(2);
+    list.add(6);
+    
+    list.set(0, 1);
+    assertEquals(1, (int) list.get(0));
+    list.set(1, 2);
+    assertEquals(2, (int) list.get(1));
+  }
+
+  private void assertImmutable(List<Integer> list) {
+    if (list.contains(1)) {
+      throw new RuntimeException("Cannot test the immutability of lists that contain 1.");
+    }
+    
+    try {
+      list.add(1);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.add(0, 1);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(Collections.<Integer>emptyList());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(Collections.singletonList(1));
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(new ProtobufArrayList<Integer>());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(UNARY_LIST);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(0, Collections.singleton(1));
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(0, UNARY_LIST);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.addAll(0, Collections.<Integer>emptyList());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    } 
+
+    try {
+      list.clear();
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+
+    try {
+      list.remove(1);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.remove(new Object());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.removeAll(Collections.<Double>emptyList());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.removeAll(Collections.singleton(1));
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.removeAll(UNARY_LIST);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.retainAll(Collections.<Double>emptyList());
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.retainAll(Collections.singleton(1));
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.retainAll(UNARY_LIST);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+    
+    try {
+      list.set(0, 0);
+      fail();
+    } catch (UnsupportedOperationException e) {
+      // expected
+    }
+  }
+  
+  private static ProtobufArrayList<Integer> newImmutableProtoArrayList(int... elements) {
+    ProtobufArrayList<Integer> list = new ProtobufArrayList<Integer>();
+    for (int element : elements) {
+      list.add(element);
+    }
+    list.makeImmutable();
+    return list;
+  }
+}

java/src/test/java/com/google/protobuf/field_presence_test.proto

@@ -45,15 +45,15 @@ message TestAllTypes {
     BAZ = 2;
   }
   message NestedMessage {
-    optional int32 value = 1;
+    int32 value = 1;
   }
 
-  optional int32 optional_int32 = 1;
-  optional string optional_string = 2;
-  optional bytes optional_bytes = 3;
-  optional NestedEnum optional_nested_enum = 4;
-  optional NestedMessage optional_nested_message = 5;
-  optional protobuf_unittest.TestRequired optional_proto2_message = 6;
+  int32 optional_int32 = 1;
+  string optional_string = 2;
+  bytes optional_bytes = 3;
+  NestedEnum optional_nested_enum = 4;
+  NestedMessage optional_nested_message = 5;
+  protobuf_unittest.TestRequired optional_proto2_message = 6;
 
   oneof oneof_field {
     int32 oneof_int32 = 11;
@@ -75,12 +75,12 @@ message TestAllTypes {
 }
 
 message TestOptionalFieldsOnly {
-  optional int32 optional_int32 = 1;
-  optional string optional_string = 2;
-  optional bytes optional_bytes = 3;
-  optional TestAllTypes.NestedEnum optional_nested_enum = 4;
-  optional TestAllTypes.NestedMessage optional_nested_message = 5;
-  optional protobuf_unittest.TestRequired optional_proto2_message = 6;
+  int32 optional_int32 = 1;
+  string optional_string = 2;
+  bytes optional_bytes = 3;
+  TestAllTypes.NestedEnum optional_nested_enum = 4;
+  TestAllTypes.NestedMessage optional_nested_message = 5;
+  protobuf_unittest.TestRequired optional_proto2_message = 6;
 }
 
 message TestRepeatedFieldsOnly {

java/src/test/java/com/google/protobuf/map_initialization_order_test.proto

@@ -0,0 +1,61 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Regression test for a map initilaization order bug. The bug only manifests
+// when:
+//   1. A message contains map fields and is also extendable.
+//   2. There is a file-level extension defined in the same file referencing
+//      the above message as the extension type.
+//   3. The program executes in the following order:
+//        a. getDescriptor() is called on another message in the same file.
+//        b. use protobuf reflection to access the map field.
+// The symptom is a NullPointerException being thrown.
+syntax = "proto2";
+
+package map_test;
+
+option java_package = "map_test";
+option java_outer_classname = "MapInitializationOrderTest";
+option java_multiple_files = true;
+
+// Mirrors the structure of
+// javatests/com/google/cloud/common/logging/logging_test.proto.
+
+message Message1 {
+  map<string, bool> map_field = 1;
+  extensions 1000 to max;
+}
+
+extend Message1 {
+  optional Message1 recursive_extension = 1001;
+}
+
+message RedactAllTypes {
+}

java/src/test/java/com/google/protobuf/map_test.proto

@@ -39,7 +39,7 @@ option java_generate_equals_and_hash = true;
 
 message TestMap {
   message MessageValue {
-    optional int32 value = 1;
+    int32 value = 1;
   }
   enum EnumValue {
     FOO = 0;
@@ -60,5 +60,5 @@ message TestMap {
 // propagate the onChange event and mark its parent dirty when a change
 // is made to a map field.
 message TestOnChangeEventPropagation {
-  optional TestMap optional_message = 1;
+  TestMap optional_message = 1;
 }

python/google/protobuf/descriptor.py

@@ -245,9 +245,6 @@ class Descriptor(_NestedDescriptorBase):
 
     is_extendable:  Does this type define any extension ranges?
 
-    options: (descriptor_pb2.MessageOptions) Protocol message options or None
-      to use default message options.
-
     oneofs: (list of OneofDescriptor) The list of descriptors for oneof fields
       in this message.
     oneofs_by_name: (dict str -> OneofDescriptor) Same objects as in |oneofs|,
@@ -265,7 +262,7 @@ class Descriptor(_NestedDescriptorBase):
                 file=None, serialized_start=None, serialized_end=None,
                 syntax=None):
       _message.Message._CheckCalledFromGeneratedFile()
-      return _message.Message._GetMessageDescriptor(full_name)
+      return _message.default_pool.FindMessageTypeByName(full_name)
 
   # NOTE(tmarek): The file argument redefining a builtin is nothing we can
   # fix right now since we don't know how many clients already rely on the
@@ -495,9 +492,9 @@ class FieldDescriptor(DescriptorBase):
                 has_default_value=True, containing_oneof=None):
       _message.Message._CheckCalledFromGeneratedFile()
       if is_extension:
-        return _message.Message._GetExtensionDescriptor(full_name)
+        return _message.default_pool.FindExtensionByName(full_name)
       else:
-        return _message.Message._GetFieldDescriptor(full_name)
+        return _message.default_pool.FindFieldByName(full_name)
 
   def __init__(self, name, full_name, index, number, type, cpp_type, label,
                default_value, message_type, enum_type, containing_type,
@@ -528,14 +525,9 @@ class FieldDescriptor(DescriptorBase):
     self.containing_oneof = containing_oneof
     if api_implementation.Type() == 'cpp':
       if is_extension:
-        # pylint: disable=protected-access
-        self._cdescriptor = (
-            _message.Message._GetExtensionDescriptor(full_name))
-        # pylint: enable=protected-access
+        self._cdescriptor = _message.default_pool.FindExtensionByName(full_name)
       else:
-        # pylint: disable=protected-access
-        self._cdescriptor = _message.Message._GetFieldDescriptor(full_name)
-        # pylint: enable=protected-access
+        self._cdescriptor = _message.default_pool.FindFieldByName(full_name)
     else:
       self._cdescriptor = None
 
@@ -592,7 +584,7 @@ class EnumDescriptor(_NestedDescriptorBase):
                 containing_type=None, options=None, file=None,
                 serialized_start=None, serialized_end=None):
       _message.Message._CheckCalledFromGeneratedFile()
-      return _message.Message._GetEnumDescriptor(full_name)
+      return _message.default_pool.FindEnumTypeByName(full_name)
 
   def __init__(self, name, full_name, filename, values,
                containing_type=None, options=None, file=None,
@@ -677,7 +669,7 @@ class OneofDescriptor(object):
 
     def __new__(cls, name, full_name, index, containing_type, fields):
       _message.Message._CheckCalledFromGeneratedFile()
-      return _message.Message._GetOneofDescriptor(full_name)
+      return _message.default_pool.FindOneofByName(full_name)
 
   def __init__(self, name, full_name, index, containing_type, fields):
     """Arguments are as described in the attribute description above."""
@@ -788,12 +780,8 @@ class FileDescriptor(DescriptorBase):
                 dependencies=None, syntax=None):
       # FileDescriptor() is called from various places, not only from generated
       # files, to register dynamic proto files and messages.
-      # TODO(amauryfa): Expose BuildFile() as a public function and make this
-      # constructor an implementation detail.
       if serialized_pb:
-        # pylint: disable=protected-access2
-        return _message.Message._BuildFile(serialized_pb)
-        # pylint: enable=protected-access
+        return _message.default_pool.AddSerializedFile(serialized_pb)
       else:
         return super(FileDescriptor, cls).__new__(cls)
 
@@ -814,9 +802,7 @@ class FileDescriptor(DescriptorBase):
 
     if (api_implementation.Type() == 'cpp' and
         self.serialized_pb is not None):
-      # pylint: disable=protected-access
-      _message.Message._BuildFile(self.serialized_pb)
-      # pylint: enable=protected-access
+      _message.default_pool.AddSerializedFile(self.serialized_pb)
 
   def CopyToProto(self, proto):
     """Copies this to a descriptor_pb2.FileDescriptorProto.
@@ -864,10 +850,10 @@ def MakeDescriptor(desc_proto, package='', build_file_if_cpp=True,
     file_descriptor_proto = descriptor_pb2.FileDescriptorProto()
     file_descriptor_proto.message_type.add().MergeFrom(desc_proto)
 
-    # Generate a random name for this proto file to prevent conflicts with
-    # any imported ones. We need to specify a file name so BuildFile accepts
-    # our FileDescriptorProto, but it is not important what that file name
-    # is actually set to.
+    # Generate a random name for this proto file to prevent conflicts with any
+    # imported ones. We need to specify a file name so the descriptor pool
+    # accepts our FileDescriptorProto, but it is not important what that file
+    # name is actually set to.
     proto_name = str(uuid.uuid4())
 
     if package:
@@ -877,10 +863,8 @@ def MakeDescriptor(desc_proto, package='', build_file_if_cpp=True,
     else:
       file_descriptor_proto.name = proto_name + '.proto'
 
-    # pylint: disable=protected-access
-    result = _message.Message._BuildFile(
-        file_descriptor_proto.SerializeToString())
-    # pylint: enable=protected-access
+    _message.default_pool.Add(file_descriptor_proto)
+    result = _message.default_pool.FindFileByName(file_descriptor_proto.name)
 
     if _USE_C_DESCRIPTORS:
       return result.message_types_by_name[desc_proto.name]

python/google/protobuf/descriptor_pool.py

@@ -113,6 +113,20 @@ class DescriptorPool(object):
 
     self._internal_db.Add(file_desc_proto)
 
+  def AddSerializedFile(self, serialized_file_desc_proto):
+    """Adds the FileDescriptorProto and its types to this pool.
+
+    Args:
+      serialized_file_desc_proto: A bytes string, serialization of the
+        FileDescriptorProto to add.
+    """
+
+    # pylint: disable=g-import-not-at-top
+    from google.protobuf import descriptor_pb2
+    file_desc_proto = descriptor_pb2.FileDescriptorProto.FromString(
+        serialized_file_desc_proto)
+    self.Add(file_desc_proto)
+
   def AddDescriptor(self, desc):
     """Adds a Descriptor to the pool, non-recursively.
 
@@ -320,17 +334,17 @@ class DescriptorPool(object):
                                           file_descriptor, None, scope))
 
         for index, extension_proto in enumerate(file_proto.extension):
-          extension_desc = self.MakeFieldDescriptor(
+          extension_desc = self._MakeFieldDescriptor(
               extension_proto, file_proto.package, index, is_extension=True)
           extension_desc.containing_type = self._GetTypeFromScope(
               file_descriptor.package, extension_proto.extendee, scope)
-          self.SetFieldType(extension_proto, extension_desc,
+          self._SetFieldType(extension_proto, extension_desc,
                             file_descriptor.package, scope)
           file_descriptor.extensions_by_name[extension_desc.name] = (
               extension_desc)
 
         for desc_proto in file_proto.message_type:
-          self.SetAllFieldTypes(file_proto.package, desc_proto, scope)
+          self._SetAllFieldTypes(file_proto.package, desc_proto, scope)
 
         if file_proto.package:
           desc_proto_prefix = _PrefixWithDot(file_proto.package)
@@ -381,10 +395,11 @@ class DescriptorPool(object):
     enums = [
         self._ConvertEnumDescriptor(enum, desc_name, file_desc, None, scope)
         for enum in desc_proto.enum_type]
-    fields = [self.MakeFieldDescriptor(field, desc_name, index)
+    fields = [self._MakeFieldDescriptor(field, desc_name, index)
               for index, field in enumerate(desc_proto.field)]
     extensions = [
-        self.MakeFieldDescriptor(extension, desc_name, index, is_extension=True)
+        self._MakeFieldDescriptor(extension, desc_name, index,
+                                  is_extension=True)
         for index, extension in enumerate(desc_proto.extension)]
     oneofs = [
         descriptor.OneofDescriptor(desc.name, '.'.join((desc_name, desc.name)),
@@ -464,8 +479,8 @@ class DescriptorPool(object):
     self._enum_descriptors[enum_name] = desc
     return desc
 
-  def MakeFieldDescriptor(self, field_proto, message_name, index,
-                          is_extension=False):
+  def _MakeFieldDescriptor(self, field_proto, message_name, index,
+                           is_extension=False):
     """Creates a field descriptor from a FieldDescriptorProto.
 
     For message and enum type fields, this method will do a look up
@@ -506,7 +521,7 @@ class DescriptorPool(object):
         extension_scope=None,
         options=field_proto.options)
 
-  def SetAllFieldTypes(self, package, desc_proto, scope):
+  def _SetAllFieldTypes(self, package, desc_proto, scope):
     """Sets all the descriptor's fields's types.
 
     This method also sets the containing types on any extensions.
@@ -527,18 +542,18 @@ class DescriptorPool(object):
       nested_package = '.'.join([package, desc_proto.name])
 
     for field_proto, field_desc in zip(desc_proto.field, main_desc.fields):
-      self.SetFieldType(field_proto, field_desc, nested_package, scope)
+      self._SetFieldType(field_proto, field_desc, nested_package, scope)
 
     for extension_proto, extension_desc in (
         zip(desc_proto.extension, main_desc.extensions)):
       extension_desc.containing_type = self._GetTypeFromScope(
           nested_package, extension_proto.extendee, scope)
-      self.SetFieldType(extension_proto, extension_desc, nested_package, scope)
+      self._SetFieldType(extension_proto, extension_desc, nested_package, scope)
 
     for nested_type in desc_proto.nested_type:
-      self.SetAllFieldTypes(nested_package, nested_type, scope)
+      self._SetAllFieldTypes(nested_package, nested_type, scope)
 
-  def SetFieldType(self, field_proto, field_desc, package, scope):
+  def _SetFieldType(self, field_proto, field_desc, package, scope):
     """Sets the field's type, cpp_type, message_type and enum_type.
 
     Args:

python/google/protobuf/internal/containers.py

@@ -41,6 +41,146 @@ are:
 
 __author__ = 'petar@google.com (Petar Petrov)'
 
+import sys
+
+if sys.version_info[0] < 3:
+  # We would use collections.MutableMapping all the time, but in Python 2 it
+  # doesn't define __slots__.  This causes two significant problems:
+  #
+  # 1. we can't disallow arbitrary attribute assignment, even if our derived
+  #    classes *do* define __slots__.
+  #
+  # 2. we can't safely derive a C type from it without __slots__ defined (the
+  #    interpreter expects to find a dict at tp_dictoffset, which we can't
+  #    robustly provide.  And we don't want an instance dict anyway.
+  #
+  # So this is the Python 2.7 definition of Mapping/MutableMapping functions
+  # verbatim, except that:
+  # 1. We declare __slots__.
+  # 2. We don't declare this as a virtual base class.  The classes defined
+  #    in collections are the interesting base classes, not us.
+  #
+  # Note: deriving from object is critical.  It is the only thing that makes
+  # this a true type, allowing us to derive from it in C++ cleanly and making
+  # __slots__ properly disallow arbitrary element assignment.
+  from collections import Mapping as _Mapping
+
+  class Mapping(object):
+    __slots__ = ()
+
+    def get(self, key, default=None):
+      try:
+        return self[key]
+      except KeyError:
+        return default
+
+    def __contains__(self, key):
+      try:
+        self[key]
+      except KeyError:
+        return False
+      else:
+        return True
+
+    def iterkeys(self):
+      return iter(self)
+
+    def itervalues(self):
+      for key in self:
+        yield self[key]
+
+    def iteritems(self):
+      for key in self:
+        yield (key, self[key])
+
+    def keys(self):
+      return list(self)
+
+    def items(self):
+      return [(key, self[key]) for key in self]
+
+    def values(self):
+      return [self[key] for key in self]
+
+    # Mappings are not hashable by default, but subclasses can change this
+    __hash__ = None
+
+    def __eq__(self, other):
+      if not isinstance(other, _Mapping):
+        return NotImplemented
+      return dict(self.items()) == dict(other.items())
+
+    def __ne__(self, other):
+      return not (self == other)
+
+  class MutableMapping(Mapping):
+    __slots__ = ()
+
+    __marker = object()
+
+    def pop(self, key, default=__marker):
+      try:
+        value = self[key]
+      except KeyError:
+        if default is self.__marker:
+          raise
+        return default
+      else:
+        del self[key]
+        return value
+
+    def popitem(self):
+      try:
+        key = next(iter(self))
+      except StopIteration:
+        raise KeyError
+      value = self[key]
+      del self[key]
+      return key, value
+
+    def clear(self):
+      try:
+        while True:
+          self.popitem()
+      except KeyError:
+        pass
+
+    def update(*args, **kwds):
+      if len(args) > 2:
+        raise TypeError("update() takes at most 2 positional "
+                        "arguments ({} given)".format(len(args)))
+      elif not args:
+        raise TypeError("update() takes at least 1 argument (0 given)")
+      self = args[0]
+      other = args[1] if len(args) >= 2 else ()
+
+      if isinstance(other, Mapping):
+        for key in other:
+          self[key] = other[key]
+      elif hasattr(other, "keys"):
+        for key in other.keys():
+          self[key] = other[key]
+      else:
+        for key, value in other:
+          self[key] = value
+      for key, value in kwds.items():
+        self[key] = value
+
+    def setdefault(self, key, default=None):
+      try:
+        return self[key]
+      except KeyError:
+        self[key] = default
+      return default
+
+  _Mapping.register(Mapping)
+
+else:
+  # In Python 3 we can just use MutableMapping directly, because it defines
+  # __slots__.
+  from collections import MutableMapping
+
+
 class BaseContainer(object):
 
   """Base container class."""
@@ -286,3 +426,160 @@ class RepeatedCompositeFieldContainer(BaseContainer):
       raise TypeError('Can only compare repeated composite fields against '
                       'other repeated composite fields.')
     return self._values == other._values
+
+
+class ScalarMap(MutableMapping):
+
+  """Simple, type-checked, dict-like container for holding repeated scalars."""
+
+  # Disallows assignment to other attributes.
+  __slots__ = ['_key_checker', '_value_checker', '_values', '_message_listener']
+
+  def __init__(self, message_listener, key_checker, value_checker):
+    """
+    Args:
+      message_listener: A MessageListener implementation.
+        The ScalarMap will call this object's Modified() method when it
+        is modified.
+      key_checker: A type_checkers.ValueChecker instance to run on keys
+        inserted into this container.
+      value_checker: A type_checkers.ValueChecker instance to run on values
+        inserted into this container.
+    """
+    self._message_listener = message_listener
+    self._key_checker = key_checker
+    self._value_checker = value_checker
+    self._values = {}
+
+  def __getitem__(self, key):
+    try:
+      return self._values[key]
+    except KeyError:
+      key = self._key_checker.CheckValue(key)
+      val = self._value_checker.DefaultValue()
+      self._values[key] = val
+      return val
+
+  def __contains__(self, item):
+    return item in self._values
+
+  # We need to override this explicitly, because our defaultdict-like behavior
+  # will make the default implementation (from our base class) always insert
+  # the key.
+  def get(self, key, default=None):
+    if key in self:
+      return self[key]
+    else:
+      return default
+
+  def __setitem__(self, key, value):
+    checked_key = self._key_checker.CheckValue(key)
+    checked_value = self._value_checker.CheckValue(value)
+    self._values[checked_key] = checked_value
+    self._message_listener.Modified()
+
+  def __delitem__(self, key):
+    del self._values[key]
+    self._message_listener.Modified()
+
+  def __len__(self):
+    return len(self._values)
+
+  def __iter__(self):
+    return iter(self._values)
+
+  def MergeFrom(self, other):
+    self._values.update(other._values)
+    self._message_listener.Modified()
+
+  # This is defined in the abstract base, but we can do it much more cheaply.
+  def clear(self):
+    self._values.clear()
+    self._message_listener.Modified()
+
+
+class MessageMap(MutableMapping):
+
+  """Simple, type-checked, dict-like container for with submessage values."""
+
+  # Disallows assignment to other attributes.
+  __slots__ = ['_key_checker', '_values', '_message_listener',
+               '_message_descriptor']
+
+  def __init__(self, message_listener, message_descriptor, key_checker):
+    """
+    Args:
+      message_listener: A MessageListener implementation.
+        The ScalarMap will call this object's Modified() method when it
+        is modified.
+      key_checker: A type_checkers.ValueChecker instance to run on keys
+        inserted into this container.
+      value_checker: A type_checkers.ValueChecker instance to run on values
+        inserted into this container.
+    """
+    self._message_listener = message_listener
+    self._message_descriptor = message_descriptor
+    self._key_checker = key_checker
+    self._values = {}
+
+  def __getitem__(self, key):
+    try:
+      return self._values[key]
+    except KeyError:
+      key = self._key_checker.CheckValue(key)
+      new_element = self._message_descriptor._concrete_class()
+      new_element._SetListener(self._message_listener)
+      self._values[key] = new_element
+      self._message_listener.Modified()
+
+      return new_element
+
+  def get_or_create(self, key):
+    """get_or_create() is an alias for getitem (ie. map[key]).
+
+    Args:
+      key: The key to get or create in the map.
+
+    This is useful in cases where you want to be explicit that the call is
+    mutating the map.  This can avoid lint errors for statements like this
+    that otherwise would appear to be pointless statements:
+
+      msg.my_map[key]
+    """
+    return self[key]
+
+  # We need to override this explicitly, because our defaultdict-like behavior
+  # will make the default implementation (from our base class) always insert
+  # the key.
+  def get(self, key, default=None):
+    if key in self:
+      return self[key]
+    else:
+      return default
+
+  def __contains__(self, item):
+    return item in self._values
+
+  def __setitem__(self, key, value):
+    raise ValueError('May not set values directly, call my_map[key].foo = 5')
+
+  def __delitem__(self, key):
+    del self._values[key]
+    self._message_listener.Modified()
+
+  def __len__(self):
+    return len(self._values)
+
+  def __iter__(self):
+    return iter(self._values)
+
+  def MergeFrom(self, other):
+    for key in other:
+      self[key].MergeFrom(other[key])
+    # self._message_listener.Modified() not required here, because
+    # mutations to submessages already propagate.
+
+  # This is defined in the abstract base, but we can do it much more cheaply.
+  def clear(self):
+    self._values.clear()
+    self._message_listener.Modified()

python/google/protobuf/internal/decoder.py

@@ -732,6 +732,50 @@ def MessageSetItemDecoder(extensions_by_number):
 
   return DecodeItem
 
+# --------------------------------------------------------------------
+
+def MapDecoder(field_descriptor, new_default, is_message_map):
+  """Returns a decoder for a map field."""
+
+  key = field_descriptor
+  tag_bytes = encoder.TagBytes(field_descriptor.number,
+                               wire_format.WIRETYPE_LENGTH_DELIMITED)
+  tag_len = len(tag_bytes)
+  local_DecodeVarint = _DecodeVarint
+  # Can't read _concrete_class yet; might not be initialized.
+  message_type = field_descriptor.message_type
+
+  def DecodeMap(buffer, pos, end, message, field_dict):
+    submsg = message_type._concrete_class()
+    value = field_dict.get(key)
+    if value is None:
+      value = field_dict.setdefault(key, new_default(message))
+    while 1:
+      # Read length.
+      (size, pos) = local_DecodeVarint(buffer, pos)
+      new_pos = pos + size
+      if new_pos > end:
+        raise _DecodeError('Truncated message.')
+      # Read sub-message.
+      submsg.Clear()
+      if submsg._InternalParse(buffer, pos, new_pos) != new_pos:
+        # The only reason _InternalParse would return early is if it
+        # encountered an end-group tag.
+        raise _DecodeError('Unexpected end-group tag.')
+
+      if is_message_map:
+        value[submsg.key].MergeFrom(submsg.value)
+      else:
+        value[submsg.key] = submsg.value
+
+      # Predict that the next tag is another copy of the same repeated field.
+      pos = new_pos + tag_len
+      if buffer[new_pos:pos] != tag_bytes or new_pos == end:
+        # Prediction failed.  Return.
+        return new_pos
+
+  return DecodeMap
+
 # --------------------------------------------------------------------
 # Optimization is not as heavy here because calls to SkipField() are rare,
 # except for handling end-group tags.

python/google/protobuf/internal/descriptor_database_test.py

@@ -35,7 +35,6 @@
 __author__ = 'matthewtoia@google.com (Matt Toia)'
 
 import unittest
-
 from google.protobuf import descriptor_pb2
 from google.protobuf.internal import factory_test2_pb2
 from google.protobuf import descriptor_database

python/google/protobuf/internal/descriptor_pool_test.py

@@ -37,6 +37,7 @@ __author__ = 'matthewtoia@google.com (Matt Toia)'
 import os
 import unittest
 
+import unittest
 from google.protobuf import unittest_pb2
 from google.protobuf import descriptor_pb2
 from google.protobuf.internal import api_implementation
@@ -226,6 +227,13 @@ class DescriptorPoolTest(unittest.TestCase):
     db.Add(self.factory_test2_fd)
     self.testFindMessageTypeByName()
 
+  def testAddSerializedFile(self):
+    db = descriptor_database.DescriptorDatabase()
+    self.pool = descriptor_pool.DescriptorPool(db)
+    self.pool.AddSerializedFile(self.factory_test1_fd.SerializeToString())
+    self.pool.AddSerializedFile(self.factory_test2_fd.SerializeToString())
+    self.testFindMessageTypeByName()
+
   def testComplexNesting(self):
     test1_desc = descriptor_pb2.FileDescriptorProto.FromString(
         descriptor_pool_test1_pb2.DESCRIPTOR.serialized_pb)
@@ -510,6 +518,43 @@ class AddDescriptorTest(unittest.TestCase):
           'protobuf_unittest.TestAllTypes')
 
 
+@unittest.skipIf(
+    api_implementation.Type() != 'cpp',
+    'default_pool is only supported by the C++ implementation')
+class DefaultPoolTest(unittest.TestCase):
+
+  def testFindMethods(self):
+    # pylint: disable=g-import-not-at-top
+    from google.protobuf.pyext import _message
+    pool = _message.default_pool
+    self.assertIs(
+        pool.FindFileByName('google/protobuf/unittest.proto'),
+        unittest_pb2.DESCRIPTOR)
+    self.assertIs(
+        pool.FindMessageTypeByName('protobuf_unittest.TestAllTypes'),
+        unittest_pb2.TestAllTypes.DESCRIPTOR)
+    self.assertIs(
+        pool.FindFieldByName('protobuf_unittest.TestAllTypes.optional_int32'),
+        unittest_pb2.TestAllTypes.DESCRIPTOR.fields_by_name['optional_int32'])
+    self.assertIs(
+        pool.FindExtensionByName('protobuf_unittest.optional_int32_extension'),
+        unittest_pb2.DESCRIPTOR.extensions_by_name['optional_int32_extension'])
+    self.assertIs(
+        pool.FindEnumTypeByName('protobuf_unittest.ForeignEnum'),
+        unittest_pb2.ForeignEnum.DESCRIPTOR)
+    self.assertIs(
+        pool.FindOneofByName('protobuf_unittest.TestAllTypes.oneof_field'),
+        unittest_pb2.TestAllTypes.DESCRIPTOR.oneofs_by_name['oneof_field'])
+
+  def testAddFileDescriptor(self):
+    # pylint: disable=g-import-not-at-top
+    from google.protobuf.pyext import _message
+    pool = _message.default_pool
+    file_desc = descriptor_pb2.FileDescriptorProto(name='some/file.proto')
+    pool.Add(file_desc)
+    pool.AddSerializedFile(file_desc.SerializeToString())
+
+
 TEST1_FILE = ProtoFile(
     'google/protobuf/internal/descriptor_pool_test1.proto',
     'google.protobuf.python.internal',

python/google/protobuf/internal/descriptor_test.py

@@ -35,8 +35,8 @@
 __author__ = 'robinson@google.com (Will Robinson)'
 
 import sys
-import unittest
 
+import unittest
 from google.protobuf import unittest_custom_options_pb2
 from google.protobuf import unittest_import_pb2
 from google.protobuf import unittest_pb2

python/google/protobuf/internal/encoder.py

@@ -314,7 +314,7 @@ def MessageSizer(field_number, is_repeated, is_packed):
 
 
 # --------------------------------------------------------------------
-# MessageSet is special.
+# MessageSet is special: it needs custom logic to compute its size properly.
 
 
 def MessageSetItemSizer(field_number):
@@ -339,6 +339,32 @@ def MessageSetItemSizer(field_number):
   return FieldSize
 
 
+# --------------------------------------------------------------------
+# Map is special: it needs custom logic to compute its size properly.
+
+
+def MapSizer(field_descriptor):
+  """Returns a sizer for a map field."""
+
+  # Can't look at field_descriptor.message_type._concrete_class because it may
+  # not have been initialized yet.
+  message_type = field_descriptor.message_type
+  message_sizer = MessageSizer(field_descriptor.number, False, False)
+
+  def FieldSize(map_value):
+    total = 0
+    for key in map_value:
+      value = map_value[key]
+      # It's wasteful to create the messages and throw them away one second
+      # later since we'll do the same for the actual encode.  But there's not an
+      # obvious way to avoid this within the current design without tons of code
+      # duplication.
+      entry_msg = message_type._concrete_class(key=key, value=value)
+      total += message_sizer(entry_msg)
+    return total
+
+  return FieldSize
+
 # ====================================================================
 # Encoders!
 
@@ -786,3 +812,30 @@ def MessageSetItemEncoder(field_number):
     return write(end_bytes)
 
   return EncodeField
+
+
+# --------------------------------------------------------------------
+# As before, Map is special.
+
+
+def MapEncoder(field_descriptor):
+  """Encoder for extensions of MessageSet.
+
+  Maps always have a wire format like this:
+    message MapEntry {
+      key_type key = 1;
+      value_type value = 2;
+    }
+    repeated MapEntry map = N;
+  """
+  # Can't look at field_descriptor.message_type._concrete_class because it may
+  # not have been initialized yet.
+  message_type = field_descriptor.message_type
+  encode_message = MessageEncoder(field_descriptor.number, False, False)
+
+  def EncodeField(write, value):
+    for key in value:
+      entry_msg = message_type._concrete_class(key=key, value=value[key])
+      encode_message(write, entry_msg)
+
+  return EncodeField

python/google/protobuf/internal/generator_test.py

@@ -42,7 +42,6 @@ further ensures that we can use Python protocol message objects as we expect.
 __author__ = 'robinson@google.com (Will Robinson)'
 
 import unittest
-
 from google.protobuf.internal import test_bad_identifiers_pb2
 from google.protobuf import unittest_custom_options_pb2
 from google.protobuf import unittest_import_pb2

python/google/protobuf/internal/message_factory_test.py

@@ -35,7 +35,6 @@
 __author__ = 'matthewtoia@google.com (Matt Toia)'
 
 import unittest
-
 from google.protobuf import descriptor_pb2
 from google.protobuf.internal import factory_test1_pb2
 from google.protobuf.internal import factory_test2_pb2

python/google/protobuf/internal/message_test.py

@@ -50,7 +50,9 @@ import pickle
 import sys
 import unittest
 
+import unittest
 from google.protobuf.internal import _parameterized
+from google.protobuf import map_unittest_pb2
 from google.protobuf import unittest_pb2
 from google.protobuf import unittest_proto3_arena_pb2
 from google.protobuf.internal import api_implementation
@@ -125,10 +127,17 @@ class MessageTest(unittest.TestCase):
     self.assertEqual(unpickled_message, golden_message)
 
   def testPositiveInfinity(self, message_module):
-    golden_data = (b'\x5D\x00\x00\x80\x7F'
-                   b'\x61\x00\x00\x00\x00\x00\x00\xF0\x7F'
-                   b'\xCD\x02\x00\x00\x80\x7F'
-                   b'\xD1\x02\x00\x00\x00\x00\x00\x00\xF0\x7F')
+    if message_module is unittest_pb2:
+      golden_data = (b'\x5D\x00\x00\x80\x7F'
+                     b'\x61\x00\x00\x00\x00\x00\x00\xF0\x7F'
+                     b'\xCD\x02\x00\x00\x80\x7F'
+                     b'\xD1\x02\x00\x00\x00\x00\x00\x00\xF0\x7F')
+    else:
+      golden_data = (b'\x5D\x00\x00\x80\x7F'
+                     b'\x61\x00\x00\x00\x00\x00\x00\xF0\x7F'
+                     b'\xCA\x02\x04\x00\x00\x80\x7F'
+                     b'\xD2\x02\x08\x00\x00\x00\x00\x00\x00\xF0\x7F')
+
     golden_message = message_module.TestAllTypes()
     golden_message.ParseFromString(golden_data)
     self.assertTrue(IsPosInf(golden_message.optional_float))
@@ -138,10 +147,17 @@ class MessageTest(unittest.TestCase):
     self.assertEqual(golden_data, golden_message.SerializeToString())
 
   def testNegativeInfinity(self, message_module):
-    golden_data = (b'\x5D\x00\x00\x80\xFF'
-                   b'\x61\x00\x00\x00\x00\x00\x00\xF0\xFF'
-                   b'\xCD\x02\x00\x00\x80\xFF'
-                   b'\xD1\x02\x00\x00\x00\x00\x00\x00\xF0\xFF')
+    if message_module is unittest_pb2:
+      golden_data = (b'\x5D\x00\x00\x80\xFF'
+                     b'\x61\x00\x00\x00\x00\x00\x00\xF0\xFF'
+                     b'\xCD\x02\x00\x00\x80\xFF'
+                     b'\xD1\x02\x00\x00\x00\x00\x00\x00\xF0\xFF')
+    else:
+      golden_data = (b'\x5D\x00\x00\x80\xFF'
+                     b'\x61\x00\x00\x00\x00\x00\x00\xF0\xFF'
+                     b'\xCA\x02\x04\x00\x00\x80\xFF'
+                     b'\xD2\x02\x08\x00\x00\x00\x00\x00\x00\xF0\xFF')
+
     golden_message = message_module.TestAllTypes()
     golden_message.ParseFromString(golden_data)
     self.assertTrue(IsNegInf(golden_message.optional_float))
@@ -1034,64 +1050,132 @@ class Proto2Test(unittest.TestCase):
     self.assertEqual(len(parsing_merge.Extensions[
         unittest_pb2.TestParsingMerge.repeated_ext]), 3)
 
+  def testPythonicInit(self):
+    message = unittest_pb2.TestAllTypes(
+        optional_int32=100,
+        optional_fixed32=200,
+        optional_float=300.5,
+        optional_bytes=b'x',
+        optionalgroup={'a': 400},
+        optional_nested_message={'bb': 500},
+        optional_nested_enum='BAZ',
+        repeatedgroup=[{'a': 600},
+                       {'a': 700}],
+        repeated_nested_enum=['FOO', unittest_pb2.TestAllTypes.BAR],
+        default_int32=800,
+        oneof_string='y')
+    self.assertTrue(isinstance(message, unittest_pb2.TestAllTypes))
+    self.assertEqual(100, message.optional_int32)
+    self.assertEqual(200, message.optional_fixed32)
+    self.assertEqual(300.5, message.optional_float)
+    self.assertEqual(b'x', message.optional_bytes)
+    self.assertEqual(400, message.optionalgroup.a)
+    self.assertTrue(isinstance(message.optional_nested_message,
+                               unittest_pb2.TestAllTypes.NestedMessage))
+    self.assertEqual(500, message.optional_nested_message.bb)
+    self.assertEqual(unittest_pb2.TestAllTypes.BAZ,
+                     message.optional_nested_enum)
+    self.assertEqual(2, len(message.repeatedgroup))
+    self.assertEqual(600, message.repeatedgroup[0].a)
+    self.assertEqual(700, message.repeatedgroup[1].a)
+    self.assertEqual(2, len(message.repeated_nested_enum))
+    self.assertEqual(unittest_pb2.TestAllTypes.FOO,
+                     message.repeated_nested_enum[0])
+    self.assertEqual(unittest_pb2.TestAllTypes.BAR,
+                     message.repeated_nested_enum[1])
+    self.assertEqual(800, message.default_int32)
+    self.assertEqual('y', message.oneof_string)
+    self.assertFalse(message.HasField('optional_int64'))
+    self.assertEqual(0, len(message.repeated_float))
+    self.assertEqual(42, message.default_int64)
+
+    message = unittest_pb2.TestAllTypes(optional_nested_enum=u'BAZ')
+    self.assertEqual(unittest_pb2.TestAllTypes.BAZ,
+                     message.optional_nested_enum)
+
+    with self.assertRaises(ValueError):
+      unittest_pb2.TestAllTypes(
+          optional_nested_message={'INVALID_NESTED_FIELD': 17})
+
+    with self.assertRaises(TypeError):
+      unittest_pb2.TestAllTypes(
+          optional_nested_message={'bb': 'INVALID_VALUE_TYPE'})
+
+    with self.assertRaises(ValueError):
+      unittest_pb2.TestAllTypes(optional_nested_enum='INVALID_LABEL')
+
+    with self.assertRaises(ValueError):
+      unittest_pb2.TestAllTypes(repeated_nested_enum='FOO')
+
 
 # Class to test proto3-only features/behavior (updated field presence & enums)
 class Proto3Test(unittest.TestCase):
 
+  # Utility method for comparing equality with a map.
+  def assertMapIterEquals(self, map_iter, dict_value):
+    # Avoid mutating caller's copy.
+    dict_value = dict(dict_value)
+
+    for k, v in map_iter:
+      self.assertEqual(v, dict_value[k])
+      del dict_value[k]
+
+    self.assertEqual({}, dict_value)
+
   def testFieldPresence(self):
     message = unittest_proto3_arena_pb2.TestAllTypes()
 
     # We can't test presence of non-repeated, non-submessage fields.
     with self.assertRaises(ValueError):
-      message.HasField("optional_int32")
+      message.HasField('optional_int32')
     with self.assertRaises(ValueError):
-      message.HasField("optional_float")
+      message.HasField('optional_float')
     with self.assertRaises(ValueError):
-      message.HasField("optional_string")
+      message.HasField('optional_string')
     with self.assertRaises(ValueError):
-      message.HasField("optional_bool")
+      message.HasField('optional_bool')
 
     # But we can still test presence of submessage fields.
-    self.assertFalse(message.HasField("optional_nested_message"))
+    self.assertFalse(message.HasField('optional_nested_message'))
 
     # As with proto2, we can't test presence of fields that don't exist, or
     # repeated fields.
     with self.assertRaises(ValueError):
-      message.HasField("field_doesnt_exist")
+      message.HasField('field_doesnt_exist')
 
     with self.assertRaises(ValueError):
-      message.HasField("repeated_int32")
+      message.HasField('repeated_int32')
     with self.assertRaises(ValueError):
-      message.HasField("repeated_nested_message")
+      message.HasField('repeated_nested_message')
 
     # Fields should default to their type-specific default.
     self.assertEqual(0, message.optional_int32)
     self.assertEqual(0, message.optional_float)
-    self.assertEqual("", message.optional_string)
+    self.assertEqual('', message.optional_string)
     self.assertEqual(False, message.optional_bool)
     self.assertEqual(0, message.optional_nested_message.bb)
 
     # Setting a submessage should still return proper presence information.
     message.optional_nested_message.bb = 0
-    self.assertTrue(message.HasField("optional_nested_message"))
+    self.assertTrue(message.HasField('optional_nested_message'))
 
     # Set the fields to non-default values.
     message.optional_int32 = 5
     message.optional_float = 1.1
-    message.optional_string = "abc"
+    message.optional_string = 'abc'
     message.optional_bool = True
     message.optional_nested_message.bb = 15
 
     # Clearing the fields unsets them and resets their value to default.
-    message.ClearField("optional_int32")
-    message.ClearField("optional_float")
-    message.ClearField("optional_string")
-    message.ClearField("optional_bool")
-    message.ClearField("optional_nested_message")
+    message.ClearField('optional_int32')
+    message.ClearField('optional_float')
+    message.ClearField('optional_string')
+    message.ClearField('optional_bool')
+    message.ClearField('optional_nested_message')
 
     self.assertEqual(0, message.optional_int32)
     self.assertEqual(0, message.optional_float)
-    self.assertEqual("", message.optional_string)
+    self.assertEqual('', message.optional_string)
     self.assertEqual(False, message.optional_bool)
     self.assertEqual(0, message.optional_nested_message.bb)
 
@@ -1113,6 +1197,393 @@ class Proto3Test(unittest.TestCase):
     self.assertEqual(1234567, m2.optional_nested_enum)
     self.assertEqual(7654321, m2.repeated_nested_enum[0])
 
+  # Map isn't really a proto3-only feature. But there is no proto2 equivalent
+  # of google/protobuf/map_unittest.proto right now, so it's not easy to
+  # test both with the same test like we do for the other proto2/proto3 tests.
+  # (google/protobuf/map_protobuf_unittest.proto is very different in the set
+  # of messages and fields it contains).
+  def testScalarMapDefaults(self):
+    msg = map_unittest_pb2.TestMap()
+
+    # Scalars start out unset.
+    self.assertFalse(-123 in msg.map_int32_int32)
+    self.assertFalse(-2**33 in msg.map_int64_int64)
+    self.assertFalse(123 in msg.map_uint32_uint32)
+    self.assertFalse(2**33 in msg.map_uint64_uint64)
+    self.assertFalse('abc' in msg.map_string_string)
+    self.assertFalse(888 in msg.map_int32_enum)
+
+    # Accessing an unset key returns the default.
+    self.assertEqual(0, msg.map_int32_int32[-123])
+    self.assertEqual(0, msg.map_int64_int64[-2**33])
+    self.assertEqual(0, msg.map_uint32_uint32[123])
+    self.assertEqual(0, msg.map_uint64_uint64[2**33])
+    self.assertEqual('', msg.map_string_string['abc'])
+    self.assertEqual(0, msg.map_int32_enum[888])
+
+    # It also sets the value in the map
+    self.assertTrue(-123 in msg.map_int32_int32)
+    self.assertTrue(-2**33 in msg.map_int64_int64)
+    self.assertTrue(123 in msg.map_uint32_uint32)
+    self.assertTrue(2**33 in msg.map_uint64_uint64)
+    self.assertTrue('abc' in msg.map_string_string)
+    self.assertTrue(888 in msg.map_int32_enum)
+
+    self.assertTrue(isinstance(msg.map_string_string['abc'], unicode))
+
+    # Accessing an unset key still throws TypeError of the type of the key
+    # is incorrect.
+    with self.assertRaises(TypeError):
+      msg.map_string_string[123]
+
+    self.assertFalse(123 in msg.map_string_string)
+
+  def testMapGet(self):
+    # Need to test that get() properly returns the default, even though the dict
+    # has defaultdict-like semantics.
+    msg = map_unittest_pb2.TestMap()
+
+    self.assertIsNone(msg.map_int32_int32.get(5))
+    self.assertEquals(10, msg.map_int32_int32.get(5, 10))
+    self.assertIsNone(msg.map_int32_int32.get(5))
+
+    msg.map_int32_int32[5] = 15
+    self.assertEquals(15, msg.map_int32_int32.get(5))
+
+    self.assertIsNone(msg.map_int32_foreign_message.get(5))
+    self.assertEquals(10, msg.map_int32_foreign_message.get(5, 10))
+
+    submsg = msg.map_int32_foreign_message[5]
+    self.assertIs(submsg, msg.map_int32_foreign_message.get(5))
+
+  def testScalarMap(self):
+    msg = map_unittest_pb2.TestMap()
+
+    self.assertEqual(0, len(msg.map_int32_int32))
+    self.assertFalse(5 in msg.map_int32_int32)
+
+    msg.map_int32_int32[-123] = -456
+    msg.map_int64_int64[-2**33] = -2**34
+    msg.map_uint32_uint32[123] = 456
+    msg.map_uint64_uint64[2**33] = 2**34
+    msg.map_string_string['abc'] = '123'
+    msg.map_int32_enum[888] = 2
+
+    self.assertEqual([], msg.FindInitializationErrors())
+
+    self.assertEqual(1, len(msg.map_string_string))
+
+    # Bad key.
+    with self.assertRaises(TypeError):
+      msg.map_string_string[123] = '123'
+
+    # Verify that trying to assign a bad key doesn't actually add a member to
+    # the map.
+    self.assertEqual(1, len(msg.map_string_string))
+
+    # Bad value.
+    with self.assertRaises(TypeError):
+      msg.map_string_string['123'] = 123
+
+    serialized = msg.SerializeToString()
+    msg2 = map_unittest_pb2.TestMap()
+    msg2.ParseFromString(serialized)
+
+    # Bad key.
+    with self.assertRaises(TypeError):
+      msg2.map_string_string[123] = '123'
+
+    # Bad value.
+    with self.assertRaises(TypeError):
+      msg2.map_string_string['123'] = 123
+
+    self.assertEqual(-456, msg2.map_int32_int32[-123])
+    self.assertEqual(-2**34, msg2.map_int64_int64[-2**33])
+    self.assertEqual(456, msg2.map_uint32_uint32[123])
+    self.assertEqual(2**34, msg2.map_uint64_uint64[2**33])
+    self.assertEqual('123', msg2.map_string_string['abc'])
+    self.assertEqual(2, msg2.map_int32_enum[888])
+
+  def testStringUnicodeConversionInMap(self):
+    msg = map_unittest_pb2.TestMap()
+
+    unicode_obj = u'\u1234'
+    bytes_obj = unicode_obj.encode('utf8') 
+
+    msg.map_string_string[bytes_obj] = bytes_obj
+
+    (key, value) = msg.map_string_string.items()[0]
+
+    self.assertEqual(key, unicode_obj)
+    self.assertEqual(value, unicode_obj)
+
+    self.assertTrue(isinstance(key, unicode))
+    self.assertTrue(isinstance(value, unicode))
+
+  def testMessageMap(self):
+    msg = map_unittest_pb2.TestMap()
+
+    self.assertEqual(0, len(msg.map_int32_foreign_message))
+    self.assertFalse(5 in msg.map_int32_foreign_message)
+
+    msg.map_int32_foreign_message[123]
+    # get_or_create() is an alias for getitem.
+    msg.map_int32_foreign_message.get_or_create(-456)
+
+    self.assertEqual(2, len(msg.map_int32_foreign_message))
+    self.assertIn(123, msg.map_int32_foreign_message)
+    self.assertIn(-456, msg.map_int32_foreign_message)
+    self.assertEqual(2, len(msg.map_int32_foreign_message))
+
+    # Bad key.
+    with self.assertRaises(TypeError):
+      msg.map_int32_foreign_message['123']
+
+    # Can't assign directly to submessage.
+    with self.assertRaises(ValueError):
+      msg.map_int32_foreign_message[999] = msg.map_int32_foreign_message[123]
+
+    # Verify that trying to assign a bad key doesn't actually add a member to
+    # the map.
+    self.assertEqual(2, len(msg.map_int32_foreign_message))
+
+    serialized = msg.SerializeToString()
+    msg2 = map_unittest_pb2.TestMap()
+    msg2.ParseFromString(serialized)
+
+    self.assertEqual(2, len(msg2.map_int32_foreign_message))
+    self.assertIn(123, msg2.map_int32_foreign_message)
+    self.assertIn(-456, msg2.map_int32_foreign_message)
+    self.assertEqual(2, len(msg2.map_int32_foreign_message))
+
+  def testMergeFrom(self):
+    msg = map_unittest_pb2.TestMap()
+    msg.map_int32_int32[12] = 34
+    msg.map_int32_int32[56] = 78
+    msg.map_int64_int64[22] = 33
+    msg.map_int32_foreign_message[111].c = 5
+    msg.map_int32_foreign_message[222].c = 10
+
+    msg2 = map_unittest_pb2.TestMap()
+    msg2.map_int32_int32[12] = 55
+    msg2.map_int64_int64[88] = 99
+    msg2.map_int32_foreign_message[222].c = 15
+
+    msg2.MergeFrom(msg)
+
+    self.assertEqual(34, msg2.map_int32_int32[12])
+    self.assertEqual(78, msg2.map_int32_int32[56])
+    self.assertEqual(33, msg2.map_int64_int64[22])
+    self.assertEqual(99, msg2.map_int64_int64[88])
+    self.assertEqual(5, msg2.map_int32_foreign_message[111].c)
+    self.assertEqual(10, msg2.map_int32_foreign_message[222].c)
+
+    # Verify that there is only one entry per key, even though the MergeFrom
+    # may have internally created multiple entries for a single key in the
+    # list representation.
+    as_dict = {}
+    for key in msg2.map_int32_foreign_message:
+      self.assertFalse(key in as_dict)
+      as_dict[key] = msg2.map_int32_foreign_message[key].c
+
+    self.assertEqual({111: 5, 222: 10}, as_dict)
+
+    # Special case: test that delete of item really removes the item, even if
+    # there might have physically been duplicate keys due to the previous merge.
+    # This is only a special case for the C++ implementation which stores the
+    # map as an array.
+    del msg2.map_int32_int32[12]
+    self.assertFalse(12 in msg2.map_int32_int32)
+
+    del msg2.map_int32_foreign_message[222]
+    self.assertFalse(222 in msg2.map_int32_foreign_message)
+
+  def testIntegerMapWithLongs(self):
+    msg = map_unittest_pb2.TestMap()
+    msg.map_int32_int32[long(-123)] = long(-456)
+    msg.map_int64_int64[long(-2**33)] = long(-2**34)
+    msg.map_uint32_uint32[long(123)] = long(456)
+    msg.map_uint64_uint64[long(2**33)] = long(2**34)
+
+    serialized = msg.SerializeToString()
+    msg2 = map_unittest_pb2.TestMap()
+    msg2.ParseFromString(serialized)
+
+    self.assertEqual(-456, msg2.map_int32_int32[-123])
+    self.assertEqual(-2**34, msg2.map_int64_int64[-2**33])
+    self.assertEqual(456, msg2.map_uint32_uint32[123])
+    self.assertEqual(2**34, msg2.map_uint64_uint64[2**33])
+
+  def testMapAssignmentCausesPresence(self):
+    msg = map_unittest_pb2.TestMapSubmessage()
+    msg.test_map.map_int32_int32[123] = 456
+
+    serialized = msg.SerializeToString()
+    msg2 = map_unittest_pb2.TestMapSubmessage()
+    msg2.ParseFromString(serialized)
+
+    self.assertEqual(msg, msg2)
+
+    # Now test that various mutations of the map properly invalidate the
+    # cached size of the submessage.
+    msg.test_map.map_int32_int32[888] = 999
+    serialized = msg.SerializeToString()
+    msg2.ParseFromString(serialized)
+    self.assertEqual(msg, msg2)
+
+    msg.test_map.map_int32_int32.clear()
+    serialized = msg.SerializeToString()
+    msg2.ParseFromString(serialized)
+    self.assertEqual(msg, msg2)
+
+  def testMapAssignmentCausesPresenceForSubmessages(self):
+    msg = map_unittest_pb2.TestMapSubmessage()
+    msg.test_map.map_int32_foreign_message[123].c = 5
+
+    serialized = msg.SerializeToString()
+    msg2 = map_unittest_pb2.TestMapSubmessage()
+    msg2.ParseFromString(serialized)
+
+    self.assertEqual(msg, msg2)
+
+    # Now test that various mutations of the map properly invalidate the
+    # cached size of the submessage.
+    msg.test_map.map_int32_foreign_message[888].c = 7
+    serialized = msg.SerializeToString()
+    msg2.ParseFromString(serialized)
+    self.assertEqual(msg, msg2)
+
+    msg.test_map.map_int32_foreign_message[888].MergeFrom(
+        msg.test_map.map_int32_foreign_message[123])
+    serialized = msg.SerializeToString()
+    msg2.ParseFromString(serialized)
+    self.assertEqual(msg, msg2)
+
+    msg.test_map.map_int32_foreign_message.clear()
+    serialized = msg.SerializeToString()
+    msg2.ParseFromString(serialized)
+    self.assertEqual(msg, msg2)
+
+  def testModifyMapWhileIterating(self):
+    msg = map_unittest_pb2.TestMap()
+
+    string_string_iter = iter(msg.map_string_string)
+    int32_foreign_iter = iter(msg.map_int32_foreign_message)
+
+    msg.map_string_string['abc'] = '123'
+    msg.map_int32_foreign_message[5].c = 5
+
+    with self.assertRaises(RuntimeError):
+      for key in string_string_iter:
+        pass
+
+    with self.assertRaises(RuntimeError):
+      for key in int32_foreign_iter:
+        pass
+
+  def testSubmessageMap(self):
+    msg = map_unittest_pb2.TestMap()
+
+    submsg = msg.map_int32_foreign_message[111]
+    self.assertIs(submsg, msg.map_int32_foreign_message[111])
+    self.assertTrue(isinstance(submsg, unittest_pb2.ForeignMessage))
+
+    submsg.c = 5
+
+    serialized = msg.SerializeToString()
+    msg2 = map_unittest_pb2.TestMap()
+    msg2.ParseFromString(serialized)
+
+    self.assertEqual(5, msg2.map_int32_foreign_message[111].c)
+
+    # Doesn't allow direct submessage assignment.
+    with self.assertRaises(ValueError):
+      msg.map_int32_foreign_message[88] = unittest_pb2.ForeignMessage()
+
+  def testMapIteration(self):
+    msg = map_unittest_pb2.TestMap()
+
+    for k, v in msg.map_int32_int32.iteritems():
+      # Should not be reached.
+      self.assertTrue(False)
+
+    msg.map_int32_int32[2] = 4
+    msg.map_int32_int32[3] = 6
+    msg.map_int32_int32[4] = 8
+    self.assertEqual(3, len(msg.map_int32_int32))
+
+    matching_dict = {2: 4, 3: 6, 4: 8}
+    self.assertMapIterEquals(msg.map_int32_int32.iteritems(), matching_dict)
+
+  def testMapIterationClearMessage(self):
+    # Iterator needs to work even if message and map are deleted.
+    msg = map_unittest_pb2.TestMap()
+
+    msg.map_int32_int32[2] = 4
+    msg.map_int32_int32[3] = 6
+    msg.map_int32_int32[4] = 8
+
+    it = msg.map_int32_int32.iteritems()
+    del msg
+
+    matching_dict = {2: 4, 3: 6, 4: 8}
+    self.assertMapIterEquals(it, matching_dict)
+
+  def testMapConstruction(self):
+    msg = map_unittest_pb2.TestMap(map_int32_int32={1: 2, 3: 4})
+    self.assertEqual(2, msg.map_int32_int32[1])
+    self.assertEqual(4, msg.map_int32_int32[3])
+
+    msg = map_unittest_pb2.TestMap(
+        map_int32_foreign_message={3: unittest_pb2.ForeignMessage(c=5)})
+    self.assertEqual(5, msg.map_int32_foreign_message[3].c)
+
+  def testMapValidAfterFieldCleared(self):
+    # Map needs to work even if field is cleared.
+    # For the C++ implementation this tests the correctness of
+    # ScalarMapContainer::Release()
+    msg = map_unittest_pb2.TestMap()
+    map = msg.map_int32_int32
+
+    map[2] = 4
+    map[3] = 6
+    map[4] = 8
+
+    msg.ClearField('map_int32_int32')
+    matching_dict = {2: 4, 3: 6, 4: 8}
+    self.assertMapIterEquals(map.iteritems(), matching_dict)
+
+  def testMapIterValidAfterFieldCleared(self):
+    # Map iterator needs to work even if field is cleared.
+    # For the C++ implementation this tests the correctness of
+    # ScalarMapContainer::Release()
+    msg = map_unittest_pb2.TestMap()
+
+    msg.map_int32_int32[2] = 4
+    msg.map_int32_int32[3] = 6
+    msg.map_int32_int32[4] = 8
+
+    it = msg.map_int32_int32.iteritems()
+
+    msg.ClearField('map_int32_int32')
+    matching_dict = {2: 4, 3: 6, 4: 8}
+    self.assertMapIterEquals(it, matching_dict)
+
+  def testMapDelete(self):
+    msg = map_unittest_pb2.TestMap()
+
+    self.assertEqual(0, len(msg.map_int32_int32))
+
+    msg.map_int32_int32[4] = 6
+    self.assertEqual(1, len(msg.map_int32_int32))
+
+    with self.assertRaises(KeyError):
+      del msg.map_int32_int32[88]
+
+    del msg.map_int32_int32[4]
+    self.assertEqual(0, len(msg.map_int32_int32))
+
+
 
 class ValidTypeNamesTest(unittest.TestCase):
 

python/google/protobuf/internal/proto_builder_test.py

@@ -32,6 +32,7 @@
 
 """Tests for google.protobuf.proto_builder."""
 
+import collections
 import unittest
 
 from google.protobuf import descriptor_pb2
@@ -43,10 +44,11 @@ from google.protobuf import text_format
 class ProtoBuilderTest(unittest.TestCase):
 
   def setUp(self):
-    self._fields = {
-        'foo': descriptor_pb2.FieldDescriptorProto.TYPE_INT64,
-        'bar': descriptor_pb2.FieldDescriptorProto.TYPE_STRING,
-        }
+    self.ordered_fields = collections.OrderedDict([
+        ('foo', descriptor_pb2.FieldDescriptorProto.TYPE_INT64),
+        ('bar', descriptor_pb2.FieldDescriptorProto.TYPE_STRING),
+        ])
+    self._fields = dict(self.ordered_fields)
 
   def testMakeSimpleProtoClass(self):
     """Test that we can create a proto class."""
@@ -59,6 +61,17 @@ class ProtoBuilderTest(unittest.TestCase):
     self.assertMultiLineEqual(
         'bar: "asdf"\nfoo: 12345\n', text_format.MessageToString(proto))
 
+  def testOrderedFields(self):
+    """Test that the field order is maintained when given an OrderedDict."""
+    proto_cls = proto_builder.MakeSimpleProtoClass(
+        self.ordered_fields,
+        full_name='net.proto2.python.public.proto_builder_test.OrderedTest')
+    proto = proto_cls()
+    proto.foo = 12345
+    proto.bar = 'asdf'
+    self.assertMultiLineEqual(
+        'foo: 12345\nbar: "asdf"\n', text_format.MessageToString(proto))
+
   def testMakeSameProtoClassTwice(self):
     """Test that the DescriptorPool is used."""
     pool = descriptor_pool.DescriptorPool()

python/google/protobuf/internal/python_message.py

@@ -61,9 +61,11 @@ if sys.version_info[0] < 3:
   except ImportError:
     from StringIO import StringIO as BytesIO
   import copy_reg as copyreg
+  _basestring = basestring
 else:
   from io import BytesIO
   import copyreg
+  _basestring = str
 import struct
 import weakref
 
@@ -77,6 +79,7 @@ from google.protobuf.internal import type_checkers
 from google.protobuf.internal import wire_format
 from google.protobuf import descriptor as descriptor_mod
 from google.protobuf import message as message_mod
+from google.protobuf import symbol_database
 from google.protobuf import text_format
 
 _FieldDescriptor = descriptor_mod.FieldDescriptor
@@ -101,6 +104,7 @@ def InitMessage(descriptor, cls):
   for field in descriptor.fields:
     _AttachFieldHelpers(cls, field)
 
+  descriptor._concrete_class = cls  # pylint: disable=protected-access
   _AddEnumValues(descriptor, cls)
   _AddInitMethod(descriptor, cls)
   _AddPropertiesForFields(descriptor, cls)
@@ -198,12 +202,37 @@ def _IsMessageSetExtension(field):
           field.label == _FieldDescriptor.LABEL_OPTIONAL)
 
 
+def _IsMapField(field):
+  return (field.type == _FieldDescriptor.TYPE_MESSAGE and
+          field.message_type.has_options and
+          field.message_type.GetOptions().map_entry)
+
+
+def _IsMessageMapField(field):
+  value_type = field.message_type.fields_by_name["value"]
+  return value_type.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE
+
+
 def _AttachFieldHelpers(cls, field_descriptor):
   is_repeated = (field_descriptor.label == _FieldDescriptor.LABEL_REPEATED)
-  is_packed = (field_descriptor.has_options and
-               field_descriptor.GetOptions().packed)
-
-  if _IsMessageSetExtension(field_descriptor):
+  is_packable = (is_repeated and
+                 wire_format.IsTypePackable(field_descriptor.type))
+  if not is_packable:
+    is_packed = False
+  elif field_descriptor.containing_type.syntax == "proto2":
+    is_packed = (field_descriptor.has_options and
+                field_descriptor.GetOptions().packed)
+  else:
+    has_packed_false = (field_descriptor.has_options and
+                        field_descriptor.GetOptions().HasField("packed") and
+                        field_descriptor.GetOptions().packed == False)
+    is_packed = not has_packed_false
+  is_map_entry = _IsMapField(field_descriptor)
+
+  if is_map_entry:
+    field_encoder = encoder.MapEncoder(field_descriptor)
+    sizer = encoder.MapSizer(field_descriptor)
+  elif _IsMessageSetExtension(field_descriptor):
     field_encoder = encoder.MessageSetItemEncoder(field_descriptor.number)
     sizer = encoder.MessageSetItemSizer(field_descriptor.number)
   else:
@@ -228,9 +257,16 @@ def _AttachFieldHelpers(cls, field_descriptor):
     if field_descriptor.containing_oneof is not None:
       oneof_descriptor = field_descriptor
 
-    field_decoder = type_checkers.TYPE_TO_DECODER[decode_type](
-        field_descriptor.number, is_repeated, is_packed,
-        field_descriptor, field_descriptor._default_constructor)
+    if is_map_entry:
+      is_message_map = _IsMessageMapField(field_descriptor)
+
+      field_decoder = decoder.MapDecoder(
+          field_descriptor, _GetInitializeDefaultForMap(field_descriptor),
+          is_message_map)
+    else:
+      field_decoder = type_checkers.TYPE_TO_DECODER[decode_type](
+              field_descriptor.number, is_repeated, is_packed,
+              field_descriptor, field_descriptor._default_constructor)
 
     cls._decoders_by_tag[tag_bytes] = (field_decoder, oneof_descriptor)
 
@@ -265,6 +301,26 @@ def _AddEnumValues(descriptor, cls):
       setattr(cls, enum_value.name, enum_value.number)
 
 
+def _GetInitializeDefaultForMap(field):
+  if field.label != _FieldDescriptor.LABEL_REPEATED:
+    raise ValueError('map_entry set on non-repeated field %s' % (
+        field.name))
+  fields_by_name = field.message_type.fields_by_name
+  key_checker = type_checkers.GetTypeChecker(fields_by_name['key'])
+
+  value_field = fields_by_name['value']
+  if _IsMessageMapField(field):
+    def MakeMessageMapDefault(message):
+      return containers.MessageMap(
+          message._listener_for_children, value_field.message_type, key_checker)
+    return MakeMessageMapDefault
+  else:
+    value_checker = type_checkers.GetTypeChecker(value_field)
+    def MakePrimitiveMapDefault(message):
+      return containers.ScalarMap(
+          message._listener_for_children, key_checker, value_checker)
+    return MakePrimitiveMapDefault
+
 def _DefaultValueConstructorForField(field):
   """Returns a function which returns a default value for a field.
 
@@ -279,6 +335,9 @@ def _DefaultValueConstructorForField(field):
     value may refer back to |message| via a weak reference.
   """
 
+  if _IsMapField(field):
+    return _GetInitializeDefaultForMap(field)
+
   if field.label == _FieldDescriptor.LABEL_REPEATED:
     if field.has_default_value and field.default_value != []:
       raise ValueError('Repeated field default value not empty list: %s' % (
@@ -329,7 +388,22 @@ def _ReraiseTypeErrorWithFieldName(message_name, field_name):
 
 def _AddInitMethod(message_descriptor, cls):
   """Adds an __init__ method to cls."""
-  fields = message_descriptor.fields
+
+  def _GetIntegerEnumValue(enum_type, value):
+    """Convert a string or integer enum value to an integer.
+
+    If the value is a string, it is converted to the enum value in
+    enum_type with the same name.  If the value is not a string, it's
+    returned as-is.  (No conversion or bounds-checking is done.)
+    """
+    if isinstance(value, _basestring):
+      try:
+        return enum_type.values_by_name[value].number
+      except KeyError:
+        raise ValueError('Enum type %s: unknown label "%s"' % (
+            enum_type.full_name, value))
+    return value
+
   def init(self, **kwargs):
     self._cached_byte_size = 0
     self._cached_byte_size_dirty = len(kwargs) > 0
@@ -352,19 +426,37 @@ def _AddInitMethod(message_descriptor, cls):
       if field.label == _FieldDescriptor.LABEL_REPEATED:
         copy = field._default_constructor(self)
         if field.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE:  # Composite
-          for val in field_value:
-            copy.add().MergeFrom(val)
+          if _IsMapField(field):
+            if _IsMessageMapField(field):
+              for key in field_value:
+                copy[key].MergeFrom(field_value[key])
+            else:
+              copy.update(field_value)
+          else:
+            for val in field_value:
+              if isinstance(val, dict):
+                copy.add(**val)
+              else:
+                copy.add().MergeFrom(val)
         else:  # Scalar
+          if field.cpp_type == _FieldDescriptor.CPPTYPE_ENUM:
+            field_value = [_GetIntegerEnumValue(field.enum_type, val)
+                           for val in field_value]
           copy.extend(field_value)
         self._fields[field] = copy
       elif field.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE:
         copy = field._default_constructor(self)
+        new_val = field_value
+        if isinstance(field_value, dict):
+          new_val = field.message_type._concrete_class(**field_value)
         try:
-          copy.MergeFrom(field_value)
+          copy.MergeFrom(new_val)
         except TypeError:
           _ReraiseTypeErrorWithFieldName(message_descriptor.name, field_name)
         self._fields[field] = copy
       else:
+        if field.cpp_type == _FieldDescriptor.CPPTYPE_ENUM:
+          field_value = _GetIntegerEnumValue(field.enum_type, field_value)
         try:
           setattr(self, field_name, field_value)
         except TypeError:
@@ -758,6 +850,26 @@ def _AddHasExtensionMethod(cls):
       return extension_handle in self._fields
   cls.HasExtension = HasExtension
 
+def _UnpackAny(msg):
+  type_url = msg.type_url
+  db = symbol_database.Default()
+
+  if not type_url:
+    return None
+
+  # TODO(haberman): For now we just strip the hostname.  Better logic will be
+  # required.
+  type_name = type_url.split("/")[-1]
+  descriptor = db.pool.FindMessageTypeByName(type_name)
+
+  if descriptor is None:
+    return None
+
+  message_class = db.GetPrototype(descriptor)
+  message = message_class()
+
+  message.ParseFromString(msg.value)
+  return message
 
 def _AddEqualsMethod(message_descriptor, cls):
   """Helper for _AddMessageMethods()."""
@@ -769,6 +881,12 @@ def _AddEqualsMethod(message_descriptor, cls):
     if self is other:
       return True
 
+    if self.DESCRIPTOR.full_name == "google.protobuf.Any":
+      any_a = _UnpackAny(self)
+      any_b = _UnpackAny(other)
+      if any_a and any_b:
+        return any_a == any_b
+
     if not self.ListFields() == other.ListFields():
       return False
 
@@ -961,6 +1079,9 @@ def _AddIsInitializedMethod(message_descriptor, cls):
     for field, value in list(self._fields.items()):  # dict can change size!
       if field.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE:
         if field.label == _FieldDescriptor.LABEL_REPEATED:
+          if (field.message_type.has_options and
+              field.message_type.GetOptions().map_entry):
+            continue
           for element in value:
             if not element.IsInitialized():
               if errors is not None:
@@ -996,16 +1117,26 @@ def _AddIsInitializedMethod(message_descriptor, cls):
         else:
           name = field.name
 
-        if field.label == _FieldDescriptor.LABEL_REPEATED:
+        if _IsMapField(field):
+          if _IsMessageMapField(field):
+            for key in value:
+              element = value[key]
+              prefix = "%s[%d]." % (name, key)
+              sub_errors = element.FindInitializationErrors()
+              errors += [prefix + error for error in sub_errors]
+          else:
+            # ScalarMaps can't have any initialization errors.
+            pass
+        elif field.label == _FieldDescriptor.LABEL_REPEATED:
           for i in xrange(len(value)):
             element = value[i]
             prefix = "%s[%d]." % (name, i)
             sub_errors = element.FindInitializationErrors()
-            errors += [ prefix + error for error in sub_errors ]
+            errors += [prefix + error for error in sub_errors]
         else:
           prefix = name + "."
           sub_errors = value.FindInitializationErrors()
-          errors += [ prefix + error for error in sub_errors ]
+          errors += [prefix + error for error in sub_errors]
 
     return errors
 

python/google/protobuf/internal/reflection_test.py

@@ -39,8 +39,8 @@ import copy
 import gc
 import operator
 import struct
-import unittest
 
+import unittest
 from google.protobuf import unittest_import_pb2
 from google.protobuf import unittest_mset_pb2
 from google.protobuf import unittest_pb2
@@ -1798,8 +1798,8 @@ class ReflectionTest(unittest.TestCase):
   def testBadArguments(self):
     # Some of these assertions used to segfault.
     from google.protobuf.pyext import _message
-    self.assertRaises(TypeError, _message.Message._GetFieldDescriptor, 3)
-    self.assertRaises(TypeError, _message.Message._GetExtensionDescriptor, 42)
+    self.assertRaises(TypeError, _message.default_pool.FindFieldByName, 3)
+    self.assertRaises(TypeError, _message.default_pool.FindExtensionByName, 42)
     self.assertRaises(TypeError,
                       unittest_pb2.TestAllTypes().__getattribute__, 42)
 

python/google/protobuf/internal/service_reflection_test.py

@@ -35,7 +35,6 @@
 __author__ = 'petar@google.com (Petar Petrov)'
 
 import unittest
-
 from google.protobuf import unittest_pb2
 from google.protobuf import service_reflection
 from google.protobuf import service
@@ -81,7 +80,7 @@ class FooUnitTest(unittest.TestCase):
     self.assertEqual('Method Bar not implemented.',
                      rpc_controller.failure_message)
     self.assertEqual(None, self.callback_response)
-
+    
     class MyServiceImpl(unittest_pb2.TestService):
       def Foo(self, rpc_controller, request, done):
         self.foo_called = True

python/google/protobuf/internal/symbol_database_test.py

@@ -33,7 +33,6 @@
 """Tests for google.protobuf.symbol_database."""
 
 import unittest
-
 from google.protobuf import unittest_pb2
 from google.protobuf import symbol_database
 

python/google/protobuf/internal/test_util.py

@@ -75,7 +75,8 @@ def SetAllNonLazyFields(message):
   message.optional_string   = u'115'
   message.optional_bytes    = b'116'
 
-  message.optionalgroup.a = 117
+  if IsProto2(message):
+    message.optionalgroup.a = 117
   message.optional_nested_message.bb = 118
   message.optional_foreign_message.c = 119
   message.optional_import_message.d = 120
@@ -109,7 +110,8 @@ def SetAllNonLazyFields(message):
   message.repeated_string.append(u'215')
   message.repeated_bytes.append(b'216')
 
-  message.repeatedgroup.add().a = 217
+  if IsProto2(message):
+    message.repeatedgroup.add().a = 217
   message.repeated_nested_message.add().bb = 218
   message.repeated_foreign_message.add().c = 219
   message.repeated_import_message.add().d = 220
@@ -140,7 +142,8 @@ def SetAllNonLazyFields(message):
   message.repeated_string.append(u'315')
   message.repeated_bytes.append(b'316')
 
-  message.repeatedgroup.add().a = 317
+  if IsProto2(message):
+    message.repeatedgroup.add().a = 317
   message.repeated_nested_message.add().bb = 318
   message.repeated_foreign_message.add().c = 319
   message.repeated_import_message.add().d = 320
@@ -396,7 +399,8 @@ def ExpectAllFieldsSet(test_case, message):
   test_case.assertTrue(message.HasField('optional_string'))
   test_case.assertTrue(message.HasField('optional_bytes'))
 
-  test_case.assertTrue(message.HasField('optionalgroup'))
+  if IsProto2(message):
+    test_case.assertTrue(message.HasField('optionalgroup'))
   test_case.assertTrue(message.HasField('optional_nested_message'))
   test_case.assertTrue(message.HasField('optional_foreign_message'))
   test_case.assertTrue(message.HasField('optional_import_message'))
@@ -430,7 +434,8 @@ def ExpectAllFieldsSet(test_case, message):
   test_case.assertEqual('115', message.optional_string)
   test_case.assertEqual(b'116', message.optional_bytes)
 
-  test_case.assertEqual(117, message.optionalgroup.a)
+  if IsProto2(message):
+    test_case.assertEqual(117, message.optionalgroup.a)
   test_case.assertEqual(118, message.optional_nested_message.bb)
   test_case.assertEqual(119, message.optional_foreign_message.c)
   test_case.assertEqual(120, message.optional_import_message.d)
@@ -463,7 +468,8 @@ def ExpectAllFieldsSet(test_case, message):
   test_case.assertEqual(2, len(message.repeated_string))
   test_case.assertEqual(2, len(message.repeated_bytes))
 
-  test_case.assertEqual(2, len(message.repeatedgroup))
+  if IsProto2(message):
+    test_case.assertEqual(2, len(message.repeatedgroup))
   test_case.assertEqual(2, len(message.repeated_nested_message))
   test_case.assertEqual(2, len(message.repeated_foreign_message))
   test_case.assertEqual(2, len(message.repeated_import_message))
@@ -491,7 +497,8 @@ def ExpectAllFieldsSet(test_case, message):
   test_case.assertEqual('215', message.repeated_string[0])
   test_case.assertEqual(b'216', message.repeated_bytes[0])
 
-  test_case.assertEqual(217, message.repeatedgroup[0].a)
+  if IsProto2(message):
+    test_case.assertEqual(217, message.repeatedgroup[0].a)
   test_case.assertEqual(218, message.repeated_nested_message[0].bb)
   test_case.assertEqual(219, message.repeated_foreign_message[0].c)
   test_case.assertEqual(220, message.repeated_import_message[0].d)
@@ -521,7 +528,8 @@ def ExpectAllFieldsSet(test_case, message):
   test_case.assertEqual('315', message.repeated_string[1])
   test_case.assertEqual(b'316', message.repeated_bytes[1])
 
-  test_case.assertEqual(317, message.repeatedgroup[1].a)
+  if IsProto2(message):
+    test_case.assertEqual(317, message.repeatedgroup[1].a)
   test_case.assertEqual(318, message.repeated_nested_message[1].bb)
   test_case.assertEqual(319, message.repeated_foreign_message[1].c)
   test_case.assertEqual(320, message.repeated_import_message[1].d)

python/google/protobuf/internal/text_encoding_test.py

@@ -33,7 +33,6 @@
 """Tests for google.protobuf.text_encoding."""
 
 import unittest
-
 from google.protobuf import text_encoding
 
 TEST_VALUES = [

python/google/protobuf/internal/text_format_test.py

@@ -37,8 +37,10 @@ __author__ = 'kenton@google.com (Kenton Varda)'
 import re
 import unittest
 
+import unittest
 from google.protobuf.internal import _parameterized
 
+from google.protobuf import map_unittest_pb2
 from google.protobuf import unittest_mset_pb2
 from google.protobuf import unittest_pb2
 from google.protobuf import unittest_proto3_arena_pb2
@@ -309,31 +311,6 @@ class TextFormatTest(TextFormatBase):
          r'"unknown_field".'),
         text_format.Parse, text, message)
 
-  def testParseGroupNotClosed(self, message_module):
-    message = message_module.TestAllTypes()
-    text = 'RepeatedGroup: <'
-    self.assertRaisesRegexp(
-        text_format.ParseError, '1:16 : Expected ">".',
-        text_format.Parse, text, message)
-
-    text = 'RepeatedGroup: {'
-    self.assertRaisesRegexp(
-        text_format.ParseError, '1:16 : Expected "}".',
-        text_format.Parse, text, message)
-
-  def testParseEmptyGroup(self, message_module):
-    message = message_module.TestAllTypes()
-    text = 'OptionalGroup: {}'
-    text_format.Parse(text, message)
-    self.assertTrue(message.HasField('optionalgroup'))
-
-    message.Clear()
-
-    message = message_module.TestAllTypes()
-    text = 'OptionalGroup: <>'
-    text_format.Parse(text, message)
-    self.assertTrue(message.HasField('optionalgroup'))
-
   def testParseBadEnumValue(self, message_module):
     message = message_module.TestAllTypes()
     text = 'optional_nested_enum: BARR'
@@ -408,6 +385,14 @@ class TextFormatTest(TextFormatBase):
 # Ideally the schemas would be made more similar so these tests could pass.
 class OnlyWorksWithProto2RightNowTests(TextFormatBase):
 
+  def testPrintAllFieldsPointy(self, message_module):
+    message = unittest_pb2.TestAllTypes()
+    test_util.SetAllFields(message)
+    self.CompareToGoldenFile(
+        self.RemoveRedundantZeros(
+            text_format.MessageToString(message, pointy_brackets=True)),
+        'text_format_unittest_data_pointy_oneof.txt')
+
   def testParseGolden(self):
     golden_text = '\n'.join(self.ReadGolden('text_format_unittest_data.txt'))
     parsed_message = unittest_pb2.TestAllTypes()
@@ -471,8 +456,49 @@ class OnlyWorksWithProto2RightNowTests(TextFormatBase):
     test_util.SetAllFields(message)
     self.assertEquals(message, parsed_message)
 
+  def testPrintMap(self):
+    message = map_unittest_pb2.TestMap()
 
-# Tests of proto2-only features (MessageSet and extensions).
+    message.map_int32_int32[-123] = -456
+    message.map_int64_int64[-2**33] = -2**34
+    message.map_uint32_uint32[123] = 456
+    message.map_uint64_uint64[2**33] = 2**34
+    message.map_string_string["abc"] = "123"
+    message.map_int32_foreign_message[111].c = 5
+
+    # Maps are serialized to text format using their underlying repeated
+    # representation.
+    self.CompareToGoldenText(
+        text_format.MessageToString(message),
+        'map_int32_int32 {\n'
+        '  key: -123\n'
+        '  value: -456\n'
+        '}\n'
+        'map_int64_int64 {\n'
+        '  key: -8589934592\n'
+        '  value: -17179869184\n'
+        '}\n'
+        'map_uint32_uint32 {\n'
+        '  key: 123\n'
+        '  value: 456\n'
+        '}\n'
+        'map_uint64_uint64 {\n'
+        '  key: 8589934592\n'
+        '  value: 17179869184\n'
+        '}\n'
+        'map_string_string {\n'
+        '  key: "abc"\n'
+        '  value: "123"\n'
+        '}\n'
+        'map_int32_foreign_message {\n'
+        '  key: 111\n'
+        '  value {\n'
+        '    c: 5\n'
+        '  }\n'
+        '}\n')
+
+
+# Tests of proto2-only features (MessageSet, extensions, etc.).
 class Proto2Tests(TextFormatBase):
 
   def testPrintMessageSet(self):
@@ -620,6 +646,69 @@ class Proto2Tests(TextFormatBase):
          'have multiple "optional_int32" fields.'),
         text_format.Parse, text, message)
 
+  def testParseGroupNotClosed(self):
+    message = unittest_pb2.TestAllTypes()
+    text = 'RepeatedGroup: <'
+    self.assertRaisesRegexp(
+        text_format.ParseError, '1:16 : Expected ">".',
+        text_format.Parse, text, message)
+    text = 'RepeatedGroup: {'
+    self.assertRaisesRegexp(
+        text_format.ParseError, '1:16 : Expected "}".',
+        text_format.Parse, text, message)
+
+  def testParseEmptyGroup(self):
+    message = unittest_pb2.TestAllTypes()
+    text = 'OptionalGroup: {}'
+    text_format.Parse(text, message)
+    self.assertTrue(message.HasField('optionalgroup'))
+
+    message.Clear()
+
+    message = unittest_pb2.TestAllTypes()
+    text = 'OptionalGroup: <>'
+    text_format.Parse(text, message)
+    self.assertTrue(message.HasField('optionalgroup'))
+
+  # Maps aren't really proto2-only, but our test schema only has maps for
+  # proto2.
+  def testParseMap(self):
+    text = ('map_int32_int32 {\n'
+            '  key: -123\n'
+            '  value: -456\n'
+            '}\n'
+            'map_int64_int64 {\n'
+            '  key: -8589934592\n'
+            '  value: -17179869184\n'
+            '}\n'
+            'map_uint32_uint32 {\n'
+            '  key: 123\n'
+            '  value: 456\n'
+            '}\n'
+            'map_uint64_uint64 {\n'
+            '  key: 8589934592\n'
+            '  value: 17179869184\n'
+            '}\n'
+            'map_string_string {\n'
+            '  key: "abc"\n'
+            '  value: "123"\n'
+            '}\n'
+            'map_int32_foreign_message {\n'
+            '  key: 111\n'
+            '  value {\n'
+            '    c: 5\n'
+            '  }\n'
+            '}\n')
+    message = map_unittest_pb2.TestMap()
+    text_format.Parse(text, message)
+
+    self.assertEqual(-456, message.map_int32_int32[-123])
+    self.assertEqual(-2**34, message.map_int64_int64[-2**33])
+    self.assertEqual(456, message.map_uint32_uint32[123])
+    self.assertEqual(2**34, message.map_uint64_uint64[2**33])
+    self.assertEqual("123", message.map_string_string["abc"])
+    self.assertEqual(5, message.map_int32_foreign_message[111].c)
+
 
 class TokenizerTest(unittest.TestCase):
 

python/google/protobuf/internal/type_checkers.py

@@ -129,6 +129,9 @@ class IntValueChecker(object):
     proposed_value = self._TYPE(proposed_value)
     return proposed_value
 
+  def DefaultValue(self):
+    return 0
+
 
 class EnumValueChecker(object):
 
@@ -146,6 +149,9 @@ class EnumValueChecker(object):
       raise ValueError('Unknown enum value: %d' % proposed_value)
     return proposed_value
 
+  def DefaultValue(self):
+    return self._enum_type.values[0].number
+
 
 class UnicodeValueChecker(object):
 
@@ -171,6 +177,9 @@ class UnicodeValueChecker(object):
                          (proposed_value))
     return proposed_value
 
+  def DefaultValue(self):
+    return u""
+
 
 class Int32ValueChecker(IntValueChecker):
   # We're sure to use ints instead of longs here since comparison may be more

python/google/protobuf/internal/unknown_fields_test.py

@@ -36,7 +36,6 @@
 __author__ = 'bohdank@google.com (Bohdan Koval)'
 
 import unittest
-
 from google.protobuf import unittest_mset_pb2
 from google.protobuf import unittest_pb2
 from google.protobuf import unittest_proto3_arena_pb2

python/google/protobuf/internal/wire_format_test.py

@@ -35,7 +35,6 @@
 __author__ = 'robinson@google.com (Will Robinson)'
 
 import unittest
-
 from google.protobuf import message
 from google.protobuf.internal import wire_format
 

python/google/protobuf/proto_builder.py

@@ -30,6 +30,7 @@
 
 """Dynamic Protobuf class creator."""
 
+import collections
 import hashlib
 import os
 
@@ -59,7 +60,9 @@ def MakeSimpleProtoClass(fields, full_name, pool=None):
   Note: this doesn't validate field names!
 
   Args:
-    fields: dict of {name: field_type} mappings for each field in the proto.
+    fields: dict of {name: field_type} mappings for each field in the proto. If
+        this is an OrderedDict the order will be maintained, otherwise the
+        fields will be sorted by name.
     full_name: str, the fully-qualified name of the proto type.
     pool: optional DescriptorPool instance.
   Returns:
@@ -73,12 +76,19 @@ def MakeSimpleProtoClass(fields, full_name, pool=None):
     # The factory's DescriptorPool doesn't know about this class yet.
     pass
 
+  # Get a list of (name, field_type) tuples from the fields dict. If fields was
+  # an OrderedDict we keep the order, but otherwise we sort the field to ensure
+  # consistent ordering.
+  field_items = fields.items()
+  if not isinstance(fields, collections.OrderedDict):
+    field_items = sorted(field_items)
+
   # Use a consistent file name that is unlikely to conflict with any imported
   # proto files.
   fields_hash = hashlib.sha1()
-  for f_name, f_type in sorted(fields.items()):
-    fields_hash.update(f_name.encode('utf8'))
-    fields_hash.update(str(f_type).encode('utf8'))
+  for f_name, f_type in field_items:
+    fields_hash.update(f_name.encode('utf-8'))
+    fields_hash.update(str(f_type).encode('utf-8'))
   proto_file_name = fields_hash.hexdigest() + '.proto'
 
   package, name = full_name.rsplit('.', 1)
@@ -87,7 +97,7 @@ def MakeSimpleProtoClass(fields, full_name, pool=None):
   file_proto.package = package
   desc_proto = file_proto.message_type.add()
   desc_proto.name = name
-  for f_number, (f_name, f_type) in enumerate(sorted(fields.items()), 1):
+  for f_number, (f_name, f_type) in enumerate(field_items, 1):
     field_proto = desc_proto.field.add()
     field_proto.name = f_name
     field_proto.number = f_number

python/google/protobuf/pyext/descriptor.cc

@@ -43,8 +43,6 @@
 #include <google/protobuf/pyext/message.h>
 #include <google/protobuf/pyext/scoped_pyobject_ptr.h>
 
-#define C(str) const_cast<char*>(str)
-
 #if PY_MAJOR_VERSION >= 3
   #define PyString_FromStringAndSize PyUnicode_FromStringAndSize
   #define PyString_Check PyUnicode_Check
@@ -257,8 +255,14 @@ namespace descriptor {
 // Creates or retrieve a Python descriptor of the specified type.
 // Objects are interned: the same descriptor will return the same object if it
 // was kept alive.
+// 'was_created' is an optional pointer to a bool, and is set to true if a new
+// object was allocated.
 // Always return a new reference.
-PyObject* NewInternedDescriptor(PyTypeObject* type, const void* descriptor) {
+PyObject* NewInternedDescriptor(PyTypeObject* type, const void* descriptor,
+                                bool* was_created) {
+  if (was_created) {
+    *was_created = false;
+  }
   if (descriptor == NULL) {
     PyErr_BadInternalCall();
     return NULL;
@@ -283,6 +287,9 @@ PyObject* NewInternedDescriptor(PyTypeObject* type, const void* descriptor) {
   GetDescriptorPool()->interned_descriptors->insert(
       std::make_pair(descriptor, reinterpret_cast<PyObject*>(py_descriptor)));
 
+  if (was_created) {
+    *was_created = true;
+  }
   return reinterpret_cast<PyObject*>(py_descriptor);
 }
 
@@ -298,9 +305,7 @@ static PyGetSetDef Getters[] = {
 
 PyTypeObject PyBaseDescriptor_Type = {
   PyVarObject_HEAD_INIT(&PyType_Type, 0)
-  // Keep the fully qualified _message symbol in a line for opensource.
-  "google.protobuf.internal._message."
-  "DescriptorBase",                     // tp_name
+  FULL_MODULE_NAME ".DescriptorBase",   // tp_name
   sizeof(PyBaseDescriptor),             // tp_basicsize
   0,                                    // tp_itemsize
   (destructor)Dealloc,                  // tp_dealloc
@@ -357,7 +362,7 @@ static PyObject* GetFullName(PyBaseDescriptor* self, void *closure) {
 }
 
 static PyObject* GetFile(PyBaseDescriptor *self, void *closure) {
-  return PyFileDescriptor_New(_GetDescriptor(self)->file());
+  return PyFileDescriptor_FromDescriptor(_GetDescriptor(self)->file());
 }
 
 static PyObject* GetConcreteClass(PyBaseDescriptor* self, void *closure) {
@@ -367,17 +372,6 @@ static PyObject* GetConcreteClass(PyBaseDescriptor* self, void *closure) {
   return concrete_class;
 }
 
-static int SetConcreteClass(PyBaseDescriptor *self, PyObject *value,
-                            void *closure) {
-  // This attribute is also set from reflection.py. Check that it's actually a
-  // no-op.
-  if (value != cdescriptor_pool::GetMessageClass(
-          GetDescriptorPool(), _GetDescriptor(self))) {
-    PyErr_SetString(PyExc_AttributeError, "Cannot change _concrete_class");
-  }
-  return 0;
-}
-
 static PyObject* GetFieldsByName(PyBaseDescriptor* self, void *closure) {
   return NewMessageFieldsByName(_GetDescriptor(self));
 }
@@ -452,7 +446,7 @@ static PyObject* GetContainingType(PyBaseDescriptor *self, void *closure) {
   const Descriptor* containing_type =
       _GetDescriptor(self)->containing_type();
   if (containing_type) {
-    return PyMessageDescriptor_New(containing_type);
+    return PyMessageDescriptor_FromDescriptor(containing_type);
   } else {
     Py_RETURN_NONE;
   }
@@ -515,29 +509,34 @@ static PyObject* GetSyntax(PyBaseDescriptor *self, void *closure) {
 }
 
 static PyGetSetDef Getters[] = {
-  { C("name"), (getter)GetName, NULL, "Last name", NULL},
-  { C("full_name"), (getter)GetFullName, NULL, "Full name", NULL},
-  { C("_concrete_class"), (getter)GetConcreteClass, (setter)SetConcreteClass, "concrete class", NULL},
-  { C("file"), (getter)GetFile, NULL, "File descriptor", NULL},
-
-  { C("fields"), (getter)GetFieldsSeq, NULL, "Fields sequence", NULL},
-  { C("fields_by_name"), (getter)GetFieldsByName, NULL, "Fields by name", NULL},
-  { C("fields_by_number"), (getter)GetFieldsByNumber, NULL, "Fields by number", NULL},
-  { C("nested_types"), (getter)GetNestedTypesSeq, NULL, "Nested types sequence", NULL},
-  { C("nested_types_by_name"), (getter)GetNestedTypesByName, NULL, "Nested types by name", NULL},
-  { C("extensions"), (getter)GetExtensions, NULL, "Extensions Sequence", NULL},
-  { C("extensions_by_name"), (getter)GetExtensionsByName, NULL, "Extensions by name", NULL},
-  { C("extension_ranges"), (getter)GetExtensionRanges, NULL, "Extension ranges", NULL},
-  { C("enum_types"), (getter)GetEnumsSeq, NULL, "Enum sequence", NULL},
-  { C("enum_types_by_name"), (getter)GetEnumTypesByName, NULL, "Enum types by name", NULL},
-  { C("enum_values_by_name"), (getter)GetEnumValuesByName, NULL, "Enum values by name", NULL},
-  { C("oneofs_by_name"), (getter)GetOneofsByName, NULL, "Oneofs by name", NULL},
-  { C("oneofs"), (getter)GetOneofsSeq, NULL, "Oneofs by name", NULL},
-  { C("containing_type"), (getter)GetContainingType, (setter)SetContainingType, "Containing type", NULL},
-  { C("is_extendable"), (getter)IsExtendable, (setter)NULL, NULL, NULL},
-  { C("has_options"), (getter)GetHasOptions, (setter)SetHasOptions, "Has Options", NULL},
-  { C("_options"), (getter)NULL, (setter)SetOptions, "Options", NULL},
-  { C("syntax"), (getter)GetSyntax, (setter)NULL, "Syntax", NULL},
+  { "name", (getter)GetName, NULL, "Last name"},
+  { "full_name", (getter)GetFullName, NULL, "Full name"},
+  { "_concrete_class", (getter)GetConcreteClass, NULL, "concrete class"},
+  { "file", (getter)GetFile, NULL, "File descriptor"},
+
+  { "fields", (getter)GetFieldsSeq, NULL, "Fields sequence"},
+  { "fields_by_name", (getter)GetFieldsByName, NULL, "Fields by name"},
+  { "fields_by_number", (getter)GetFieldsByNumber, NULL, "Fields by number"},
+  { "nested_types", (getter)GetNestedTypesSeq, NULL, "Nested types sequence"},
+  { "nested_types_by_name", (getter)GetNestedTypesByName, NULL,
+    "Nested types by name"},
+  { "extensions", (getter)GetExtensions, NULL, "Extensions Sequence"},
+  { "extensions_by_name", (getter)GetExtensionsByName, NULL,
+    "Extensions by name"},
+  { "extension_ranges", (getter)GetExtensionRanges, NULL, "Extension ranges"},
+  { "enum_types", (getter)GetEnumsSeq, NULL, "Enum sequence"},
+  { "enum_types_by_name", (getter)GetEnumTypesByName, NULL,
+    "Enum types by name"},
+  { "enum_values_by_name", (getter)GetEnumValuesByName, NULL,
+    "Enum values by name"},
+  { "oneofs_by_name", (getter)GetOneofsByName, NULL, "Oneofs by name"},
+  { "oneofs", (getter)GetOneofsSeq, NULL, "Oneofs by name"},
+  { "containing_type", (getter)GetContainingType, (setter)SetContainingType,
+    "Containing type"},
+  { "is_extendable", (getter)IsExtendable, (setter)NULL},
+  { "has_options", (getter)GetHasOptions, (setter)SetHasOptions, "Has Options"},
+  { "_options", (getter)NULL, (setter)SetOptions, "Options"},
+  { "syntax", (getter)GetSyntax, (setter)NULL, "Syntax"},
   {NULL}
 };
 
@@ -552,9 +551,7 @@ static PyMethodDef Methods[] = {
 
 PyTypeObject PyMessageDescriptor_Type = {
   PyVarObject_HEAD_INIT(&PyType_Type, 0)
-  // Keep the fully qualified _message symbol in a line for opensource.
-  C("google.protobuf.internal._message."
-    "MessageDescriptor"),      // tp_name
+  FULL_MODULE_NAME ".MessageDescriptor",  // tp_name
   sizeof(PyBaseDescriptor),             // tp_basicsize
   0,                                    // tp_itemsize
   0,                                    // tp_dealloc
@@ -573,7 +570,7 @@ PyTypeObject PyMessageDescriptor_Type = {
   0,                                    // tp_setattro
   0,                                    // tp_as_buffer
   Py_TPFLAGS_DEFAULT,                   // tp_flags
-  C("A Message Descriptor"),            // tp_doc
+  "A Message Descriptor",               // tp_doc
   0,                                    // tp_traverse
   0,                                    // tp_clear
   0,                                    // tp_richcompare
@@ -586,10 +583,10 @@ PyTypeObject PyMessageDescriptor_Type = {
   &descriptor::PyBaseDescriptor_Type,   // tp_base
 };
 
-PyObject* PyMessageDescriptor_New(
+PyObject* PyMessageDescriptor_FromDescriptor(
     const Descriptor* message_descriptor) {
   return descriptor::NewInternedDescriptor(
-      &PyMessageDescriptor_Type, message_descriptor);
+      &PyMessageDescriptor_Type, message_descriptor, NULL);
 }
 
 const Descriptor* PyMessageDescriptor_AsDescriptor(PyObject* obj) {
@@ -715,7 +712,7 @@ static PyObject* GetCDescriptor(PyObject *self, void *closure) {
 static PyObject *GetEnumType(PyBaseDescriptor *self, void *closure) {
   const EnumDescriptor* enum_type = _GetDescriptor(self)->enum_type();
   if (enum_type) {
-    return PyEnumDescriptor_New(enum_type);
+    return PyEnumDescriptor_FromDescriptor(enum_type);
   } else {
     Py_RETURN_NONE;
   }
@@ -728,7 +725,7 @@ static int SetEnumType(PyBaseDescriptor *self, PyObject *value, void *closure) {
 static PyObject *GetMessageType(PyBaseDescriptor *self, void *closure) {
   const Descriptor* message_type = _GetDescriptor(self)->message_type();
   if (message_type) {
-    return PyMessageDescriptor_New(message_type);
+    return PyMessageDescriptor_FromDescriptor(message_type);
   } else {
     Py_RETURN_NONE;
   }
@@ -743,7 +740,7 @@ static PyObject* GetContainingType(PyBaseDescriptor *self, void *closure) {
   const Descriptor* containing_type =
       _GetDescriptor(self)->containing_type();
   if (containing_type) {
-    return PyMessageDescriptor_New(containing_type);
+    return PyMessageDescriptor_FromDescriptor(containing_type);
   } else {
     Py_RETURN_NONE;
   }
@@ -758,7 +755,7 @@ static PyObject* GetExtensionScope(PyBaseDescriptor *self, void *closure) {
   const Descriptor* extension_scope =
       _GetDescriptor(self)->extension_scope();
   if (extension_scope) {
-    return PyMessageDescriptor_New(extension_scope);
+    return PyMessageDescriptor_FromDescriptor(extension_scope);
   } else {
     Py_RETURN_NONE;
   }
@@ -768,7 +765,7 @@ static PyObject* GetContainingOneof(PyBaseDescriptor *self, void *closure) {
   const OneofDescriptor* containing_oneof =
       _GetDescriptor(self)->containing_oneof();
   if (containing_oneof) {
-    return PyOneofDescriptor_New(containing_oneof);
+    return PyOneofDescriptor_FromDescriptor(containing_oneof);
   } else {
     Py_RETURN_NONE;
   }
@@ -803,26 +800,30 @@ static int SetOptions(PyBaseDescriptor *self, PyObject *value,
 
 
 static PyGetSetDef Getters[] = {
-  { C("full_name"), (getter)GetFullName, NULL, "Full name", NULL},
-  { C("name"), (getter)GetName, NULL, "Unqualified name", NULL},
-  { C("type"), (getter)GetType, NULL, "C++ Type", NULL},
-  { C("cpp_type"), (getter)GetCppType, NULL, "C++ Type", NULL},
-  { C("label"), (getter)GetLabel, NULL, "Label", NULL},
-  { C("number"), (getter)GetNumber, NULL, "Number", NULL},
-  { C("index"), (getter)GetIndex, NULL, "Index", NULL},
-  { C("default_value"), (getter)GetDefaultValue, NULL, "Default Value", NULL},
-  { C("has_default_value"), (getter)HasDefaultValue, NULL, NULL, NULL},
-  { C("is_extension"), (getter)IsExtension, NULL, "ID", NULL},
-  { C("id"), (getter)GetID, NULL, "ID", NULL},
-  { C("_cdescriptor"), (getter)GetCDescriptor, NULL, "HAACK REMOVE ME", NULL},
-
-  { C("message_type"), (getter)GetMessageType, (setter)SetMessageType, "Message type", NULL},
-  { C("enum_type"), (getter)GetEnumType, (setter)SetEnumType, "Enum type", NULL},
-  { C("containing_type"), (getter)GetContainingType, (setter)SetContainingType, "Containing type", NULL},
-  { C("extension_scope"), (getter)GetExtensionScope, (setter)NULL, "Extension scope", NULL},
-  { C("containing_oneof"), (getter)GetContainingOneof, (setter)SetContainingOneof, "Containing oneof", NULL},
-  { C("has_options"), (getter)GetHasOptions, (setter)SetHasOptions, "Has Options", NULL},
-  { C("_options"), (getter)NULL, (setter)SetOptions, "Options", NULL},
+  { "full_name", (getter)GetFullName, NULL, "Full name"},
+  { "name", (getter)GetName, NULL, "Unqualified name"},
+  { "type", (getter)GetType, NULL, "C++ Type"},
+  { "cpp_type", (getter)GetCppType, NULL, "C++ Type"},
+  { "label", (getter)GetLabel, NULL, "Label"},
+  { "number", (getter)GetNumber, NULL, "Number"},
+  { "index", (getter)GetIndex, NULL, "Index"},
+  { "default_value", (getter)GetDefaultValue, NULL, "Default Value"},
+  { "has_default_value", (getter)HasDefaultValue},
+  { "is_extension", (getter)IsExtension, NULL, "ID"},
+  { "id", (getter)GetID, NULL, "ID"},
+  { "_cdescriptor", (getter)GetCDescriptor, NULL, "HAACK REMOVE ME"},
+
+  { "message_type", (getter)GetMessageType, (setter)SetMessageType,
+    "Message type"},
+  { "enum_type", (getter)GetEnumType, (setter)SetEnumType, "Enum type"},
+  { "containing_type", (getter)GetContainingType, (setter)SetContainingType,
+    "Containing type"},
+  { "extension_scope", (getter)GetExtensionScope, (setter)NULL,
+    "Extension scope"},
+  { "containing_oneof", (getter)GetContainingOneof, (setter)SetContainingOneof,
+    "Containing oneof"},
+  { "has_options", (getter)GetHasOptions, (setter)SetHasOptions, "Has Options"},
+  { "_options", (getter)NULL, (setter)SetOptions, "Options"},
   {NULL}
 };
 
@@ -835,8 +836,7 @@ static PyMethodDef Methods[] = {
 
 PyTypeObject PyFieldDescriptor_Type = {
   PyVarObject_HEAD_INIT(&PyType_Type, 0)
-  C("google.protobuf.internal."
-    "_message.FieldDescriptor"),        // tp_name
+  FULL_MODULE_NAME ".FieldDescriptor",  // tp_name
   sizeof(PyBaseDescriptor),             // tp_basicsize
   0,                                    // tp_itemsize
   0,                                    // tp_dealloc
@@ -855,7 +855,7 @@ PyTypeObject PyFieldDescriptor_Type = {
   0,                                    // tp_setattro
   0,                                    // tp_as_buffer
   Py_TPFLAGS_DEFAULT,                   // tp_flags
-  C("A Field Descriptor"),              // tp_doc
+  "A Field Descriptor",                 // tp_doc
   0,                                    // tp_traverse
   0,                                    // tp_clear
   0,                                    // tp_richcompare
@@ -868,10 +868,10 @@ PyTypeObject PyFieldDescriptor_Type = {
   &descriptor::PyBaseDescriptor_Type,   // tp_base
 };
 
-PyObject* PyFieldDescriptor_New(
+PyObject* PyFieldDescriptor_FromDescriptor(
     const FieldDescriptor* field_descriptor) {
   return descriptor::NewInternedDescriptor(
-      &PyFieldDescriptor_Type, field_descriptor);
+      &PyFieldDescriptor_Type, field_descriptor, NULL);
 }
 
 const FieldDescriptor* PyFieldDescriptor_AsDescriptor(PyObject* obj) {
@@ -900,7 +900,7 @@ static PyObject* GetName(PyBaseDescriptor *self, void *closure) {
 }
 
 static PyObject* GetFile(PyBaseDescriptor *self, void *closure) {
-  return PyFileDescriptor_New(_GetDescriptor(self)->file());
+  return PyFileDescriptor_FromDescriptor(_GetDescriptor(self)->file());
 }
 
 static PyObject* GetEnumvaluesByName(PyBaseDescriptor* self, void *closure) {
@@ -919,7 +919,7 @@ static PyObject* GetContainingType(PyBaseDescriptor *self, void *closure) {
   const Descriptor* containing_type =
       _GetDescriptor(self)->containing_type();
   if (containing_type) {
-    return PyMessageDescriptor_New(containing_type);
+    return PyMessageDescriptor_FromDescriptor(containing_type);
   } else {
     Py_RETURN_NONE;
   }
@@ -964,16 +964,19 @@ static PyMethodDef Methods[] = {
 };
 
 static PyGetSetDef Getters[] = {
-  { C("full_name"), (getter)GetFullName, NULL, "Full name", NULL},
-  { C("name"), (getter)GetName, NULL, "last name", NULL},
-  { C("file"), (getter)GetFile, NULL, "File descriptor", NULL},
-  { C("values"), (getter)GetEnumvaluesSeq, NULL, "values", NULL},
-  { C("values_by_name"), (getter)GetEnumvaluesByName, NULL, "Enumvalues by name", NULL},
-  { C("values_by_number"), (getter)GetEnumvaluesByNumber, NULL, "Enumvalues by number", NULL},
-
-  { C("containing_type"), (getter)GetContainingType, (setter)SetContainingType, "Containing type", NULL},
-  { C("has_options"), (getter)GetHasOptions, (setter)SetHasOptions, "Has Options", NULL},
-  { C("_options"), (getter)NULL, (setter)SetOptions, "Options", NULL},
+  { "full_name", (getter)GetFullName, NULL, "Full name"},
+  { "name", (getter)GetName, NULL, "last name"},
+  { "file", (getter)GetFile, NULL, "File descriptor"},
+  { "values", (getter)GetEnumvaluesSeq, NULL, "values"},
+  { "values_by_name", (getter)GetEnumvaluesByName, NULL,
+    "Enum values by name"},
+  { "values_by_number", (getter)GetEnumvaluesByNumber, NULL,
+    "Enum values by number"},
+
+  { "containing_type", (getter)GetContainingType, (setter)SetContainingType,
+    "Containing type"},
+  { "has_options", (getter)GetHasOptions, (setter)SetHasOptions, "Has Options"},
+  { "_options", (getter)NULL, (setter)SetOptions, "Options"},
   {NULL}
 };
 
@@ -981,9 +984,7 @@ static PyGetSetDef Getters[] = {
 
 PyTypeObject PyEnumDescriptor_Type = {
   PyVarObject_HEAD_INIT(&PyType_Type, 0)
-  // Keep the fully qualified _message symbol in a line for opensource.
-  C("google.protobuf.internal._message."
-    "EnumDescriptor"),                  // tp_name
+  FULL_MODULE_NAME ".EnumDescriptor",   // tp_name
   sizeof(PyBaseDescriptor),             // tp_basicsize
   0,                                    // tp_itemsize
   0,                                    // tp_dealloc
@@ -1002,7 +1003,7 @@ PyTypeObject PyEnumDescriptor_Type = {
   0,                                    // tp_setattro
   0,                                    // tp_as_buffer
   Py_TPFLAGS_DEFAULT,                   // tp_flags
-  C("A Enum Descriptor"),               // tp_doc
+  "A Enum Descriptor",                  // tp_doc
   0,                                    // tp_traverse
   0,                                    // tp_clear
   0,                                    // tp_richcompare
@@ -1015,10 +1016,10 @@ PyTypeObject PyEnumDescriptor_Type = {
   &descriptor::PyBaseDescriptor_Type,   // tp_base
 };
 
-PyObject* PyEnumDescriptor_New(
+PyObject* PyEnumDescriptor_FromDescriptor(
     const EnumDescriptor* enum_descriptor) {
   return descriptor::NewInternedDescriptor(
-      &PyEnumDescriptor_Type, enum_descriptor);
+      &PyEnumDescriptor_Type, enum_descriptor, NULL);
 }
 
 namespace enumvalue_descriptor {
@@ -1042,7 +1043,7 @@ static PyObject* GetIndex(PyBaseDescriptor *self, void *closure) {
 }
 
 static PyObject* GetType(PyBaseDescriptor *self, void *closure) {
-  return PyEnumDescriptor_New(_GetDescriptor(self)->type());
+  return PyEnumDescriptor_FromDescriptor(_GetDescriptor(self)->type());
 }
 
 static PyObject* GetHasOptions(PyBaseDescriptor *self, void *closure) {
@@ -1069,13 +1070,13 @@ static int SetOptions(PyBaseDescriptor *self, PyObject *value,
 
 
 static PyGetSetDef Getters[] = {
-  { C("name"), (getter)GetName, NULL, "name", NULL},
-  { C("number"), (getter)GetNumber, NULL, "number", NULL},
-  { C("index"), (getter)GetIndex, NULL, "index", NULL},
-  { C("type"), (getter)GetType, NULL, "index", NULL},
+  { "name", (getter)GetName, NULL, "name"},
+  { "number", (getter)GetNumber, NULL, "number"},
+  { "index", (getter)GetIndex, NULL, "index"},
+  { "type", (getter)GetType, NULL, "index"},
 
-  { C("has_options"), (getter)GetHasOptions, (setter)SetHasOptions, "Has Options", NULL},
-  { C("_options"), (getter)NULL, (setter)SetOptions, "Options", NULL},
+  { "has_options", (getter)GetHasOptions, (setter)SetHasOptions, "Has Options"},
+  { "_options", (getter)NULL, (setter)SetOptions, "Options"},
   {NULL}
 };
 
@@ -1088,8 +1089,7 @@ static PyMethodDef Methods[] = {
 
 PyTypeObject PyEnumValueDescriptor_Type = {
   PyVarObject_HEAD_INIT(&PyType_Type, 0)
-  C("google.protobuf.internal."
-    "_message.EnumValueDescriptor"),    // tp_name
+  FULL_MODULE_NAME ".EnumValueDescriptor",  // tp_name
   sizeof(PyBaseDescriptor),             // tp_basicsize
   0,                                    // tp_itemsize
   0,                                    // tp_dealloc
@@ -1108,7 +1108,7 @@ PyTypeObject PyEnumValueDescriptor_Type = {
   0,                                    // tp_setattro
   0,                                    // tp_as_buffer
   Py_TPFLAGS_DEFAULT,                   // tp_flags
-  C("A EnumValue Descriptor"),          // tp_doc
+  "A EnumValue Descriptor",             // tp_doc
   0,                                    // tp_traverse
   0,                                    // tp_clear
   0,                                    // tp_richcompare
@@ -1121,10 +1121,10 @@ PyTypeObject PyEnumValueDescriptor_Type = {
   &descriptor::PyBaseDescriptor_Type,   // tp_base
 };
 
-PyObject* PyEnumValueDescriptor_New(
+PyObject* PyEnumValueDescriptor_FromDescriptor(
     const EnumValueDescriptor* enumvalue_descriptor) {
   return descriptor::NewInternedDescriptor(
-      &PyEnumValueDescriptor_Type, enumvalue_descriptor);
+      &PyEnumValueDescriptor_Type, enumvalue_descriptor, NULL);
 }
 
 namespace file_descriptor {
@@ -1218,18 +1218,20 @@ static PyObject* CopyToProto(PyFileDescriptor *self, PyObject *target) {
 }
 
 static PyGetSetDef Getters[] = {
-  { C("name"), (getter)GetName, NULL, "name", NULL},
-  { C("package"), (getter)GetPackage, NULL, "package", NULL},
-  { C("serialized_pb"), (getter)GetSerializedPb, NULL, NULL, NULL},
-  { C("message_types_by_name"), (getter)GetMessageTypesByName, NULL, "Messages by name", NULL},
-  { C("enum_types_by_name"), (getter)GetEnumTypesByName, NULL, "Enums by name", NULL},
-  { C("extensions_by_name"), (getter)GetExtensionsByName, NULL, "Extensions by name", NULL},
-  { C("dependencies"), (getter)GetDependencies, NULL, "Dependencies", NULL},
-  { C("public_dependencies"), (getter)GetPublicDependencies, NULL, "Dependencies", NULL},
-
-  { C("has_options"), (getter)GetHasOptions, (setter)SetHasOptions, "Has Options", NULL},
-  { C("_options"), (getter)NULL, (setter)SetOptions, "Options", NULL},
-  { C("syntax"), (getter)GetSyntax, (setter)NULL, "Syntax", NULL},
+  { "name", (getter)GetName, NULL, "name"},
+  { "package", (getter)GetPackage, NULL, "package"},
+  { "serialized_pb", (getter)GetSerializedPb},
+  { "message_types_by_name", (getter)GetMessageTypesByName, NULL,
+    "Messages by name"},
+  { "enum_types_by_name", (getter)GetEnumTypesByName, NULL, "Enums by name"},
+  { "extensions_by_name", (getter)GetExtensionsByName, NULL,
+    "Extensions by name"},
+  { "dependencies", (getter)GetDependencies, NULL, "Dependencies"},
+  { "public_dependencies", (getter)GetPublicDependencies, NULL, "Dependencies"},
+
+  { "has_options", (getter)GetHasOptions, (setter)SetHasOptions, "Has Options"},
+  { "_options", (getter)NULL, (setter)SetOptions, "Options"},
+  { "syntax", (getter)GetSyntax, (setter)NULL, "Syntax"},
   {NULL}
 };
 
@@ -1243,11 +1245,10 @@ static PyMethodDef Methods[] = {
 
 PyTypeObject PyFileDescriptor_Type = {
   PyVarObject_HEAD_INIT(&PyType_Type, 0)
-  C("google.protobuf.internal."
-    "_message.FileDescriptor"),         // tp_name
+  FULL_MODULE_NAME ".FileDescriptor",   // tp_name
   sizeof(PyFileDescriptor),             // tp_basicsize
   0,                                    // tp_itemsize
-  (destructor)file_descriptor::Dealloc, // tp_dealloc
+  (destructor)file_descriptor::Dealloc,  // tp_dealloc
   0,                                    // tp_print
   0,                                    // tp_getattr
   0,                                    // tp_setattr
@@ -1263,7 +1264,7 @@ PyTypeObject PyFileDescriptor_Type = {
   0,                                    // tp_setattro
   0,                                    // tp_as_buffer
   Py_TPFLAGS_DEFAULT,                   // tp_flags
-  C("A File Descriptor"),               // tp_doc
+  "A File Descriptor",                  // tp_doc
   0,                                    // tp_traverse
   0,                                    // tp_clear
   0,                                    // tp_richcompare
@@ -1284,23 +1285,28 @@ PyTypeObject PyFileDescriptor_Type = {
   PyObject_Del,                         // tp_free
 };
 
-PyObject* PyFileDescriptor_New(const FileDescriptor* file_descriptor) {
-  return descriptor::NewInternedDescriptor(
-      &PyFileDescriptor_Type, file_descriptor);
+PyObject* PyFileDescriptor_FromDescriptor(
+    const FileDescriptor* file_descriptor) {
+  return PyFileDescriptor_FromDescriptorWithSerializedPb(file_descriptor,
+                                                         NULL);
 }
 
-PyObject* PyFileDescriptor_NewWithPb(
+PyObject* PyFileDescriptor_FromDescriptorWithSerializedPb(
     const FileDescriptor* file_descriptor, PyObject *serialized_pb) {
-  PyObject* py_descriptor = PyFileDescriptor_New(file_descriptor);
+  bool was_created;
+  PyObject* py_descriptor = descriptor::NewInternedDescriptor(
+      &PyFileDescriptor_Type, file_descriptor, &was_created);
   if (py_descriptor == NULL) {
     return NULL;
   }
-  if (serialized_pb != NULL) {
+  if (was_created) {
     PyFileDescriptor* cfile_descriptor =
         reinterpret_cast<PyFileDescriptor*>(py_descriptor);
     Py_XINCREF(serialized_pb);
     cfile_descriptor->serialized_pb = serialized_pb;
   }
+  // TODO(amauryfa): In the case of a cached object, check that serialized_pb
+  // is the same as before.
 
   return py_descriptor;
 }
@@ -1333,19 +1339,19 @@ static PyObject* GetContainingType(PyBaseDescriptor *self, void *closure) {
   const Descriptor* containing_type =
       _GetDescriptor(self)->containing_type();
   if (containing_type) {
-    return PyMessageDescriptor_New(containing_type);
+    return PyMessageDescriptor_FromDescriptor(containing_type);
   } else {
     Py_RETURN_NONE;
   }
 }
 
 static PyGetSetDef Getters[] = {
-  { C("name"), (getter)GetName, NULL, "Name", NULL},
-  { C("full_name"), (getter)GetFullName, NULL, "Full name", NULL},
-  { C("index"), (getter)GetIndex, NULL, "Index", NULL},
+  { "name", (getter)GetName, NULL, "Name"},
+  { "full_name", (getter)GetFullName, NULL, "Full name"},
+  { "index", (getter)GetIndex, NULL, "Index"},
 
-  { C("containing_type"), (getter)GetContainingType, NULL, "Containing type", NULL},
-  { C("fields"), (getter)GetFields, NULL, "Fields", NULL},
+  { "containing_type", (getter)GetContainingType, NULL, "Containing type"},
+  { "fields", (getter)GetFields, NULL, "Fields"},
   {NULL}
 };
 
@@ -1353,8 +1359,7 @@ static PyGetSetDef Getters[] = {
 
 PyTypeObject PyOneofDescriptor_Type = {
   PyVarObject_HEAD_INIT(&PyType_Type, 0)
-  C("google.protobuf.internal."
-    "_message.OneofDescriptor"),        // tp_name
+  FULL_MODULE_NAME ".OneofDescriptor",  // tp_name
   sizeof(PyBaseDescriptor),             // tp_basicsize
   0,                                    // tp_itemsize
   0,                                    // tp_dealloc
@@ -1373,7 +1378,7 @@ PyTypeObject PyOneofDescriptor_Type = {
   0,                                    // tp_setattro
   0,                                    // tp_as_buffer
   Py_TPFLAGS_DEFAULT,                   // tp_flags
-  C("A Oneof Descriptor"),              // tp_doc
+  "A Oneof Descriptor",                 // tp_doc
   0,                                    // tp_traverse
   0,                                    // tp_clear
   0,                                    // tp_richcompare
@@ -1386,10 +1391,10 @@ PyTypeObject PyOneofDescriptor_Type = {
   &descriptor::PyBaseDescriptor_Type,   // tp_base
 };
 
-PyObject* PyOneofDescriptor_New(
+PyObject* PyOneofDescriptor_FromDescriptor(
     const OneofDescriptor* oneof_descriptor) {
   return descriptor::NewInternedDescriptor(
-      &PyOneofDescriptor_Type, oneof_descriptor);
+      &PyOneofDescriptor_Type, oneof_descriptor, NULL);
 }
 
 // Add a enum values to a type dictionary.

python/google/protobuf/pyext/descriptor.h

@@ -48,21 +48,24 @@ extern PyTypeObject PyEnumValueDescriptor_Type;
 extern PyTypeObject PyFileDescriptor_Type;
 extern PyTypeObject PyOneofDescriptor_Type;
 
-// Return a new reference to a Descriptor object.
+// Wraps a Descriptor in a Python object.
 // The C++ pointer is usually borrowed from the global DescriptorPool.
 // In any case, it must stay alive as long as the Python object.
-PyObject* PyMessageDescriptor_New(const Descriptor* descriptor);
-PyObject* PyFieldDescriptor_New(const FieldDescriptor* descriptor);
-PyObject* PyEnumDescriptor_New(const EnumDescriptor* descriptor);
-PyObject* PyEnumValueDescriptor_New(const EnumValueDescriptor* descriptor);
-PyObject* PyOneofDescriptor_New(const OneofDescriptor* descriptor);
-PyObject* PyFileDescriptor_New(const FileDescriptor* file_descriptor);
+// Returns a new reference.
+PyObject* PyMessageDescriptor_FromDescriptor(const Descriptor* descriptor);
+PyObject* PyFieldDescriptor_FromDescriptor(const FieldDescriptor* descriptor);
+PyObject* PyEnumDescriptor_FromDescriptor(const EnumDescriptor* descriptor);
+PyObject* PyEnumValueDescriptor_FromDescriptor(
+    const EnumValueDescriptor* descriptor);
+PyObject* PyOneofDescriptor_FromDescriptor(const OneofDescriptor* descriptor);
+PyObject* PyFileDescriptor_FromDescriptor(
+    const FileDescriptor* file_descriptor);
 
 // Alternate constructor of PyFileDescriptor, used when we already have a
 // serialized FileDescriptorProto that can be cached.
 // Returns a new reference.
-PyObject* PyFileDescriptor_NewWithPb(const FileDescriptor* file_descriptor,
-                                     PyObject* serialized_pb);
+PyObject* PyFileDescriptor_FromDescriptorWithSerializedPb(
+    const FileDescriptor* file_descriptor, PyObject* serialized_pb);
 
 // Return the C++ descriptor pointer.
 // This function checks the parameter type; on error, return NULL with a Python

python/google/protobuf/pyext/descriptor_containers.cc

@@ -898,7 +898,7 @@ static ItemDescriptor GetByIndex(PyContainer* self, int index) {
 }
 
 static PyObject* NewObjectFromItem(ItemDescriptor item) {
-  return PyFieldDescriptor_New(item);
+  return PyFieldDescriptor_FromDescriptor(item);
 }
 
 static const string& GetItemName(ItemDescriptor item) {
@@ -956,7 +956,7 @@ static ItemDescriptor GetByIndex(PyContainer* self, int index) {
 }
 
 static PyObject* NewObjectFromItem(ItemDescriptor item) {
-  return PyMessageDescriptor_New(item);
+  return PyMessageDescriptor_FromDescriptor(item);
 }
 
 static const string& GetItemName(ItemDescriptor item) {
@@ -1006,7 +1006,7 @@ static ItemDescriptor GetByIndex(PyContainer* self, int index) {
 }
 
 static PyObject* NewObjectFromItem(ItemDescriptor item) {
-  return PyEnumDescriptor_New(item);
+  return PyEnumDescriptor_FromDescriptor(item);
 }
 
 static const string& GetItemName(ItemDescriptor item) {
@@ -1082,7 +1082,7 @@ static ItemDescriptor GetByIndex(PyContainer* self, int index) {
 }
 
 static PyObject* NewObjectFromItem(ItemDescriptor item) {
-  return PyEnumValueDescriptor_New(item);
+  return PyEnumValueDescriptor_FromDescriptor(item);
 }
 
 static const string& GetItemName(ItemDescriptor item) {
@@ -1124,7 +1124,7 @@ static ItemDescriptor GetByIndex(PyContainer* self, int index) {
 }
 
 static PyObject* NewObjectFromItem(ItemDescriptor item) {
-  return PyFieldDescriptor_New(item);
+  return PyFieldDescriptor_FromDescriptor(item);
 }
 
 static const string& GetItemName(ItemDescriptor item) {
@@ -1174,7 +1174,7 @@ static ItemDescriptor GetByIndex(PyContainer* self, int index) {
 }
 
 static PyObject* NewObjectFromItem(ItemDescriptor item) {
-  return PyOneofDescriptor_New(item);
+  return PyOneofDescriptor_FromDescriptor(item);
 }
 
 static const string& GetItemName(ItemDescriptor item) {
@@ -1238,7 +1238,7 @@ static ItemDescriptor GetByNumber(PyContainer* self, int number) {
 }
 
 static PyObject* NewObjectFromItem(ItemDescriptor item) {
-  return PyEnumValueDescriptor_New(item);
+  return PyEnumValueDescriptor_FromDescriptor(item);
 }
 
 static const string& GetItemName(ItemDescriptor item) {
@@ -1302,7 +1302,7 @@ static ItemDescriptor GetByIndex(PyContainer* self, int index) {
 }
 
 static PyObject* NewObjectFromItem(ItemDescriptor item) {
-  return PyFieldDescriptor_New(item);
+  return PyFieldDescriptor_FromDescriptor(item);
 }
 
 static int GetItemIndex(ItemDescriptor item) {
@@ -1354,7 +1354,7 @@ static ItemDescriptor GetByIndex(PyContainer* self, int index) {
 }
 
 static PyObject* NewObjectFromItem(ItemDescriptor item) {
-  return PyMessageDescriptor_New(item);
+  return PyMessageDescriptor_FromDescriptor(item);
 }
 
 static const string& GetItemName(ItemDescriptor item) {
@@ -1400,7 +1400,7 @@ static ItemDescriptor GetByIndex(PyContainer* self, int index) {
 }
 
 static PyObject* NewObjectFromItem(ItemDescriptor item) {
-  return PyEnumDescriptor_New(item);
+  return PyEnumDescriptor_FromDescriptor(item);
 }
 
 static const string& GetItemName(ItemDescriptor item) {
@@ -1446,7 +1446,7 @@ static ItemDescriptor GetByIndex(PyContainer* self, int index) {
 }
 
 static PyObject* NewObjectFromItem(ItemDescriptor item) {
-  return PyFieldDescriptor_New(item);
+  return PyFieldDescriptor_FromDescriptor(item);
 }
 
 static const string& GetItemName(ItemDescriptor item) {
@@ -1488,7 +1488,7 @@ static ItemDescriptor GetByIndex(PyContainer* self, int index) {
 }
 
 static PyObject* NewObjectFromItem(ItemDescriptor item) {
-  return PyFileDescriptor_New(item);
+  return PyFileDescriptor_FromDescriptor(item);
 }
 
 static DescriptorContainerDef ContainerDef = {
@@ -1522,7 +1522,7 @@ static ItemDescriptor GetByIndex(PyContainer* self, int index) {
 }
 
 static PyObject* NewObjectFromItem(ItemDescriptor item) {
-  return PyFileDescriptor_New(item);
+  return PyFileDescriptor_FromDescriptor(item);
 }
 
 static DescriptorContainerDef ContainerDef = {

python/google/protobuf/pyext/descriptor_containers.h

@@ -28,6 +28,9 @@
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_CONTAINERS_H__
+#define GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_CONTAINERS_H__
+
 // Mappings and Sequences of descriptors.
 // They implement containers like fields_by_name, EnumDescriptor.values...
 // See descriptor_containers.cc for more description.
@@ -92,4 +95,6 @@ PyObject* NewFilePublicDependencies(const FileDescriptor* descriptor);
 
 }  // namespace python
 }  // namespace protobuf
+
 }  // namespace google
+#endif  // GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_CONTAINERS_H__

python/google/protobuf/pyext/descriptor_pool.cc

@@ -35,10 +35,9 @@
 #include <google/protobuf/descriptor.pb.h>
 #include <google/protobuf/pyext/descriptor_pool.h>
 #include <google/protobuf/pyext/descriptor.h>
+#include <google/protobuf/pyext/message.h>
 #include <google/protobuf/pyext/scoped_pyobject_ptr.h>
 
-#define C(str) const_cast<char*>(str)
-
 #if PY_MAJOR_VERSION >= 3
   #define PyString_FromStringAndSize PyUnicode_FromStringAndSize
   #if PY_VERSION_HEX < 0x03030000
@@ -108,11 +107,11 @@ PyObject* FindMessageByName(PyDescriptorPool* self, PyObject* arg) {
       self->pool->FindMessageTypeByName(string(name, name_size));
 
   if (message_descriptor == NULL) {
-    PyErr_Format(PyExc_TypeError, "Couldn't find message %.200s", name);
+    PyErr_Format(PyExc_KeyError, "Couldn't find message %.200s", name);
     return NULL;
   }
 
-  return PyMessageDescriptor_New(message_descriptor);
+  return PyMessageDescriptor_FromDescriptor(message_descriptor);
 }
 
 // Add a message class to our database.
@@ -158,6 +157,24 @@ PyObject *GetMessageClass(PyDescriptorPool* self,
   }
 }
 
+PyObject* FindFileByName(PyDescriptorPool* self, PyObject* arg) {
+  Py_ssize_t name_size;
+  char* name;
+  if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) {
+    return NULL;
+  }
+
+  const FileDescriptor* file_descriptor =
+      self->pool->FindFileByName(string(name, name_size));
+  if (file_descriptor == NULL) {
+    PyErr_Format(PyExc_KeyError, "Couldn't find file %.200s",
+                 name);
+    return NULL;
+  }
+
+  return PyFileDescriptor_FromDescriptor(file_descriptor);
+}
+
 PyObject* FindFieldByName(PyDescriptorPool* self, PyObject* arg) {
   Py_ssize_t name_size;
   char* name;
@@ -168,12 +185,12 @@ PyObject* FindFieldByName(PyDescriptorPool* self, PyObject* arg) {
   const FieldDescriptor* field_descriptor =
       self->pool->FindFieldByName(string(name, name_size));
   if (field_descriptor == NULL) {
-    PyErr_Format(PyExc_TypeError, "Couldn't find field %.200s",
+    PyErr_Format(PyExc_KeyError, "Couldn't find field %.200s",
                  name);
     return NULL;
   }
 
-  return PyFieldDescriptor_New(field_descriptor);
+  return PyFieldDescriptor_FromDescriptor(field_descriptor);
 }
 
 PyObject* FindExtensionByName(PyDescriptorPool* self, PyObject* arg) {
@@ -186,11 +203,11 @@ PyObject* FindExtensionByName(PyDescriptorPool* self, PyObject* arg) {
   const FieldDescriptor* field_descriptor =
       self->pool->FindExtensionByName(string(name, name_size));
   if (field_descriptor == NULL) {
-    PyErr_Format(PyExc_TypeError, "Couldn't find field %.200s", name);
+    PyErr_Format(PyExc_KeyError, "Couldn't find extension field %.200s", name);
     return NULL;
   }
 
-  return PyFieldDescriptor_New(field_descriptor);
+  return PyFieldDescriptor_FromDescriptor(field_descriptor);
 }
 
 PyObject* FindEnumTypeByName(PyDescriptorPool* self, PyObject* arg) {
@@ -203,11 +220,11 @@ PyObject* FindEnumTypeByName(PyDescriptorPool* self, PyObject* arg) {
   const EnumDescriptor* enum_descriptor =
       self->pool->FindEnumTypeByName(string(name, name_size));
   if (enum_descriptor == NULL) {
-    PyErr_Format(PyExc_TypeError, "Couldn't find enum %.200s", name);
+    PyErr_Format(PyExc_KeyError, "Couldn't find enum %.200s", name);
     return NULL;
   }
 
-  return PyEnumDescriptor_New(enum_descriptor);
+  return PyEnumDescriptor_FromDescriptor(enum_descriptor);
 }
 
 PyObject* FindOneofByName(PyDescriptorPool* self, PyObject* arg) {
@@ -220,70 +237,13 @@ PyObject* FindOneofByName(PyDescriptorPool* self, PyObject* arg) {
   const OneofDescriptor* oneof_descriptor =
       self->pool->FindOneofByName(string(name, name_size));
   if (oneof_descriptor == NULL) {
-    PyErr_Format(PyExc_TypeError, "Couldn't find oneof %.200s", name);
+    PyErr_Format(PyExc_KeyError, "Couldn't find oneof %.200s", name);
     return NULL;
   }
 
-  return PyOneofDescriptor_New(oneof_descriptor);
+  return PyOneofDescriptor_FromDescriptor(oneof_descriptor);
 }
 
-static PyMethodDef Methods[] = {
-  { C("FindFieldByName"),
-    (PyCFunction)FindFieldByName,
-    METH_O,
-    C("Searches for a field descriptor by full name.") },
-  { C("FindExtensionByName"),
-    (PyCFunction)FindExtensionByName,
-    METH_O,
-    C("Searches for extension descriptor by full name.") },
-  {NULL}
-};
-
-}  // namespace cdescriptor_pool
-
-PyTypeObject PyDescriptorPool_Type = {
-  PyVarObject_HEAD_INIT(&PyType_Type, 0)
-  C("google.protobuf.internal."
-    "_message.DescriptorPool"),        // tp_name
-  sizeof(PyDescriptorPool),            // tp_basicsize
-  0,                                   // tp_itemsize
-  (destructor)cdescriptor_pool::Dealloc,  // tp_dealloc
-  0,                                   // tp_print
-  0,                                   // tp_getattr
-  0,                                   // tp_setattr
-  0,                                   // tp_compare
-  0,                                   // tp_repr
-  0,                                   // tp_as_number
-  0,                                   // tp_as_sequence
-  0,                                   // tp_as_mapping
-  0,                                   // tp_hash
-  0,                                   // tp_call
-  0,                                   // tp_str
-  0,                                   // tp_getattro
-  0,                                   // tp_setattro
-  0,                                   // tp_as_buffer
-  Py_TPFLAGS_DEFAULT,                  // tp_flags
-  C("A Descriptor Pool"),              // tp_doc
-  0,                                   // tp_traverse
-  0,                                   // tp_clear
-  0,                                   // tp_richcompare
-  0,                                   // tp_weaklistoffset
-  0,                                   // tp_iter
-  0,                                   // tp_iternext
-  cdescriptor_pool::Methods,           // tp_methods
-  0,                                   // tp_members
-  0,                                   // tp_getset
-  0,                                   // tp_base
-  0,                                   // tp_dict
-  0,                                   // tp_descr_get
-  0,                                   // tp_descr_set
-  0,                                   // tp_dictoffset
-  0,                                   // tp_init
-  0,                                   // tp_alloc
-  0,                                   // tp_new
-  PyObject_Del,                        // tp_free
-};
-
 // The code below loads new Descriptors from a serialized FileDescriptorProto.
 
 
@@ -301,6 +261,7 @@ class BuildFileErrorCollector : public DescriptorPool::ErrorCollector {
     if (!had_errors) {
       error_message +=
           ("Invalid proto descriptor for file \"" + filename + "\":\n");
+      had_errors = true;
     }
     // As this only happens on failure and will result in the program not
     // running at all, no effort is made to optimize this string manipulation.
@@ -311,7 +272,7 @@ class BuildFileErrorCollector : public DescriptorPool::ErrorCollector {
   bool had_errors;
 };
 
-PyObject* Python_BuildFile(PyObject* ignored, PyObject* serialized_pb) {
+PyObject* AddSerializedFile(PyDescriptorPool* self, PyObject* serialized_pb) {
   char* message_type;
   Py_ssize_t message_len;
 
@@ -330,13 +291,14 @@ PyObject* Python_BuildFile(PyObject* ignored, PyObject* serialized_pb) {
   const FileDescriptor* generated_file =
       DescriptorPool::generated_pool()->FindFileByName(file_proto.name());
   if (generated_file != NULL) {
-    return PyFileDescriptor_NewWithPb(generated_file, serialized_pb);
+    return PyFileDescriptor_FromDescriptorWithSerializedPb(
+        generated_file, serialized_pb);
   }
 
   BuildFileErrorCollector error_collector;
   const FileDescriptor* descriptor =
-      GetDescriptorPool()->pool->BuildFileCollectingErrors(file_proto,
-                                                           &error_collector);
+      self->pool->BuildFileCollectingErrors(file_proto,
+                                            &error_collector);
   if (descriptor == NULL) {
     PyErr_Format(PyExc_TypeError,
                  "Couldn't build proto file into descriptor pool!\n%s",
@@ -344,9 +306,84 @@ PyObject* Python_BuildFile(PyObject* ignored, PyObject* serialized_pb) {
     return NULL;
   }
 
-  return PyFileDescriptor_NewWithPb(descriptor, serialized_pb);
+  return PyFileDescriptor_FromDescriptorWithSerializedPb(
+      descriptor, serialized_pb);
+}
+
+PyObject* Add(PyDescriptorPool* self, PyObject* file_descriptor_proto) {
+  ScopedPyObjectPtr serialized_pb(
+      PyObject_CallMethod(file_descriptor_proto, "SerializeToString", NULL));
+  if (serialized_pb == NULL) {
+    return NULL;
+  }
+  return AddSerializedFile(self, serialized_pb);
 }
 
+static PyMethodDef Methods[] = {
+  { "Add", (PyCFunction)Add, METH_O,
+    "Adds the FileDescriptorProto and its types to this pool." },
+  { "AddSerializedFile", (PyCFunction)AddSerializedFile, METH_O,
+    "Adds a serialized FileDescriptorProto to this pool." },
+
+  { "FindFileByName", (PyCFunction)FindFileByName, METH_O,
+    "Searches for a file descriptor by its .proto name." },
+  { "FindMessageTypeByName", (PyCFunction)FindMessageByName, METH_O,
+    "Searches for a message descriptor by full name." },
+  { "FindFieldByName", (PyCFunction)FindFieldByName, METH_O,
+    "Searches for a field descriptor by full name." },
+  { "FindExtensionByName", (PyCFunction)FindExtensionByName, METH_O,
+    "Searches for extension descriptor by full name." },
+  { "FindEnumTypeByName", (PyCFunction)FindEnumTypeByName, METH_O,
+    "Searches for enum type descriptor by full name." },
+  { "FindOneofByName", (PyCFunction)FindOneofByName, METH_O,
+    "Searches for oneof descriptor by full name." },
+  {NULL}
+};
+
+}  // namespace cdescriptor_pool
+
+PyTypeObject PyDescriptorPool_Type = {
+  PyVarObject_HEAD_INIT(&PyType_Type, 0)
+  FULL_MODULE_NAME ".DescriptorPool",  // tp_name
+  sizeof(PyDescriptorPool),            // tp_basicsize
+  0,                                   // tp_itemsize
+  (destructor)cdescriptor_pool::Dealloc,  // tp_dealloc
+  0,                                   // tp_print
+  0,                                   // tp_getattr
+  0,                                   // tp_setattr
+  0,                                   // tp_compare
+  0,                                   // tp_repr
+  0,                                   // tp_as_number
+  0,                                   // tp_as_sequence
+  0,                                   // tp_as_mapping
+  0,                                   // tp_hash
+  0,                                   // tp_call
+  0,                                   // tp_str
+  0,                                   // tp_getattro
+  0,                                   // tp_setattro
+  0,                                   // tp_as_buffer
+  Py_TPFLAGS_DEFAULT,                  // tp_flags
+  "A Descriptor Pool",                 // tp_doc
+  0,                                   // tp_traverse
+  0,                                   // tp_clear
+  0,                                   // tp_richcompare
+  0,                                   // tp_weaklistoffset
+  0,                                   // tp_iter
+  0,                                   // tp_iternext
+  cdescriptor_pool::Methods,           // tp_methods
+  0,                                   // tp_members
+  0,                                   // tp_getset
+  0,                                   // tp_base
+  0,                                   // tp_dict
+  0,                                   // tp_descr_get
+  0,                                   // tp_descr_set
+  0,                                   // tp_dictoffset
+  0,                                   // tp_init
+  0,                                   // tp_alloc
+  0,                                   // tp_new
+  PyObject_Del,                        // tp_free
+};
+
 static PyDescriptorPool* global_cdescriptor_pool = NULL;
 
 bool InitDescriptorPool() {

python/google/protobuf/pyext/descriptor_pool.h

@@ -95,6 +95,8 @@ const Descriptor* FindMessageTypeByName(PyDescriptorPool* self,
 const Descriptor* RegisterMessageClass(
     PyDescriptorPool* self, PyObject* message_class, PyObject* descriptor);
 
+// The function below are also exposed as methods of the DescriptorPool type.
+
 // Retrieves the Python class registered with the given message descriptor.
 //
 // Returns a *borrowed* reference if found, otherwise returns NULL with an
@@ -134,12 +136,8 @@ PyObject* FindOneofByName(PyDescriptorPool* self, PyObject* arg);
 
 }  // namespace cdescriptor_pool
 
-// Implement the Python "_BuildFile" method, it takes a serialized
-// FileDescriptorProto, and adds it to the C++ DescriptorPool.
-// It returns a new FileDescriptor object, or NULL when an exception is raised.
-PyObject* Python_BuildFile(PyObject* ignored, PyObject* args);
-
 // Retrieve the global descriptor pool owned by the _message module.
+// Returns a *borrowed* reference.
 PyDescriptorPool* GetDescriptorPool();
 
 // Initialize objects used by this module.

python/google/protobuf/pyext/extension_dict.cc

@@ -51,16 +51,6 @@ namespace python {
 
 namespace extension_dict {
 
-// TODO(tibell): Always use self->message for clarity, just like in
-// RepeatedCompositeContainer.
-static Message* GetMessage(ExtensionDict* self) {
-  if (self->parent != NULL) {
-    return self->parent->message;
-  } else {
-    return self->message;
-  }
-}
-
 PyObject* len(ExtensionDict* self) {
 #if PY_MAJOR_VERSION >= 3
   return PyLong_FromLong(PyDict_Size(self->values));
@@ -89,7 +79,7 @@ int ReleaseExtension(ExtensionDict* self,
     }
   } else if (descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
     if (cmessage::ReleaseSubMessage(
-            GetMessage(self), descriptor,
+            self->parent, descriptor,
             reinterpret_cast<CMessage*>(extension)) < 0) {
       return -1;
     }
@@ -109,7 +99,7 @@ PyObject* subscript(ExtensionDict* self, PyObject* key) {
 
   if (descriptor->label() != FieldDescriptor::LABEL_REPEATED &&
       descriptor->cpp_type() != FieldDescriptor::CPPTYPE_MESSAGE) {
-    return cmessage::InternalGetScalar(self->parent, descriptor);
+    return cmessage::InternalGetScalar(self->parent->message, descriptor);
   }
 
   PyObject* value = PyDict_GetItem(self->values, key);
@@ -266,8 +256,7 @@ static PyMethodDef Methods[] = {
 
 PyTypeObject ExtensionDict_Type = {
   PyVarObject_HEAD_INIT(&PyType_Type, 0)
-  "google.protobuf.internal."
-  "cpp._message.ExtensionDict",        // tp_name
+  FULL_MODULE_NAME ".ExtensionDict",   // tp_name
   sizeof(ExtensionDict),               // tp_basicsize
   0,                                   //  tp_itemsize
   (destructor)extension_dict::dealloc,  //  tp_dealloc

python/google/protobuf/pyext/message.cc

@@ -59,6 +59,8 @@
 #include <google/protobuf/pyext/extension_dict.h>
 #include <google/protobuf/pyext/repeated_composite_container.h>
 #include <google/protobuf/pyext/repeated_scalar_container.h>
+#include <google/protobuf/pyext/message_map_container.h>
+#include <google/protobuf/pyext/scalar_map_container.h>
 #include <google/protobuf/pyext/scoped_pyobject_ptr.h>
 #include <google/protobuf/stubs/strutil.h>
 
@@ -93,9 +95,9 @@ static const FieldDescriptor* GetFieldDescriptor(
 static const Descriptor* GetMessageDescriptor(PyTypeObject* cls);
 static string GetMessageName(CMessage* self);
 int InternalReleaseFieldByDescriptor(
+    CMessage* self,
     const FieldDescriptor* field_descriptor,
-    PyObject* composite_field,
-    Message* parent_message);
+    PyObject* composite_field);
 }  // namespace cmessage
 
 // ---------------------------------------------------------------------
@@ -127,10 +129,29 @@ static int VisitCompositeField(const FieldDescriptor* descriptor,
                                Visitor visitor) {
   if (descriptor->label() == FieldDescriptor::LABEL_REPEATED) {
     if (descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
-      RepeatedCompositeContainer* container =
-        reinterpret_cast<RepeatedCompositeContainer*>(child);
-      if (visitor.VisitRepeatedCompositeContainer(container) == -1)
-        return -1;
+      if (descriptor->is_map()) {
+        const Descriptor* entry_type = descriptor->message_type();
+        const FieldDescriptor* value_type =
+            entry_type->FindFieldByName("value");
+        if (value_type->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
+          MessageMapContainer* container =
+            reinterpret_cast<MessageMapContainer*>(child);
+          if (visitor.VisitMessageMapContainer(container) == -1) {
+            return -1;
+          }
+        } else {
+          ScalarMapContainer* container =
+            reinterpret_cast<ScalarMapContainer*>(child);
+          if (visitor.VisitScalarMapContainer(container) == -1) {
+            return -1;
+          }
+        }
+      } else {
+        RepeatedCompositeContainer* container =
+          reinterpret_cast<RepeatedCompositeContainer*>(child);
+        if (visitor.VisitRepeatedCompositeContainer(container) == -1)
+          return -1;
+      }
     } else {
       RepeatedScalarContainer* container =
         reinterpret_cast<RepeatedScalarContainer*>(child);
@@ -444,7 +465,7 @@ static int MaybeReleaseOverlappingOneofField(
   }
 
   if (InternalReleaseFieldByDescriptor(
-          existing_field, child_message, message) < 0) {
+          cmessage, existing_field, child_message) < 0) {
     return -1;
   }
   return PyDict_DelItemString(cmessage->composite_fields, field_name);
@@ -483,6 +504,16 @@ struct FixupMessageReference : public ChildVisitor {
     return 0;
   }
 
+  int VisitScalarMapContainer(ScalarMapContainer* container) {
+    container->message = message_;
+    return 0;
+  }
+
+  int VisitMessageMapContainer(MessageMapContainer* container) {
+    container->message = message_;
+    return 0;
+  }
+
  private:
   Message* message_;
 };
@@ -500,6 +531,9 @@ int AssureWritable(CMessage* self) {
         self->message->GetDescriptor());
     self->message = prototype->New();
     self->owner.reset(self->message);
+    // Cascade the new owner to eventual children: even if this message is
+    // empty, some submessages or repeated containers might exist already.
+    SetOwner(self, self->owner);
   } else {
     // Otherwise, we need a mutable child message.
     if (AssureWritable(self->parent) == -1)
@@ -520,8 +554,9 @@ int AssureWritable(CMessage* self) {
   // When a CMessage is made writable its Message pointer is updated
   // to point to a new mutable Message.  When that happens we need to
   // update any references to the old, read-only CMessage.  There are
-  // three places such references occur: RepeatedScalarContainer,
-  // RepeatedCompositeContainer, and ExtensionDict.
+  // five places such references occur: RepeatedScalarContainer,
+  // RepeatedCompositeContainer, ScalarMapContainer, MessageMapContainer,
+  // and ExtensionDict.
   if (self->extensions != NULL)
     self->extensions->message = self->message;
   if (ForEachCompositeField(self, FixupMessageReference(self->message)) == -1)
@@ -583,15 +618,43 @@ const FieldDescriptor* GetExtensionDescriptor(PyObject* extension) {
   return PyFieldDescriptor_AsDescriptor(extension);
 }
 
+// If value is a string, convert it into an enum value based on the labels in
+// descriptor, otherwise simply return value.  Always returns a new reference.
+static PyObject* GetIntegerEnumValue(const FieldDescriptor& descriptor,
+                                     PyObject* value) {
+  if (PyString_Check(value) || PyUnicode_Check(value)) {
+    const EnumDescriptor* enum_descriptor = descriptor.enum_type();
+    if (enum_descriptor == NULL) {
+      PyErr_SetString(PyExc_TypeError, "not an enum field");
+      return NULL;
+    }
+    char* enum_label;
+    Py_ssize_t size;
+    if (PyString_AsStringAndSize(value, &enum_label, &size) < 0) {
+      return NULL;
+    }
+    const EnumValueDescriptor* enum_value_descriptor =
+        enum_descriptor->FindValueByName(string(enum_label, size));
+    if (enum_value_descriptor == NULL) {
+      PyErr_SetString(PyExc_ValueError, "unknown enum label");
+      return NULL;
+    }
+    return PyInt_FromLong(enum_value_descriptor->number());
+  }
+  Py_INCREF(value);
+  return value;
+}
+
 // If cmessage_list is not NULL, this function releases values into the
 // container CMessages instead of just removing. Repeated composite container
 // needs to do this to make sure CMessages stay alive if they're still
 // referenced after deletion. Repeated scalar container doesn't need to worry.
 int InternalDeleteRepeatedField(
-    Message* message,
+    CMessage* self,
     const FieldDescriptor* field_descriptor,
     PyObject* slice,
     PyObject* cmessage_list) {
+  Message* message = self->message;
   Py_ssize_t length, from, to, step, slice_length;
   const Reflection* reflection = message->GetReflection();
   int min, max;
@@ -665,7 +728,7 @@ int InternalDeleteRepeatedField(
       CMessage* last_cmessage = reinterpret_cast<CMessage*>(
           PyList_GET_ITEM(cmessage_list, PyList_GET_SIZE(cmessage_list) - 1));
       repeated_composite_container::ReleaseLastTo(
-          field_descriptor, message, last_cmessage);
+          self, field_descriptor, last_cmessage);
       if (PySequence_DelItem(cmessage_list, -1) < 0) {
         return -1;
       }
@@ -696,16 +759,90 @@ int InitAttributes(CMessage* self, PyObject* kwargs) {
                    PyString_AsString(name));
       return -1;
     }
-    if (descriptor->label() == FieldDescriptor::LABEL_REPEATED) {
+    if (descriptor->is_map()) {
+      ScopedPyObjectPtr map(GetAttr(self, name));
+      const FieldDescriptor* value_descriptor =
+          descriptor->message_type()->FindFieldByName("value");
+      if (value_descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
+        Py_ssize_t map_pos = 0;
+        PyObject* map_key;
+        PyObject* map_value;
+        while (PyDict_Next(value, &map_pos, &map_key, &map_value)) {
+          ScopedPyObjectPtr function_return;
+          function_return.reset(PyObject_GetItem(map.get(), map_key));
+          if (function_return.get() == NULL) {
+            return -1;
+          }
+          ScopedPyObjectPtr ok(PyObject_CallMethod(
+              function_return.get(), "MergeFrom", "O", map_value));
+          if (ok.get() == NULL) {
+            return -1;
+          }
+        }
+      } else {
+        ScopedPyObjectPtr function_return;
+        function_return.reset(
+            PyObject_CallMethod(map.get(), "update", "O", value));
+        if (function_return.get() == NULL) {
+          return -1;
+        }
+      }
+    } else if (descriptor->label() == FieldDescriptor::LABEL_REPEATED) {
       ScopedPyObjectPtr container(GetAttr(self, name));
       if (container == NULL) {
         return -1;
       }
       if (descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
-        if (repeated_composite_container::Extend(
-                reinterpret_cast<RepeatedCompositeContainer*>(container.get()),
-                value)
-            == NULL) {
+        RepeatedCompositeContainer* rc_container =
+            reinterpret_cast<RepeatedCompositeContainer*>(container.get());
+        ScopedPyObjectPtr iter(PyObject_GetIter(value));
+        if (iter == NULL) {
+          PyErr_SetString(PyExc_TypeError, "Value must be iterable");
+          return -1;
+        }
+        ScopedPyObjectPtr next;
+        while ((next.reset(PyIter_Next(iter))) != NULL) {
+          PyObject* kwargs = (PyDict_Check(next) ? next.get() : NULL);
+          ScopedPyObjectPtr new_msg(
+              repeated_composite_container::Add(rc_container, NULL, kwargs));
+          if (new_msg == NULL) {
+            return -1;
+          }
+          if (kwargs == NULL) {
+            // next was not a dict, it's a message we need to merge
+            ScopedPyObjectPtr merged(
+                MergeFrom(reinterpret_cast<CMessage*>(new_msg.get()), next));
+            if (merged == NULL) {
+              return -1;
+            }
+          }
+        }
+        if (PyErr_Occurred()) {
+          // Check to see how PyIter_Next() exited.
+          return -1;
+        }
+      } else if (descriptor->cpp_type() == FieldDescriptor::CPPTYPE_ENUM) {
+        RepeatedScalarContainer* rs_container =
+            reinterpret_cast<RepeatedScalarContainer*>(container.get());
+        ScopedPyObjectPtr iter(PyObject_GetIter(value));
+        if (iter == NULL) {
+          PyErr_SetString(PyExc_TypeError, "Value must be iterable");
+          return -1;
+        }
+        ScopedPyObjectPtr next;
+        while ((next.reset(PyIter_Next(iter))) != NULL) {
+          ScopedPyObjectPtr enum_value(GetIntegerEnumValue(*descriptor, next));
+          if (enum_value == NULL) {
+            return -1;
+          }
+          ScopedPyObjectPtr new_msg(
+              repeated_scalar_container::Append(rs_container, enum_value));
+          if (new_msg == NULL) {
+            return -1;
+          }
+        }
+        if (PyErr_Occurred()) {
+          // Check to see how PyIter_Next() exited.
           return -1;
         }
       } else {
@@ -721,12 +858,26 @@ int InitAttributes(CMessage* self, PyObject* kwargs) {
       if (message == NULL) {
         return -1;
       }
-      if (MergeFrom(reinterpret_cast<CMessage*>(message.get()),
-                             value) == NULL) {
-        return -1;
+      CMessage* cmessage = reinterpret_cast<CMessage*>(message.get());
+      if (PyDict_Check(value)) {
+        if (InitAttributes(cmessage, value) < 0) {
+          return -1;
+        }
+      } else {
+        ScopedPyObjectPtr merged(MergeFrom(cmessage, value));
+        if (merged == NULL) {
+          return -1;
+        }
       }
     } else {
-      if (SetAttr(self, name, value) < 0) {
+      ScopedPyObjectPtr new_val;
+      if (descriptor->cpp_type() == FieldDescriptor::CPPTYPE_ENUM) {
+        new_val.reset(GetIntegerEnumValue(*descriptor, value));
+        if (new_val == NULL) {
+          return -1;
+        }
+      }
+      if (SetAttr(self, name, (new_val == NULL) ? value : new_val) < 0) {
         return -1;
       }
     }
@@ -789,7 +940,6 @@ static PyObject* New(PyTypeObject* type,
   }
   self->message = default_message->New();
   self->owner.reset(self->message);
-
   return reinterpret_cast<PyObject*>(self);
 }
 
@@ -830,6 +980,16 @@ struct ClearWeakReferences : public ChildVisitor {
     return 0;
   }
 
+  int VisitScalarMapContainer(ScalarMapContainer* container) {
+    container->parent = NULL;
+    return 0;
+  }
+
+  int VisitMessageMapContainer(MessageMapContainer* container) {
+    container->parent = NULL;
+    return 0;
+  }
+
   int VisitCMessage(CMessage* cmessage,
                     const FieldDescriptor* field_descriptor) {
     cmessage->parent = NULL;
@@ -1064,6 +1224,16 @@ struct SetOwnerVisitor : public ChildVisitor {
     return 0;
   }
 
+  int VisitScalarMapContainer(ScalarMapContainer* container) {
+    scalar_map_container::SetOwner(container, new_owner_);
+    return 0;
+  }
+
+  int VisitMessageMapContainer(MessageMapContainer* container) {
+    message_map_container::SetOwner(container, new_owner_);
+    return 0;
+  }
+
   int VisitCMessage(CMessage* cmessage,
                     const FieldDescriptor* field_descriptor) {
     return SetOwner(cmessage, new_owner_);
@@ -1084,11 +1254,11 @@ int SetOwner(CMessage* self, const shared_ptr<Message>& new_owner) {
 // Releases the message specified by 'field' and returns the
 // pointer. If the field does not exist a new message is created using
 // 'descriptor'. The caller takes ownership of the returned pointer.
-Message* ReleaseMessage(Message* message,
+Message* ReleaseMessage(CMessage* self,
                         const Descriptor* descriptor,
                         const FieldDescriptor* field_descriptor) {
-  Message* released_message = message->GetReflection()->ReleaseMessage(
-      message, field_descriptor, message_factory);
+  Message* released_message = self->message->GetReflection()->ReleaseMessage(
+      self->message, field_descriptor, message_factory);
   // ReleaseMessage will return NULL which differs from
   // child_cmessage->message, if the field does not exist.  In this case,
   // the latter points to the default instance via a const_cast<>, so we
@@ -1102,12 +1272,12 @@ Message* ReleaseMessage(Message* message,
   return released_message;
 }
 
-int ReleaseSubMessage(Message* message,
+int ReleaseSubMessage(CMessage* self,
                       const FieldDescriptor* field_descriptor,
                       CMessage* child_cmessage) {
   // Release the Message
   shared_ptr<Message> released_message(ReleaseMessage(
-      message, child_cmessage->message->GetDescriptor(), field_descriptor));
+      self, child_cmessage->message->GetDescriptor(), field_descriptor));
   child_cmessage->message = released_message.get();
   child_cmessage->owner.swap(released_message);
   child_cmessage->parent = NULL;
@@ -1119,8 +1289,8 @@ int ReleaseSubMessage(Message* message,
 
 struct ReleaseChild : public ChildVisitor {
   // message must outlive this object.
-  explicit ReleaseChild(Message* parent_message) :
-      parent_message_(parent_message) {}
+  explicit ReleaseChild(CMessage* parent) :
+      parent_(parent) {}
 
   int VisitRepeatedCompositeContainer(RepeatedCompositeContainer* container) {
     return repeated_composite_container::Release(
@@ -1132,23 +1302,33 @@ struct ReleaseChild : public ChildVisitor {
         reinterpret_cast<RepeatedScalarContainer*>(container));
   }
 
+  int VisitScalarMapContainer(ScalarMapContainer* container) {
+    return scalar_map_container::Release(
+        reinterpret_cast<ScalarMapContainer*>(container));
+  }
+
+  int VisitMessageMapContainer(MessageMapContainer* container) {
+    return message_map_container::Release(
+        reinterpret_cast<MessageMapContainer*>(container));
+  }
+
   int VisitCMessage(CMessage* cmessage,
                     const FieldDescriptor* field_descriptor) {
-    return ReleaseSubMessage(parent_message_, field_descriptor,
+    return ReleaseSubMessage(parent_, field_descriptor,
         reinterpret_cast<CMessage*>(cmessage));
   }
 
-  Message* parent_message_;
+  CMessage* parent_;
 };
 
 int InternalReleaseFieldByDescriptor(
+    CMessage* self,
     const FieldDescriptor* field_descriptor,
-    PyObject* composite_field,
-    Message* parent_message) {
+    PyObject* composite_field) {
   return VisitCompositeField(
       field_descriptor,
       composite_field,
-      ReleaseChild(parent_message));
+      ReleaseChild(self));
 }
 
 PyObject* ClearFieldByDescriptor(
@@ -1200,8 +1380,8 @@ PyObject* ClearField(CMessage* self, PyObject* arg) {
   // Only release the field if there's a possibility that there are
   // references to it.
   if (composite_field != NULL) {
-    if (InternalReleaseFieldByDescriptor(field_descriptor,
-                                         composite_field, message) < 0) {
+    if (InternalReleaseFieldByDescriptor(self, field_descriptor,
+                                         composite_field) < 0) {
       return NULL;
     }
     PyDict_DelItem(self->composite_fields, arg);
@@ -1219,7 +1399,7 @@ PyObject* ClearField(CMessage* self, PyObject* arg) {
 
 PyObject* Clear(CMessage* self) {
   AssureWritable(self);
-  if (ForEachCompositeField(self, ReleaseChild(self->message)) == -1)
+  if (ForEachCompositeField(self, ReleaseChild(self)) == -1)
     return NULL;
 
   // The old ExtensionDict still aliases this CMessage, but all its
@@ -1582,7 +1762,8 @@ static PyObject* ListFields(CMessage* self) {
     }
 
     if (fields[i]->is_extension()) {
-      ScopedPyObjectPtr extension_field(PyFieldDescriptor_New(fields[i]));
+      ScopedPyObjectPtr extension_field(
+          PyFieldDescriptor_FromDescriptor(fields[i]));
       if (extension_field == NULL) {
         return NULL;
       }
@@ -1616,7 +1797,8 @@ static PyObject* ListFields(CMessage* self) {
         PyErr_SetString(PyExc_ValueError, "bad string");
         return NULL;
       }
-      ScopedPyObjectPtr field_descriptor(PyFieldDescriptor_New(fields[i]));
+      ScopedPyObjectPtr field_descriptor(
+          PyFieldDescriptor_FromDescriptor(fields[i]));
       if (field_descriptor == NULL) {
         return NULL;
       }
@@ -1683,10 +1865,8 @@ static PyObject* RichCompare(CMessage* self, PyObject* other, int opid) {
   }
 }
 
-PyObject* InternalGetScalar(
-    CMessage* self,
-    const FieldDescriptor* field_descriptor) {
-  Message* message = self->message;
+PyObject* InternalGetScalar(const Message* message,
+                            const FieldDescriptor* field_descriptor) {
   const Reflection* reflection = message->GetReflection();
 
   if (!CheckFieldBelongsToMessage(field_descriptor, message)) {
@@ -1739,12 +1919,12 @@ PyObject* InternalGetScalar(
       if (!message->GetReflection()->SupportsUnknownEnumValues() &&
           !message->GetReflection()->HasField(*message, field_descriptor)) {
         // Look for the value in the unknown fields.
-        UnknownFieldSet* unknown_field_set =
-            message->GetReflection()->MutableUnknownFields(message);
-        for (int i = 0; i < unknown_field_set->field_count(); ++i) {
-          if (unknown_field_set->field(i).number() ==
+        const UnknownFieldSet& unknown_field_set =
+            message->GetReflection()->GetUnknownFields(*message);
+        for (int i = 0; i < unknown_field_set.field_count(); ++i) {
+          if (unknown_field_set.field(i).number() ==
               field_descriptor->number()) {
-            result = PyInt_FromLong(unknown_field_set->field(i).varint());
+            result = PyInt_FromLong(unknown_field_set.field(i).varint());
             break;
           }
         }
@@ -1793,21 +1973,16 @@ PyObject* InternalGetSubMessage(
   return reinterpret_cast<PyObject*>(cmsg);
 }
 
-int InternalSetScalar(
-    CMessage* self,
+int InternalSetNonOneofScalar(
+    Message* message,
     const FieldDescriptor* field_descriptor,
     PyObject* arg) {
-  Message* message = self->message;
   const Reflection* reflection = message->GetReflection();
 
   if (!CheckFieldBelongsToMessage(field_descriptor, message)) {
     return -1;
   }
 
-  if (MaybeReleaseOverlappingOneofField(self, field_descriptor) < 0) {
-    return -1;
-  }
-
   switch (field_descriptor->cpp_type()) {
     case FieldDescriptor::CPPTYPE_INT32: {
       GOOGLE_CHECK_GET_INT32(arg, value, -1);
@@ -1878,6 +2053,21 @@ int InternalSetScalar(
   return 0;
 }
 
+int InternalSetScalar(
+    CMessage* self,
+    const FieldDescriptor* field_descriptor,
+    PyObject* arg) {
+  if (!CheckFieldBelongsToMessage(field_descriptor, self->message)) {
+    return -1;
+  }
+
+  if (MaybeReleaseOverlappingOneofField(self, field_descriptor) < 0) {
+    return -1;
+  }
+
+  return InternalSetNonOneofScalar(self->message, field_descriptor, arg);
+}
+
 PyObject* FromString(PyTypeObject* cls, PyObject* serialized) {
   PyObject* py_cmsg = PyObject_CallObject(
       reinterpret_cast<PyObject*>(cls), NULL);
@@ -1955,7 +2145,8 @@ static PyObject* AddDescriptors(PyObject* cls, PyObject* descriptor) {
   // which was built previously.
   for (int i = 0; i < message_descriptor->enum_type_count(); ++i) {
     const EnumDescriptor* enum_descriptor = message_descriptor->enum_type(i);
-    ScopedPyObjectPtr enum_type(PyEnumDescriptor_New(enum_descriptor));
+    ScopedPyObjectPtr enum_type(
+        PyEnumDescriptor_FromDescriptor(enum_descriptor));
     if (enum_type == NULL) {
       return NULL;
      }
@@ -1993,7 +2184,7 @@ static PyObject* AddDescriptors(PyObject* cls, PyObject* descriptor) {
   // which was defined previously.
   for (int i = 0; i < message_descriptor->extension_count(); ++i) {
     const google::protobuf::FieldDescriptor* field = message_descriptor->extension(i);
-    ScopedPyObjectPtr extension_field(PyFieldDescriptor_New(field));
+    ScopedPyObjectPtr extension_field(PyFieldDescriptor_FromDescriptor(field));
     if (extension_field == NULL) {
       return NULL;
     }
@@ -2097,26 +2288,6 @@ PyObject* SetState(CMessage* self, PyObject* state) {
 }
 
 // CMessage static methods:
-PyObject* _GetMessageDescriptor(PyObject* unused, PyObject* arg) {
-  return cdescriptor_pool::FindMessageByName(GetDescriptorPool(), arg);
-}
-
-PyObject* _GetFieldDescriptor(PyObject* unused, PyObject* arg) {
-  return cdescriptor_pool::FindFieldByName(GetDescriptorPool(), arg);
-}
-
-PyObject* _GetExtensionDescriptor(PyObject* unused, PyObject* arg) {
-  return cdescriptor_pool::FindExtensionByName(GetDescriptorPool(), arg);
-}
-
-PyObject* _GetEnumDescriptor(PyObject* unused, PyObject* arg) {
-  return cdescriptor_pool::FindEnumTypeByName(GetDescriptorPool(), arg);
-}
-
-PyObject* _GetOneofDescriptor(PyObject* unused, PyObject* arg) {
-  return cdescriptor_pool::FindOneofByName(GetDescriptorPool(), arg);
-}
-
 PyObject* _CheckCalledFromGeneratedFile(PyObject* unused,
                                         PyObject* unused_arg) {
   if (!_CalledFromGeneratedFile(1)) {
@@ -2188,21 +2359,6 @@ static PyMethodDef Methods[] = {
     "or None if no field is set." },
 
   // Static Methods.
-  { "_BuildFile", (PyCFunction)Python_BuildFile, METH_O | METH_STATIC,
-    "Registers a new protocol buffer file in the global C++ descriptor pool." },
-  { "_GetMessageDescriptor", (PyCFunction)_GetMessageDescriptor,
-    METH_O | METH_STATIC, "Finds a message descriptor in the message pool." },
-  { "_GetFieldDescriptor", (PyCFunction)_GetFieldDescriptor,
-    METH_O | METH_STATIC, "Finds a field descriptor in the message pool." },
-  { "_GetExtensionDescriptor", (PyCFunction)_GetExtensionDescriptor,
-    METH_O | METH_STATIC,
-    "Finds a extension descriptor in the message pool." },
-  { "_GetEnumDescriptor", (PyCFunction)_GetEnumDescriptor,
-    METH_O | METH_STATIC,
-    "Finds an enum descriptor in the message pool." },
-  { "_GetOneofDescriptor", (PyCFunction)_GetOneofDescriptor,
-    METH_O | METH_STATIC,
-    "Finds an oneof descriptor in the message pool." },
   { "_CheckCalledFromGeneratedFile", (PyCFunction)_CheckCalledFromGeneratedFile,
     METH_NOARGS | METH_STATIC,
     "Raises TypeError if the caller is not in a _pb2.py file."},
@@ -2234,6 +2390,31 @@ PyObject* GetAttr(CMessage* self, PyObject* name) {
         reinterpret_cast<PyObject*>(self), name);
   }
 
+  if (field_descriptor->is_map()) {
+    PyObject* py_container = NULL;
+    const Descriptor* entry_type = field_descriptor->message_type();
+    const FieldDescriptor* value_type = entry_type->FindFieldByName("value");
+    if (value_type->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
+      PyObject* value_class = cdescriptor_pool::GetMessageClass(
+          GetDescriptorPool(), value_type->message_type());
+      if (value_class == NULL) {
+        return NULL;
+      }
+      py_container = message_map_container::NewContainer(self, field_descriptor,
+                                                         value_class);
+    } else {
+      py_container = scalar_map_container::NewContainer(self, field_descriptor);
+    }
+    if (py_container == NULL) {
+      return NULL;
+    }
+    if (!SetCompositeField(self, name, py_container)) {
+      Py_DECREF(py_container);
+      return NULL;
+    }
+    return py_container;
+  }
+
   if (field_descriptor->label() == FieldDescriptor::LABEL_REPEATED) {
     PyObject* py_container = NULL;
     if (field_descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
@@ -2267,7 +2448,7 @@ PyObject* GetAttr(CMessage* self, PyObject* name) {
     return sub_message;
   }
 
-  return InternalGetScalar(self, field_descriptor);
+  return InternalGetScalar(self->message, field_descriptor);
 }
 
 int SetAttr(CMessage* self, PyObject* name, PyObject* value) {
@@ -2304,9 +2485,7 @@ int SetAttr(CMessage* self, PyObject* name, PyObject* value) {
 
 PyTypeObject CMessage_Type = {
   PyVarObject_HEAD_INIT(&PyType_Type, 0)
-  // Keep the fully qualified _message symbol in a line for opensource.
-  "google.protobuf.pyext._message."
-  "CMessage",                          // tp_name
+  FULL_MODULE_NAME ".CMessage",        // tp_name
   sizeof(CMessage),                    // tp_basicsize
   0,                                   //  tp_itemsize
   (destructor)cmessage::Dealloc,       //  tp_dealloc
@@ -2401,7 +2580,7 @@ void InitGlobals() {
   k_extensions_by_name = PyString_FromString("_extensions_by_name");
   k_extensions_by_number = PyString_FromString("_extensions_by_number");
 
-  message_factory = new DynamicMessageFactory(GetDescriptorPool()->pool);
+  message_factory = new DynamicMessageFactory();
   message_factory->SetDelegateToGeneratedFactory(true);
 }
 
@@ -2469,6 +2648,61 @@ bool InitProto2MessageModule(PyObject *m) {
       reinterpret_cast<PyObject*>(
           &RepeatedCompositeContainer_Type));
 
+  // ScalarMapContainer_Type derives from our MutableMapping type.
+  PyObject* containers =
+      PyImport_ImportModule("google.protobuf.internal.containers");
+  if (containers == NULL) {
+    return false;
+  }
+
+  PyObject* mutable_mapping =
+      PyObject_GetAttrString(containers, "MutableMapping");
+  Py_DECREF(containers);
+
+  if (mutable_mapping == NULL) {
+    return false;
+  }
+
+  if (!PyObject_TypeCheck(mutable_mapping, &PyType_Type)) {
+    Py_DECREF(mutable_mapping);
+    return false;
+  }
+
+  ScalarMapContainer_Type.tp_base =
+      reinterpret_cast<PyTypeObject*>(mutable_mapping);
+
+  if (PyType_Ready(&ScalarMapContainer_Type) < 0) {
+    return false;
+  }
+
+  PyModule_AddObject(m, "ScalarMapContainer",
+                     reinterpret_cast<PyObject*>(&ScalarMapContainer_Type));
+
+  if (PyType_Ready(&ScalarMapIterator_Type) < 0) {
+    return false;
+  }
+
+  PyModule_AddObject(m, "ScalarMapIterator",
+                     reinterpret_cast<PyObject*>(&ScalarMapIterator_Type));
+
+  Py_INCREF(mutable_mapping);
+  MessageMapContainer_Type.tp_base =
+      reinterpret_cast<PyTypeObject*>(mutable_mapping);
+
+  if (PyType_Ready(&MessageMapContainer_Type) < 0) {
+    return false;
+  }
+
+  PyModule_AddObject(m, "MessageMapContainer",
+                     reinterpret_cast<PyObject*>(&MessageMapContainer_Type));
+
+  if (PyType_Ready(&MessageMapIterator_Type) < 0) {
+    return false;
+  }
+
+  PyModule_AddObject(m, "MessageMapIterator",
+                     reinterpret_cast<PyObject*>(&MessageMapIterator_Type));
+
   ExtensionDict_Type.tp_hash = PyObject_HashNotImplemented;
   if (PyType_Ready(&ExtensionDict_Type) < 0) {
     return false;
@@ -2478,6 +2712,12 @@ bool InitProto2MessageModule(PyObject *m) {
       m, "ExtensionDict",
       reinterpret_cast<PyObject*>(&ExtensionDict_Type));
 
+  // Expose the DescriptorPool used to hold all descriptors added from generated
+  // pb2.py files.
+  Py_INCREF(GetDescriptorPool());  // PyModule_AddObject steals a reference.
+  PyModule_AddObject(
+      m, "default_pool", reinterpret_cast<PyObject*>(GetDescriptorPool()));
+
   // This implementation provides full Descriptor types, we advertise it so that
   // descriptor.py can use them in replacement of the Python classes.
   PyModule_AddIntConstant(m, "_USE_C_DESCRIPTORS", 1);

python/google/protobuf/pyext/message.h

@@ -120,7 +120,7 @@ CMessage* NewEmptyMessage(PyObject* type, const Descriptor* descriptor);
 // A new message will be created if this is a read-only default instance.
 //
 // Corresponds to reflection api method ReleaseMessage.
-int ReleaseSubMessage(Message* message,
+int ReleaseSubMessage(CMessage* self,
                       const FieldDescriptor* field_descriptor,
                       CMessage* child_cmessage);
 
@@ -144,7 +144,7 @@ PyObject* InternalGetSubMessage(
 // by slice will be removed from cmessage_list by this function.
 //
 // Corresponds to reflection api method RemoveLast.
-int InternalDeleteRepeatedField(Message* message,
+int InternalDeleteRepeatedField(CMessage* self,
                                 const FieldDescriptor* field_descriptor,
                                 PyObject* slice, PyObject* cmessage_list);
 
@@ -153,10 +153,15 @@ int InternalSetScalar(CMessage* self,
                       const FieldDescriptor* field_descriptor,
                       PyObject* value);
 
+// Sets the specified scalar value to the message.  Requires it is not a Oneof.
+int InternalSetNonOneofScalar(Message* message,
+                              const FieldDescriptor* field_descriptor,
+                              PyObject* arg);
+
 // Retrieves the specified scalar value from the message.
 //
 // Returns a new python reference.
-PyObject* InternalGetScalar(CMessage* self,
+PyObject* InternalGetScalar(const Message* message,
                             const FieldDescriptor* field_descriptor);
 
 // Clears the message, removing all contained data. Extension dictionary and
@@ -279,7 +284,7 @@ extern PyObject* kint64min_py;
 extern PyObject* kint64max_py;
 extern PyObject* kuint64max_py;
 
-#define C(str) const_cast<char*>(str)
+#define FULL_MODULE_NAME "google.protobuf.pyext._message"
 
 void FormatTypeError(PyObject* arg, char* expected_types);
 template<class T>

python/google/protobuf/pyext/message_map_container.cc

@@ -0,0 +1,540 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: haberman@google.com (Josh Haberman)
+
+#include <google/protobuf/pyext/message_map_container.h>
+
+#include <google/protobuf/stubs/common.h>
+#include <google/protobuf/message.h>
+#include <google/protobuf/pyext/message.h>
+#include <google/protobuf/pyext/scoped_pyobject_ptr.h>
+
+namespace google {
+namespace protobuf {
+namespace python {
+
+struct MessageMapIterator {
+  PyObject_HEAD;
+
+  // This dict contains the full contents of what we want to iterate over.
+  // There's no way to avoid building this, because the list representation
+  // (which is canonical) can contain duplicate keys.  So at the very least we
+  // need a set that lets us skip duplicate keys.  And at the point that we're
+  // doing that, we might as well just build the actual dict we're iterating
+  // over and use dict's built-in iterator.
+  PyObject* dict;
+
+  // An iterator on dict.
+  PyObject* iter;
+
+  // A pointer back to the container, so we can notice changes to the version.
+  MessageMapContainer* container;
+
+  // The version of the map when we took the iterator to it.
+  //
+  // We store this so that if the map is modified during iteration we can throw
+  // an error.
+  uint64 version;
+};
+
+static MessageMapIterator* GetIter(PyObject* obj) {
+  return reinterpret_cast<MessageMapIterator*>(obj);
+}
+
+namespace message_map_container {
+
+static MessageMapContainer* GetMap(PyObject* obj) {
+  return reinterpret_cast<MessageMapContainer*>(obj);
+}
+
+// The private constructor of MessageMapContainer objects.
+PyObject* NewContainer(CMessage* parent,
+                       const google::protobuf::FieldDescriptor* parent_field_descriptor,
+                       PyObject* concrete_class) {
+  if (!CheckFieldBelongsToMessage(parent_field_descriptor, parent->message)) {
+    return NULL;
+  }
+
+  PyObject* obj = PyType_GenericAlloc(&MessageMapContainer_Type, 0);
+  if (obj == NULL) {
+    return PyErr_Format(PyExc_RuntimeError,
+                        "Could not allocate new container.");
+  }
+
+  MessageMapContainer* self = GetMap(obj);
+
+  self->message = parent->message;
+  self->parent = parent;
+  self->parent_field_descriptor = parent_field_descriptor;
+  self->owner = parent->owner;
+  self->version = 0;
+
+  self->key_field_descriptor =
+      parent_field_descriptor->message_type()->FindFieldByName("key");
+  self->value_field_descriptor =
+      parent_field_descriptor->message_type()->FindFieldByName("value");
+
+  self->message_dict = PyDict_New();
+  if (self->message_dict == NULL) {
+    return PyErr_Format(PyExc_RuntimeError,
+                        "Could not allocate message dict.");
+  }
+
+  Py_INCREF(concrete_class);
+  self->subclass_init = concrete_class;
+
+  if (self->key_field_descriptor == NULL ||
+      self->value_field_descriptor == NULL) {
+    Py_DECREF(obj);
+    return PyErr_Format(PyExc_KeyError,
+                        "Map entry descriptor did not have key/value fields");
+  }
+
+  return obj;
+}
+
+// Initializes the underlying Message object of "to" so it becomes a new parent
+// repeated scalar, and copies all the values from "from" to it. A child scalar
+// container can be released by passing it as both from and to (e.g. making it
+// the recipient of the new parent message and copying the values from itself).
+static int InitializeAndCopyToParentContainer(
+    MessageMapContainer* from,
+    MessageMapContainer* to) {
+  // For now we require from == to, re-evaluate if we want to support deep copy
+  // as in repeated_composite_container.cc.
+  GOOGLE_DCHECK(from == to);
+  Message* old_message = from->message;
+  Message* new_message = old_message->New();
+  to->parent = NULL;
+  to->parent_field_descriptor = from->parent_field_descriptor;
+  to->message = new_message;
+  to->owner.reset(new_message);
+
+  vector<const FieldDescriptor*> fields;
+  fields.push_back(from->parent_field_descriptor);
+  old_message->GetReflection()->SwapFields(old_message, new_message, fields);
+  return 0;
+}
+
+static PyObject* GetCMessage(MessageMapContainer* self, Message* entry) {
+  // Get or create the CMessage object corresponding to this message.
+  Message* message = entry->GetReflection()->MutableMessage(
+      entry, self->value_field_descriptor);
+  ScopedPyObjectPtr key(PyLong_FromVoidPtr(message));
+  PyObject* ret = PyDict_GetItem(self->message_dict, key);
+
+  if (ret == NULL) {
+    CMessage* cmsg = cmessage::NewEmptyMessage(self->subclass_init,
+                                               message->GetDescriptor());
+    ret = reinterpret_cast<PyObject*>(cmsg);
+
+    if (cmsg == NULL) {
+      return NULL;
+    }
+    cmsg->owner = self->owner;
+    cmsg->message = message;
+    cmsg->parent = self->parent;
+
+    if (PyDict_SetItem(self->message_dict, key, ret) < 0) {
+      Py_DECREF(ret);
+      return NULL;
+    }
+  } else {
+    Py_INCREF(ret);
+  }
+
+  return ret;
+}
+
+int Release(MessageMapContainer* self) {
+  InitializeAndCopyToParentContainer(self, self);
+  return 0;
+}
+
+void SetOwner(MessageMapContainer* self,
+              const shared_ptr<Message>& new_owner) {
+  self->owner = new_owner;
+}
+
+Py_ssize_t Length(PyObject* _self) {
+  MessageMapContainer* self = GetMap(_self);
+  google::protobuf::Message* message = self->message;
+  return message->GetReflection()->FieldSize(*message,
+                                             self->parent_field_descriptor);
+}
+
+int MapKeyMatches(MessageMapContainer* self, const Message* entry,
+                  PyObject* key) {
+  // TODO(haberman): do we need more strict type checking?
+  ScopedPyObjectPtr entry_key(
+      cmessage::InternalGetScalar(entry, self->key_field_descriptor));
+  int ret = PyObject_RichCompareBool(key, entry_key, Py_EQ);
+  return ret;
+}
+
+int SetItem(PyObject *_self, PyObject *key, PyObject *v) {
+  if (v) {
+    PyErr_Format(PyExc_ValueError,
+                 "Direct assignment of submessage not allowed");
+    return -1;
+  }
+
+  // Now we know that this is a delete, not a set.
+
+  MessageMapContainer* self = GetMap(_self);
+  cmessage::AssureWritable(self->parent);
+
+  Message* message = self->message;
+  const Reflection* reflection = message->GetReflection();
+  size_t size =
+      reflection->FieldSize(*message, self->parent_field_descriptor);
+
+  // Right now the Reflection API doesn't support map lookup, so we implement it
+  // via linear search.  We need to search from the end because the underlying
+  // representation can have duplicates if a user calls MergeFrom(); the last
+  // one needs to win.
+  //
+  // TODO(haberman): add lookup API to Reflection API.
+  bool found = false;
+  for (int i = size - 1; i >= 0; i--) {
+    Message* entry = reflection->MutableRepeatedMessage(
+        message, self->parent_field_descriptor, i);
+    int matches = MapKeyMatches(self, entry, key);
+    if (matches < 0) return -1;
+    if (matches) {
+      found = true;
+      if (i != size - 1) {
+        reflection->SwapElements(message, self->parent_field_descriptor, i,
+                                 size - 1);
+      }
+      reflection->RemoveLast(message, self->parent_field_descriptor);
+
+      // Can't exit now, the repeated field representation of maps allows
+      // duplicate keys, and we have to be sure to remove all of them.
+    }
+  }
+
+  if (!found) {
+    PyErr_Format(PyExc_KeyError, "Key not present in map");
+    return -1;
+  }
+
+  self->version++;
+
+  return 0;
+}
+
+PyObject* GetIterator(PyObject *_self) {
+  MessageMapContainer* self = GetMap(_self);
+
+  ScopedPyObjectPtr obj(PyType_GenericAlloc(&MessageMapIterator_Type, 0));
+  if (obj == NULL) {
+    return PyErr_Format(PyExc_KeyError, "Could not allocate iterator");
+  }
+
+  MessageMapIterator* iter = GetIter(obj);
+
+  Py_INCREF(self);
+  iter->container = self;
+  iter->version = self->version;
+  iter->dict = PyDict_New();
+  if (iter->dict == NULL) {
+    return PyErr_Format(PyExc_RuntimeError,
+                        "Could not allocate dict for iterator.");
+  }
+
+  // Build the entire map into a dict right now.  Start from the beginning so
+  // that later entries win in the case of duplicates.
+  Message* message = self->message;
+  const Reflection* reflection = message->GetReflection();
+
+  // Right now the Reflection API doesn't support map lookup, so we implement it
+  // via linear search.  We need to search from the end because the underlying
+  // representation can have duplicates if a user calls MergeFrom(); the last
+  // one needs to win.
+  //
+  // TODO(haberman): add lookup API to Reflection API.
+  size_t size =
+      reflection->FieldSize(*message, self->parent_field_descriptor);
+  for (int i = size - 1; i >= 0; i--) {
+    Message* entry = reflection->MutableRepeatedMessage(
+        message, self->parent_field_descriptor, i);
+    ScopedPyObjectPtr key(
+        cmessage::InternalGetScalar(entry, self->key_field_descriptor));
+    if (PyDict_SetItem(iter->dict, key.get(), GetCMessage(self, entry)) < 0) {
+      return PyErr_Format(PyExc_RuntimeError,
+                          "SetItem failed in iterator construction.");
+    }
+  }
+
+  iter->iter = PyObject_GetIter(iter->dict);
+
+  return obj.release();
+}
+
+PyObject* GetItem(PyObject* _self, PyObject* key) {
+  MessageMapContainer* self = GetMap(_self);
+  cmessage::AssureWritable(self->parent);
+  Message* message = self->message;
+  const Reflection* reflection = message->GetReflection();
+
+  // Right now the Reflection API doesn't support map lookup, so we implement it
+  // via linear search.  We need to search from the end because the underlying
+  // representation can have duplicates if a user calls MergeFrom(); the last
+  // one needs to win.
+  //
+  // TODO(haberman): add lookup API to Reflection API.
+  size_t size =
+      reflection->FieldSize(*message, self->parent_field_descriptor);
+  for (int i = size - 1; i >= 0; i--) {
+    Message* entry = reflection->MutableRepeatedMessage(
+        message, self->parent_field_descriptor, i);
+    int matches = MapKeyMatches(self, entry, key);
+    if (matches < 0) return NULL;
+    if (matches) {
+      return GetCMessage(self, entry);
+    }
+  }
+
+  // Key is not already present; insert a new entry.
+  Message* entry =
+      reflection->AddMessage(message, self->parent_field_descriptor);
+
+  self->version++;
+
+  if (cmessage::InternalSetNonOneofScalar(entry, self->key_field_descriptor,
+                                          key) < 0) {
+    reflection->RemoveLast(message, self->parent_field_descriptor);
+    return NULL;
+  }
+
+  return GetCMessage(self, entry);
+}
+
+PyObject* Contains(PyObject* _self, PyObject* key) {
+  MessageMapContainer* self = GetMap(_self);
+  Message* message = self->message;
+  const Reflection* reflection = message->GetReflection();
+
+  // Right now the Reflection API doesn't support map lookup, so we implement it
+  // via linear search.
+  //
+  // TODO(haberman): add lookup API to Reflection API.
+  size_t size =
+      reflection->FieldSize(*message, self->parent_field_descriptor);
+  for (int i = 0; i < size; i++) {
+    Message* entry = reflection->MutableRepeatedMessage(
+        message, self->parent_field_descriptor, i);
+    int matches = MapKeyMatches(self, entry, key);
+    if (matches < 0) return NULL;
+    if (matches) {
+      Py_RETURN_TRUE;
+    }
+  }
+
+  Py_RETURN_FALSE;
+}
+
+PyObject* Clear(PyObject* _self) {
+  MessageMapContainer* self = GetMap(_self);
+  cmessage::AssureWritable(self->parent);
+  Message* message = self->message;
+  const Reflection* reflection = message->GetReflection();
+
+  self->version++;
+  reflection->ClearField(message, self->parent_field_descriptor);
+
+  Py_RETURN_NONE;
+}
+
+PyObject* Get(PyObject* self, PyObject* args) {
+  PyObject* key;
+  PyObject* default_value = NULL;
+  if (PyArg_ParseTuple(args, "O|O", &key, &default_value) < 0) {
+    return NULL;
+  }
+
+  ScopedPyObjectPtr is_present(Contains(self, key));
+  if (is_present.get() == NULL) {
+    return NULL;
+  }
+
+  if (PyObject_IsTrue(is_present.get())) {
+    return GetItem(self, key);
+  } else {
+    if (default_value != NULL) {
+      Py_INCREF(default_value);
+      return default_value;
+    } else {
+      Py_RETURN_NONE;
+    }
+  }
+}
+
+static PyMappingMethods MpMethods = {
+  Length,    // mp_length
+  GetItem,   // mp_subscript
+  SetItem,   // mp_ass_subscript
+};
+
+static void Dealloc(PyObject* _self) {
+  MessageMapContainer* self = GetMap(_self);
+  self->owner.reset();
+  Py_DECREF(self->message_dict);
+  Py_TYPE(_self)->tp_free(_self);
+}
+
+static PyMethodDef Methods[] = {
+  { "__contains__", (PyCFunction)Contains, METH_O,
+    "Tests whether the map contains this element."},
+  { "clear", (PyCFunction)Clear, METH_NOARGS,
+    "Removes all elements from the map."},
+  { "get", Get, METH_VARARGS,
+    "Gets the value for the given key if present, or otherwise a default" },
+  { "get_or_create", GetItem, METH_O,
+    "Alias for getitem, useful to make explicit that the map is mutated." },
+  /*
+  { "__deepcopy__", (PyCFunction)DeepCopy, METH_VARARGS,
+    "Makes a deep copy of the class." },
+  { "__reduce__", (PyCFunction)Reduce, METH_NOARGS,
+    "Outputs picklable representation of the repeated field." },
+  */
+  {NULL, NULL},
+};
+
+}  // namespace message_map_container
+
+namespace message_map_iterator {
+
+static void Dealloc(PyObject* _self) {
+  MessageMapIterator* self = GetIter(_self);
+  Py_DECREF(self->dict);
+  Py_DECREF(self->iter);
+  Py_DECREF(self->container);
+  Py_TYPE(_self)->tp_free(_self);
+}
+
+PyObject* IterNext(PyObject* _self) {
+  MessageMapIterator* self = GetIter(_self);
+
+  // This won't catch mutations to the map performed by MergeFrom(); no easy way
+  // to address that.
+  if (self->version != self->container->version) {
+    return PyErr_Format(PyExc_RuntimeError,
+                        "Map modified during iteration.");
+  }
+
+  return PyIter_Next(self->iter);
+}
+
+}  // namespace message_map_iterator
+
+PyTypeObject MessageMapContainer_Type = {
+  PyVarObject_HEAD_INIT(&PyType_Type, 0)
+  FULL_MODULE_NAME ".MessageMapContainer",  //  tp_name
+  sizeof(MessageMapContainer),         //  tp_basicsize
+  0,                                   //  tp_itemsize
+  message_map_container::Dealloc,      //  tp_dealloc
+  0,                                   //  tp_print
+  0,                                   //  tp_getattr
+  0,                                   //  tp_setattr
+  0,                                   //  tp_compare
+  0,                                   //  tp_repr
+  0,                                   //  tp_as_number
+  0,                                   //  tp_as_sequence
+  &message_map_container::MpMethods,   //  tp_as_mapping
+  0,                                   //  tp_hash
+  0,                                   //  tp_call
+  0,                                   //  tp_str
+  0,                                   //  tp_getattro
+  0,                                   //  tp_setattro
+  0,                                   //  tp_as_buffer
+  Py_TPFLAGS_DEFAULT,                  //  tp_flags
+  "A map container for message",       //  tp_doc
+  0,                                   //  tp_traverse
+  0,                                   //  tp_clear
+  0,                                   //  tp_richcompare
+  0,                                   //  tp_weaklistoffset
+  message_map_container::GetIterator,  //  tp_iter
+  0,                                   //  tp_iternext
+  message_map_container::Methods,      //  tp_methods
+  0,                                   //  tp_members
+  0,                                   //  tp_getset
+  0,                                   //  tp_base
+  0,                                   //  tp_dict
+  0,                                   //  tp_descr_get
+  0,                                   //  tp_descr_set
+  0,                                   //  tp_dictoffset
+  0,                                   //  tp_init
+};
+
+PyTypeObject MessageMapIterator_Type = {
+  PyVarObject_HEAD_INIT(&PyType_Type, 0)
+  FULL_MODULE_NAME ".MessageMapIterator",  //  tp_name
+  sizeof(MessageMapIterator),          //  tp_basicsize
+  0,                                   //  tp_itemsize
+  message_map_iterator::Dealloc,       //  tp_dealloc
+  0,                                   //  tp_print
+  0,                                   //  tp_getattr
+  0,                                   //  tp_setattr
+  0,                                   //  tp_compare
+  0,                                   //  tp_repr
+  0,                                   //  tp_as_number
+  0,                                   //  tp_as_sequence
+  0,                                   //  tp_as_mapping
+  0,                                   //  tp_hash
+  0,                                   //  tp_call
+  0,                                   //  tp_str
+  0,                                   //  tp_getattro
+  0,                                   //  tp_setattro
+  0,                                   //  tp_as_buffer
+  Py_TPFLAGS_DEFAULT,                  //  tp_flags
+  "A scalar map iterator",             //  tp_doc
+  0,                                   //  tp_traverse
+  0,                                   //  tp_clear
+  0,                                   //  tp_richcompare
+  0,                                   //  tp_weaklistoffset
+  PyObject_SelfIter,                   //  tp_iter
+  message_map_iterator::IterNext,       //  tp_iternext
+  0,                                   //  tp_methods
+  0,                                   //  tp_members
+  0,                                   //  tp_getset
+  0,                                   //  tp_base
+  0,                                   //  tp_dict
+  0,                                   //  tp_descr_get
+  0,                                   //  tp_descr_set
+  0,                                   //  tp_dictoffset
+  0,                                   //  tp_init
+};
+
+}  // namespace python
+}  // namespace protobuf
+}  // namespace google

python/google/protobuf/pyext/message_map_container.h

@@ -0,0 +1,117 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_MESSAGE_MAP_CONTAINER_H__
+#define GOOGLE_PROTOBUF_PYTHON_CPP_MESSAGE_MAP_CONTAINER_H__
+
+#include <Python.h>
+
+#include <memory>
+#ifndef _SHARED_PTR_H
+#include <google/protobuf/stubs/shared_ptr.h>
+#endif
+
+#include <google/protobuf/descriptor.h>
+
+namespace google {
+namespace protobuf {
+
+class Message;
+
+using internal::shared_ptr;
+
+namespace python {
+
+struct CMessage;
+
+struct MessageMapContainer {
+  PyObject_HEAD;
+
+  // This is the top-level C++ Message object that owns the whole
+  // proto tree.  Every Python MessageMapContainer holds a
+  // reference to it in order to keep it alive as long as there's a
+  // Python object that references any part of the tree.
+  shared_ptr<Message> owner;
+
+  // Pointer to the C++ Message that contains this container.  The
+  // MessageMapContainer does not own this pointer.
+  Message* message;
+
+  // Weak reference to a parent CMessage object (i.e. may be NULL.)
+  //
+  // Used to make sure all ancestors are also mutable when first
+  // modifying the container.
+  CMessage* parent;
+
+  // Pointer to the parent's descriptor that describes this
+  // field.  Used together with the parent's message when making a
+  // default message instance mutable.
+  // The pointer is owned by the global DescriptorPool.
+  const FieldDescriptor* parent_field_descriptor;
+  const FieldDescriptor* key_field_descriptor;
+  const FieldDescriptor* value_field_descriptor;
+
+  // A callable that is used to create new child messages.
+  PyObject* subclass_init;
+
+  // A dict mapping Message* -> CMessage.
+  PyObject* message_dict;
+
+  // We bump this whenever we perform a mutation, to invalidate existing
+  // iterators.
+  uint64 version;
+};
+
+extern PyTypeObject MessageMapContainer_Type;
+extern PyTypeObject MessageMapIterator_Type;
+
+namespace message_map_container {
+
+// Builds a MessageMapContainer object, from a parent message and a
+// field descriptor.
+extern PyObject* NewContainer(CMessage* parent,
+                              const FieldDescriptor* parent_field_descriptor,
+                              PyObject* concrete_class);
+
+// Releases the messages in the container to a new message.
+//
+// Returns 0 on success, -1 on failure.
+int Release(MessageMapContainer* self);
+
+// Set the owner field of self and any children of self.
+void SetOwner(MessageMapContainer* self,
+              const shared_ptr<Message>& new_owner);
+
+}  // namespace message_map_container
+}  // namespace python
+}  // namespace protobuf
+
+}  // namespace google
+#endif  // GOOGLE_PROTOBUF_PYTHON_CPP_MESSAGE_MAP_CONTAINER_H__

python/google/protobuf/pyext/repeated_composite_container.cc

@@ -367,8 +367,8 @@ int AssignSubscript(RepeatedCompositeContainer* self,
   }
 
   // Delete from the underlying Message, if any.
-  if (self->message != NULL) {
-    if (cmessage::InternalDeleteRepeatedField(self->message,
+  if (self->parent != NULL) {
+    if (cmessage::InternalDeleteRepeatedField(self->parent,
                                               self->parent_field_descriptor,
                                               slice,
                                               self->child_messages) < 0) {
@@ -572,47 +572,35 @@ static PyObject* Pop(RepeatedCompositeContainer* self,
   return item;
 }
 
-// The caller takes ownership of the returned Message.
-Message* ReleaseLast(const FieldDescriptor* field,
-                     const Descriptor* type,
-                     Message* message) {
+// Release field of parent message and transfer the ownership to target.
+void ReleaseLastTo(CMessage* parent,
+                   const FieldDescriptor* field,
+                   CMessage* target) {
+  GOOGLE_CHECK_NOTNULL(parent);
   GOOGLE_CHECK_NOTNULL(field);
-  GOOGLE_CHECK_NOTNULL(type);
-  GOOGLE_CHECK_NOTNULL(message);
+  GOOGLE_CHECK_NOTNULL(target);
 
-  Message* released_message = message->GetReflection()->ReleaseLast(
-      message, field);
+  shared_ptr<Message> released_message(
+      parent->message->GetReflection()->ReleaseLast(parent->message, field));
   // TODO(tibell): Deal with proto1.
 
   // ReleaseMessage will return NULL which differs from
   // child_cmessage->message, if the field does not exist.  In this case,
   // the latter points to the default instance via a const_cast<>, so we
   // have to reset it to a new mutable object since we are taking ownership.
-  if (released_message == NULL) {
+  if (released_message.get() == NULL) {
     const Message* prototype =
-        cmessage::GetMessageFactory()->GetPrototype(type);
+        cmessage::GetMessageFactory()->GetPrototype(
+            target->message->GetDescriptor());
     GOOGLE_CHECK_NOTNULL(prototype);
-    return prototype->New();
-  } else {
-    return released_message;
+    released_message.reset(prototype->New());
   }
-}
 
-// Release field of message and transfer the ownership to cmessage.
-void ReleaseLastTo(const FieldDescriptor* field,
-                   Message* message,
-                   CMessage* cmessage) {
-  GOOGLE_CHECK_NOTNULL(field);
-  GOOGLE_CHECK_NOTNULL(message);
-  GOOGLE_CHECK_NOTNULL(cmessage);
-
-  shared_ptr<Message> released_message(
-      ReleaseLast(field, cmessage->message->GetDescriptor(), message));
-  cmessage->parent = NULL;
-  cmessage->parent_field_descriptor = NULL;
-  cmessage->message = released_message.get();
-  cmessage->read_only = false;
-  cmessage::SetOwner(cmessage, released_message);
+  target->parent = NULL;
+  target->parent_field_descriptor = NULL;
+  target->message = released_message.get();
+  target->read_only = false;
+  cmessage::SetOwner(target, released_message);
 }
 
 // Called to release a container using
@@ -635,7 +623,7 @@ int Release(RepeatedCompositeContainer* self) {
   for (Py_ssize_t i = size - 1; i >= 0; --i) {
     CMessage* child_cmessage = reinterpret_cast<CMessage*>(
         PyList_GET_ITEM(self->child_messages, i));
-    ReleaseLastTo(field, message, child_cmessage);
+    ReleaseLastTo(self->parent, field, child_cmessage);
   }
 
   // Detach from containing message.
@@ -732,9 +720,7 @@ static PyMethodDef Methods[] = {
 
 PyTypeObject RepeatedCompositeContainer_Type = {
   PyVarObject_HEAD_INIT(&PyType_Type, 0)
-  // Keep the fully qualified _message symbol in a line for opensource.
-  "google.protobuf.pyext._message."
-  "RepeatedCompositeContainer",        // tp_name
+  FULL_MODULE_NAME ".RepeatedCompositeContainer",  // tp_name
   sizeof(RepeatedCompositeContainer),  // tp_basicsize
   0,                                   //  tp_itemsize
   (destructor)repeated_composite_container::Dealloc,  //  tp_dealloc

python/google/protobuf/pyext/repeated_composite_container.h

@@ -161,13 +161,13 @@ int SetOwner(RepeatedCompositeContainer* self,
              const shared_ptr<Message>& new_owner);
 
 // Removes the last element of the repeated message field 'field' on
-// the Message 'message', and transfers the ownership of the released
-// Message to 'cmessage'.
+// the Message 'parent', and transfers the ownership of the released
+// Message to 'target'.
 //
 // Corresponds to reflection api method ReleaseMessage.
-void ReleaseLastTo(const FieldDescriptor* field,
-                   Message* message,
-                   CMessage* cmessage);
+void ReleaseLastTo(CMessage* parent,
+                   const FieldDescriptor* field,
+                   CMessage* target);
 
 }  // namespace repeated_composite_container
 }  // namespace python

python/google/protobuf/pyext/repeated_scalar_container.cc

@@ -102,7 +102,7 @@ static int AssignItem(RepeatedScalarContainer* self,
 
   if (arg == NULL) {
     ScopedPyObjectPtr py_index(PyLong_FromLong(index));
-    return cmessage::InternalDeleteRepeatedField(message, field_descriptor,
+    return cmessage::InternalDeleteRepeatedField(self->parent, field_descriptor,
                                                  py_index, NULL);
   }
 
@@ -470,7 +470,7 @@ static int AssSubscript(RepeatedScalarContainer* self,
 
   if (value == NULL) {
     return cmessage::InternalDeleteRepeatedField(
-        message, field_descriptor, slice, NULL);
+        self->parent, field_descriptor, slice, NULL);
   }
 
   if (!create_list) {
@@ -769,9 +769,7 @@ static PyMethodDef Methods[] = {
 
 PyTypeObject RepeatedScalarContainer_Type = {
   PyVarObject_HEAD_INIT(&PyType_Type, 0)
-  // Keep the fully qualified _message symbol in a line for opensource.
-  "google.protobuf.pyext._message."
-  "RepeatedScalarContainer",           // tp_name
+  FULL_MODULE_NAME ".RepeatedScalarContainer",  // tp_name
   sizeof(RepeatedScalarContainer),     // tp_basicsize
   0,                                   //  tp_itemsize
   (destructor)repeated_scalar_container::Dealloc,  //  tp_dealloc

python/google/protobuf/pyext/scalar_map_container.cc

@@ -0,0 +1,514 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: haberman@google.com (Josh Haberman)
+
+#include <google/protobuf/pyext/scalar_map_container.h>
+
+#include <google/protobuf/stubs/common.h>
+#include <google/protobuf/message.h>
+#include <google/protobuf/pyext/message.h>
+#include <google/protobuf/pyext/scoped_pyobject_ptr.h>
+
+namespace google {
+namespace protobuf {
+namespace python {
+
+struct ScalarMapIterator {
+  PyObject_HEAD;
+
+  // This dict contains the full contents of what we want to iterate over.
+  // There's no way to avoid building this, because the list representation
+  // (which is canonical) can contain duplicate keys.  So at the very least we
+  // need a set that lets us skip duplicate keys.  And at the point that we're
+  // doing that, we might as well just build the actual dict we're iterating
+  // over and use dict's built-in iterator.
+  PyObject* dict;
+
+  // An iterator on dict.
+  PyObject* iter;
+
+  // A pointer back to the container, so we can notice changes to the version.
+  ScalarMapContainer* container;
+
+  // The version of the map when we took the iterator to it.
+  //
+  // We store this so that if the map is modified during iteration we can throw
+  // an error.
+  uint64 version;
+};
+
+static ScalarMapIterator* GetIter(PyObject* obj) {
+  return reinterpret_cast<ScalarMapIterator*>(obj);
+}
+
+namespace scalar_map_container {
+
+static ScalarMapContainer* GetMap(PyObject* obj) {
+  return reinterpret_cast<ScalarMapContainer*>(obj);
+}
+
+// The private constructor of ScalarMapContainer objects.
+PyObject *NewContainer(
+    CMessage* parent, const google::protobuf::FieldDescriptor* parent_field_descriptor) {
+  if (!CheckFieldBelongsToMessage(parent_field_descriptor, parent->message)) {
+    return NULL;
+  }
+
+  ScopedPyObjectPtr obj(PyType_GenericAlloc(&ScalarMapContainer_Type, 0));
+  if (obj.get() == NULL) {
+    return PyErr_Format(PyExc_RuntimeError,
+                        "Could not allocate new container.");
+  }
+
+  ScalarMapContainer* self = GetMap(obj);
+
+  self->message = parent->message;
+  self->parent = parent;
+  self->parent_field_descriptor = parent_field_descriptor;
+  self->owner = parent->owner;
+  self->version = 0;
+
+  self->key_field_descriptor =
+      parent_field_descriptor->message_type()->FindFieldByName("key");
+  self->value_field_descriptor =
+      parent_field_descriptor->message_type()->FindFieldByName("value");
+
+  if (self->key_field_descriptor == NULL ||
+      self->value_field_descriptor == NULL) {
+    return PyErr_Format(PyExc_KeyError,
+                        "Map entry descriptor did not have key/value fields");
+  }
+
+  return obj.release();
+}
+
+// Initializes the underlying Message object of "to" so it becomes a new parent
+// repeated scalar, and copies all the values from "from" to it. A child scalar
+// container can be released by passing it as both from and to (e.g. making it
+// the recipient of the new parent message and copying the values from itself).
+static int InitializeAndCopyToParentContainer(
+    ScalarMapContainer* from,
+    ScalarMapContainer* to) {
+  // For now we require from == to, re-evaluate if we want to support deep copy
+  // as in repeated_scalar_container.cc.
+  GOOGLE_DCHECK(from == to);
+  Message* old_message = from->message;
+  Message* new_message = old_message->New();
+  to->parent = NULL;
+  to->parent_field_descriptor = from->parent_field_descriptor;
+  to->message = new_message;
+  to->owner.reset(new_message);
+
+  vector<const FieldDescriptor*> fields;
+  fields.push_back(from->parent_field_descriptor);
+  old_message->GetReflection()->SwapFields(old_message, new_message, fields);
+  return 0;
+}
+
+int Release(ScalarMapContainer* self) {
+  return InitializeAndCopyToParentContainer(self, self);
+}
+
+void SetOwner(ScalarMapContainer* self,
+              const shared_ptr<Message>& new_owner) {
+  self->owner = new_owner;
+}
+
+Py_ssize_t Length(PyObject* _self) {
+  ScalarMapContainer* self = GetMap(_self);
+  google::protobuf::Message* message = self->message;
+  return message->GetReflection()->FieldSize(*message,
+                                             self->parent_field_descriptor);
+}
+
+int MapKeyMatches(ScalarMapContainer* self, const Message* entry,
+                  PyObject* key) {
+  // TODO(haberman): do we need more strict type checking?
+  ScopedPyObjectPtr entry_key(
+      cmessage::InternalGetScalar(entry, self->key_field_descriptor));
+  int ret = PyObject_RichCompareBool(key, entry_key, Py_EQ);
+  return ret;
+}
+
+PyObject* GetItem(PyObject* _self, PyObject* key) {
+  ScalarMapContainer* self = GetMap(_self);
+
+  Message* message = self->message;
+  const Reflection* reflection = message->GetReflection();
+
+  // Right now the Reflection API doesn't support map lookup, so we implement it
+  // via linear search.
+  //
+  // TODO(haberman): add lookup API to Reflection API.
+  size_t size = reflection->FieldSize(*message, self->parent_field_descriptor);
+  for (int i = size - 1; i >= 0; i--) {
+    const Message& entry = reflection->GetRepeatedMessage(
+        *message, self->parent_field_descriptor, i);
+    int matches = MapKeyMatches(self, &entry, key);
+    if (matches < 0) return NULL;
+    if (matches) {
+      return cmessage::InternalGetScalar(&entry, self->value_field_descriptor);
+    }
+  }
+
+  // Need to add a new entry.
+  Message* entry =
+      reflection->AddMessage(message, self->parent_field_descriptor);
+  PyObject* ret = NULL;
+
+  if (cmessage::InternalSetNonOneofScalar(entry, self->key_field_descriptor,
+                                          key) >= 0) {
+    ret = cmessage::InternalGetScalar(entry, self->value_field_descriptor);
+  }
+
+  self->version++;
+
+  // If there was a type error above, it set the Python exception.
+  return ret;
+}
+
+int SetItem(PyObject *_self, PyObject *key, PyObject *v) {
+  ScalarMapContainer* self = GetMap(_self);
+  cmessage::AssureWritable(self->parent);
+
+  Message* message = self->message;
+  const Reflection* reflection = message->GetReflection();
+  size_t size =
+      reflection->FieldSize(*message, self->parent_field_descriptor);
+  self->version++;
+
+  if (v) {
+    // Set item.
+    //
+    // Right now the Reflection API doesn't support map lookup, so we implement
+    // it via linear search.
+    //
+    // TODO(haberman): add lookup API to Reflection API.
+    for (int i = size - 1; i >= 0; i--) {
+      Message* entry = reflection->MutableRepeatedMessage(
+          message, self->parent_field_descriptor, i);
+      int matches = MapKeyMatches(self, entry, key);
+      if (matches < 0) return -1;
+      if (matches) {
+        return cmessage::InternalSetNonOneofScalar(
+            entry, self->value_field_descriptor, v);
+      }
+    }
+
+    // Key is not already present; insert a new entry.
+    Message* entry =
+        reflection->AddMessage(message, self->parent_field_descriptor);
+
+    if (cmessage::InternalSetNonOneofScalar(entry, self->key_field_descriptor,
+                                            key) < 0 ||
+        cmessage::InternalSetNonOneofScalar(entry, self->value_field_descriptor,
+                                            v) < 0) {
+      reflection->RemoveLast(message, self->parent_field_descriptor);
+      return -1;
+    }
+
+    return 0;
+  } else {
+    bool found = false;
+    for (int i = size - 1; i >= 0; i--) {
+      Message* entry = reflection->MutableRepeatedMessage(
+          message, self->parent_field_descriptor, i);
+      int matches = MapKeyMatches(self, entry, key);
+      if (matches < 0) return -1;
+      if (matches) {
+        found = true;
+        if (i != size - 1) {
+          reflection->SwapElements(message, self->parent_field_descriptor, i,
+                                   size - 1);
+        }
+        reflection->RemoveLast(message, self->parent_field_descriptor);
+
+        // Can't exit now, the repeated field representation of maps allows
+        // duplicate keys, and we have to be sure to remove all of them.
+      }
+    }
+
+    if (found) {
+      return 0;
+    } else {
+      PyErr_Format(PyExc_KeyError, "Key not present in map");
+      return -1;
+    }
+  }
+}
+
+PyObject* GetIterator(PyObject *_self) {
+  ScalarMapContainer* self = GetMap(_self);
+
+  ScopedPyObjectPtr obj(PyType_GenericAlloc(&ScalarMapIterator_Type, 0));
+  if (obj == NULL) {
+    return PyErr_Format(PyExc_KeyError, "Could not allocate iterator");
+  }
+
+  ScalarMapIterator* iter = GetIter(obj.get());
+
+  Py_INCREF(self);
+  iter->container = self;
+  iter->version = self->version;
+  iter->dict = PyDict_New();
+  if (iter->dict == NULL) {
+    return PyErr_Format(PyExc_RuntimeError,
+                        "Could not allocate dict for iterator.");
+  }
+
+  // Build the entire map into a dict right now.  Start from the beginning so
+  // that later entries win in the case of duplicates.
+  Message* message = self->message;
+  const Reflection* reflection = message->GetReflection();
+
+  // Right now the Reflection API doesn't support map lookup, so we implement it
+  // via linear search.  We need to search from the end because the underlying
+  // representation can have duplicates if a user calls MergeFrom(); the last
+  // one needs to win.
+  //
+  // TODO(haberman): add lookup API to Reflection API.
+  size_t size =
+      reflection->FieldSize(*message, self->parent_field_descriptor);
+  for (int i = 0; i < size; i++) {
+    Message* entry = reflection->MutableRepeatedMessage(
+        message, self->parent_field_descriptor, i);
+    ScopedPyObjectPtr key(
+        cmessage::InternalGetScalar(entry, self->key_field_descriptor));
+    ScopedPyObjectPtr val(
+        cmessage::InternalGetScalar(entry, self->value_field_descriptor));
+    if (PyDict_SetItem(iter->dict, key.get(), val.get()) < 0) {
+      return PyErr_Format(PyExc_RuntimeError,
+                          "SetItem failed in iterator construction.");
+    }
+  }
+
+
+  iter->iter = PyObject_GetIter(iter->dict);
+
+
+  return obj.release();
+}
+
+PyObject* Clear(PyObject* _self) {
+  ScalarMapContainer* self = GetMap(_self);
+  cmessage::AssureWritable(self->parent);
+  Message* message = self->message;
+  const Reflection* reflection = message->GetReflection();
+
+  reflection->ClearField(message, self->parent_field_descriptor);
+
+  Py_RETURN_NONE;
+}
+
+PyObject* Contains(PyObject* _self, PyObject* key) {
+  ScalarMapContainer* self = GetMap(_self);
+
+  Message* message = self->message;
+  const Reflection* reflection = message->GetReflection();
+
+  // Right now the Reflection API doesn't support map lookup, so we implement it
+  // via linear search.
+  //
+  // TODO(haberman): add lookup API to Reflection API.
+  size_t size = reflection->FieldSize(*message, self->parent_field_descriptor);
+  for (int i = size - 1; i >= 0; i--) {
+    const Message& entry = reflection->GetRepeatedMessage(
+        *message, self->parent_field_descriptor, i);
+    int matches = MapKeyMatches(self, &entry, key);
+    if (matches < 0) return NULL;
+    if (matches) {
+      Py_RETURN_TRUE;
+    }
+  }
+
+  Py_RETURN_FALSE;
+}
+
+PyObject* Get(PyObject* self, PyObject* args) {
+  PyObject* key;
+  PyObject* default_value = NULL;
+  if (PyArg_ParseTuple(args, "O|O", &key, &default_value) < 0) {
+    return NULL;
+  }
+
+  ScopedPyObjectPtr is_present(Contains(self, key));
+  if (is_present.get() == NULL) {
+    return NULL;
+  }
+
+  if (PyObject_IsTrue(is_present.get())) {
+    return GetItem(self, key);
+  } else {
+    if (default_value != NULL) {
+      Py_INCREF(default_value);
+      return default_value;
+    } else {
+      Py_RETURN_NONE;
+    }
+  }
+}
+
+static PyMappingMethods MpMethods = {
+  Length,    // mp_length
+  GetItem,   // mp_subscript
+  SetItem,   // mp_ass_subscript
+};
+
+static void Dealloc(PyObject* _self) {
+  ScalarMapContainer* self = GetMap(_self);
+  self->owner.reset();
+  Py_TYPE(_self)->tp_free(_self);
+}
+
+static PyMethodDef Methods[] = {
+  { "__contains__", Contains, METH_O,
+    "Tests whether a key is a member of the map." },
+  { "clear", (PyCFunction)Clear, METH_NOARGS,
+    "Removes all elements from the map." },
+  { "get", Get, METH_VARARGS,
+    "Gets the value for the given key if present, or otherwise a default" },
+  /*
+  { "__deepcopy__", (PyCFunction)DeepCopy, METH_VARARGS,
+    "Makes a deep copy of the class." },
+  { "__reduce__", (PyCFunction)Reduce, METH_NOARGS,
+    "Outputs picklable representation of the repeated field." },
+  */
+  {NULL, NULL},
+};
+
+}  // namespace scalar_map_container
+
+namespace scalar_map_iterator {
+
+static void Dealloc(PyObject* _self) {
+  ScalarMapIterator* self = GetIter(_self);
+  Py_DECREF(self->dict);
+  Py_DECREF(self->iter);
+  Py_DECREF(self->container);
+  Py_TYPE(_self)->tp_free(_self);
+}
+
+PyObject* IterNext(PyObject* _self) {
+  ScalarMapIterator* self = GetIter(_self);
+
+  // This won't catch mutations to the map performed by MergeFrom(); no easy way
+  // to address that.
+  if (self->version != self->container->version) {
+    return PyErr_Format(PyExc_RuntimeError,
+                        "Map modified during iteration.");
+  }
+
+  return PyIter_Next(self->iter);
+}
+
+}  // namespace scalar_map_iterator
+
+PyTypeObject ScalarMapContainer_Type = {
+  PyVarObject_HEAD_INIT(&PyType_Type, 0)
+  FULL_MODULE_NAME ".ScalarMapContainer",  //  tp_name
+  sizeof(ScalarMapContainer),          //  tp_basicsize
+  0,                                   //  tp_itemsize
+  scalar_map_container::Dealloc,       //  tp_dealloc
+  0,                                   //  tp_print
+  0,                                   //  tp_getattr
+  0,                                   //  tp_setattr
+  0,                                   //  tp_compare
+  0,                                   //  tp_repr
+  0,                                   //  tp_as_number
+  0,                                   //  tp_as_sequence
+  &scalar_map_container::MpMethods,    //  tp_as_mapping
+  0,                                   //  tp_hash
+  0,                                   //  tp_call
+  0,                                   //  tp_str
+  0,                                   //  tp_getattro
+  0,                                   //  tp_setattro
+  0,                                   //  tp_as_buffer
+  Py_TPFLAGS_DEFAULT,                  //  tp_flags
+  "A scalar map container",            //  tp_doc
+  0,                                   //  tp_traverse
+  0,                                   //  tp_clear
+  0,                                   //  tp_richcompare
+  0,                                   //  tp_weaklistoffset
+  scalar_map_container::GetIterator,   //  tp_iter
+  0,                                   //  tp_iternext
+  scalar_map_container::Methods,       //  tp_methods
+  0,                                   //  tp_members
+  0,                                   //  tp_getset
+  0,                                   //  tp_base
+  0,                                   //  tp_dict
+  0,                                   //  tp_descr_get
+  0,                                   //  tp_descr_set
+  0,                                   //  tp_dictoffset
+  0,                                   //  tp_init
+};
+
+PyTypeObject ScalarMapIterator_Type = {
+  PyVarObject_HEAD_INIT(&PyType_Type, 0)
+  FULL_MODULE_NAME ".ScalarMapIterator",  //  tp_name
+  sizeof(ScalarMapIterator),           //  tp_basicsize
+  0,                                   //  tp_itemsize
+  scalar_map_iterator::Dealloc,        //  tp_dealloc
+  0,                                   //  tp_print
+  0,                                   //  tp_getattr
+  0,                                   //  tp_setattr
+  0,                                   //  tp_compare
+  0,                                   //  tp_repr
+  0,                                   //  tp_as_number
+  0,                                   //  tp_as_sequence
+  0,                                   //  tp_as_mapping
+  0,                                   //  tp_hash
+  0,                                   //  tp_call
+  0,                                   //  tp_str
+  0,                                   //  tp_getattro
+  0,                                   //  tp_setattro
+  0,                                   //  tp_as_buffer
+  Py_TPFLAGS_DEFAULT,                  //  tp_flags
+  "A scalar map iterator",             //  tp_doc
+  0,                                   //  tp_traverse
+  0,                                   //  tp_clear
+  0,                                   //  tp_richcompare
+  0,                                   //  tp_weaklistoffset
+  PyObject_SelfIter,                   //  tp_iter
+  scalar_map_iterator::IterNext,       //  tp_iternext
+  0,                                   //  tp_methods
+  0,                                   //  tp_members
+  0,                                   //  tp_getset
+  0,                                   //  tp_base
+  0,                                   //  tp_dict
+  0,                                   //  tp_descr_get
+  0,                                   //  tp_descr_set
+  0,                                   //  tp_dictoffset
+  0,                                   //  tp_init
+};
+
+}  // namespace python
+}  // namespace protobuf
+}  // namespace google

python/google/protobuf/pyext/scalar_map_container.h

@@ -0,0 +1,110 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_SCALAR_MAP_CONTAINER_H__
+#define GOOGLE_PROTOBUF_PYTHON_CPP_SCALAR_MAP_CONTAINER_H__
+
+#include <Python.h>
+
+#include <memory>
+#ifndef _SHARED_PTR_H
+#include <google/protobuf/stubs/shared_ptr.h>
+#endif
+
+#include <google/protobuf/descriptor.h>
+
+namespace google {
+namespace protobuf {
+
+class Message;
+
+using internal::shared_ptr;
+
+namespace python {
+
+struct CMessage;
+
+struct ScalarMapContainer {
+  PyObject_HEAD;
+
+  // This is the top-level C++ Message object that owns the whole
+  // proto tree.  Every Python ScalarMapContainer holds a
+  // reference to it in order to keep it alive as long as there's a
+  // Python object that references any part of the tree.
+  shared_ptr<Message> owner;
+
+  // Pointer to the C++ Message that contains this container.  The
+  // ScalarMapContainer does not own this pointer.
+  Message* message;
+
+  // Weak reference to a parent CMessage object (i.e. may be NULL.)
+  //
+  // Used to make sure all ancestors are also mutable when first
+  // modifying the container.
+  CMessage* parent;
+
+  // Pointer to the parent's descriptor that describes this
+  // field.  Used together with the parent's message when making a
+  // default message instance mutable.
+  // The pointer is owned by the global DescriptorPool.
+  const FieldDescriptor* parent_field_descriptor;
+  const FieldDescriptor* key_field_descriptor;
+  const FieldDescriptor* value_field_descriptor;
+
+  // We bump this whenever we perform a mutation, to invalidate existing
+  // iterators.
+  uint64 version;
+};
+
+extern PyTypeObject ScalarMapContainer_Type;
+extern PyTypeObject ScalarMapIterator_Type;
+
+namespace scalar_map_container {
+
+// Builds a ScalarMapContainer object, from a parent message and a
+// field descriptor.
+extern PyObject *NewContainer(
+    CMessage* parent, const FieldDescriptor* parent_field_descriptor);
+
+// Releases the messages in the container to a new message.
+//
+// Returns 0 on success, -1 on failure.
+int Release(ScalarMapContainer* self);
+
+// Set the owner field of self and any children of self.
+void SetOwner(ScalarMapContainer* self,
+              const shared_ptr<Message>& new_owner);
+
+}  // namespace scalar_map_container
+}  // namespace python
+}  // namespace protobuf
+
+}  // namespace google
+#endif  // GOOGLE_PROTOBUF_PYTHON_CPP_SCALAR_MAP_CONTAINER_H__

python/google/protobuf/reflection.py

@@ -144,7 +144,6 @@ class GeneratedProtocolMessageType(type):
     _InitMessage(descriptor, cls)
     superclass = super(GeneratedProtocolMessageType, cls)
     superclass.__init__(name, bases, dictionary)
-    setattr(descriptor, '_concrete_class', cls)
 
 
 def ParseMessage(descriptor, byte_str):

python/google/protobuf/text_format.py

@@ -100,6 +100,10 @@ def MessageToString(message, as_utf8=False, as_one_line=False,
     return result.rstrip()
   return result
 
+def _IsMapEntry(field):
+  return (field.type == descriptor.FieldDescriptor.TYPE_MESSAGE and
+          field.message_type.has_options and
+          field.message_type.GetOptions().map_entry)
 
 def PrintMessage(message, out, indent=0, as_utf8=False, as_one_line=False,
                  pointy_brackets=False, use_index_order=False,
@@ -108,7 +112,19 @@ def PrintMessage(message, out, indent=0, as_utf8=False, as_one_line=False,
   if use_index_order:
     fields.sort(key=lambda x: x[0].index)
   for field, value in fields:
-    if field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
+    if _IsMapEntry(field):
+      for key in value:
+        # This is slow for maps with submessage entires because it copies the
+        # entire tree.  Unfortunately this would take significant refactoring
+        # of this file to work around.
+        #
+        # TODO(haberman): refactor and optimize if this becomes an issue.
+        entry_submsg = field.message_type._concrete_class(
+            key=key, value=value[key])
+        PrintField(field, entry_submsg, out, indent, as_utf8, as_one_line,
+                   pointy_brackets=pointy_brackets,
+                   use_index_order=use_index_order, float_format=float_format)
+    elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
       for element in value:
         PrintField(field, element, out, indent, as_utf8, as_one_line,
                    pointy_brackets=pointy_brackets,
@@ -367,6 +383,7 @@ def _MergeField(tokenizer, message, allow_multiple_scalars):
               message_descriptor.full_name, name))
 
   if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
+    is_map_entry = _IsMapEntry(field)
     tokenizer.TryConsume(':')
 
     if tokenizer.TryConsume('<'):
@@ -378,6 +395,8 @@ def _MergeField(tokenizer, message, allow_multiple_scalars):
     if field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
       if field.is_extension:
         sub_message = message.Extensions[field].add()
+      elif is_map_entry:
+        sub_message = field.message_type._concrete_class()
       else:
         sub_message = getattr(message, field.name).add()
     else:
@@ -391,6 +410,14 @@ def _MergeField(tokenizer, message, allow_multiple_scalars):
       if tokenizer.AtEnd():
         raise tokenizer.ParseErrorPreviousToken('Expected "%s".' % (end_token))
       _MergeField(tokenizer, sub_message, allow_multiple_scalars)
+
+    if is_map_entry:
+      value_cpptype = field.message_type.fields_by_name['value'].cpp_type
+      if value_cpptype == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
+        value = getattr(message, field.name)[sub_message.key]
+        value.MergeFrom(sub_message.value)
+      else:
+        getattr(message, field.name)[sub_message.key] = sub_message.value
   else:
     _MergeScalarField(tokenizer, message, field, allow_multiple_scalars)
 
@@ -701,13 +728,16 @@ class _Tokenizer(object):
     String literals (whether bytes or text) can come in multiple adjacent
     tokens which are automatically concatenated, like in C or Python.  This
     method only consumes one token.
+
+    Raises:
+      ParseError: When the wrong format data is found.
     """
     text = self.token
     if len(text) < 1 or text[0] not in ('\'', '"'):
-      raise self._ParseError('Expected string but found: "%r"' % text)
+      raise self._ParseError('Expected string but found: %r' % (text,))
 
     if len(text) < 2 or text[-1] != text[0]:
-      raise self._ParseError('String missing ending quote.')
+      raise self._ParseError('String missing ending quote: %r' % (text,))
 
     try:
       result = text_encoding.CUnescape(text[1:-1])

python/setup.py

@@ -91,6 +91,7 @@ def GenerateUnittestProtos():
   if not os.path.exists("../.git"):
     return
 
+  generate_proto("../src/google/protobuf/map_unittest.proto")
   generate_proto("../src/google/protobuf/unittest.proto")
   generate_proto("../src/google/protobuf/unittest_custom_options.proto")
   generate_proto("../src/google/protobuf/unittest_import.proto")

ruby/tests/generated_code.proto

@@ -3,17 +3,17 @@ syntax = "proto3";
 package A.B.C;
 
 message TestMessage {
-  optional int32 optional_int32 = 1;
-  optional int64 optional_int64 = 2;
-  optional uint32 optional_uint32 = 3;
-  optional uint64 optional_uint64 = 4;
-  optional bool optional_bool = 5;
-  optional double optional_double = 6;
-  optional float optional_float = 7;
-  optional string optional_string = 8;
-  optional bytes optional_bytes = 9;
-  optional TestEnum optional_enum = 10;
-  optional TestMessage optional_msg = 11;
+  int32 optional_int32 = 1;
+  int64 optional_int64 = 2;
+  uint32 optional_uint32 = 3;
+  uint64 optional_uint64 = 4;
+  bool optional_bool = 5;
+  double optional_double = 6;
+  float optional_float = 7;
+  string optional_string = 8;
+  bytes optional_bytes = 9;
+  TestEnum optional_enum = 10;
+  TestMessage optional_msg = 11;
 
   repeated int32 repeated_int32 = 21;
   repeated int64 repeated_int64 = 22;
@@ -53,10 +53,10 @@ message TestMessage {
   map<string, bool> map_string_bool = 70;
 
   message NestedMessage {
-    optional int32 foo = 1;
+    int32 foo = 1;
   }
 
-  optional NestedMessage nested_message = 80;
+  NestedMessage nested_message = 80;
 }
 
 enum TestEnum {

src/Makefile.am

@@ -88,6 +88,7 @@ nobase_include_HEADERS =                                        \
   google/protobuf/stubs/type_traits.h                           \
   google/protobuf/any.pb.h                                      \
   google/protobuf/api.pb.h                                      \
+  google/protobuf/any.h                                         \
   google/protobuf/arena.h                                       \
   google/protobuf/arenastring.h                                 \
   google/protobuf/descriptor_database.h                         \
@@ -186,6 +187,7 @@ libprotobuf_la_SOURCES =                                       \
   $(libprotobuf_lite_la_SOURCES)                               \
   google/protobuf/any.pb.cc                                    \
   google/protobuf/api.pb.cc                                    \
+  google/protobuf/any.cc                                       \
   google/protobuf/descriptor.cc                                \
   google/protobuf/descriptor_database.cc                       \
   google/protobuf/descriptor.pb.cc                             \
@@ -264,6 +266,8 @@ libprotoc_la_SOURCES =                                         \
   google/protobuf/compiler/java/java_enum.cc                   \
   google/protobuf/compiler/java/java_enum_field.cc             \
   google/protobuf/compiler/java/java_enum_field.h              \
+  google/protobuf/compiler/java/java_enum_field_lite.cc        \
+  google/protobuf/compiler/java/java_enum_field_lite.h         \
   google/protobuf/compiler/java/java_enum.h                    \
   google/protobuf/compiler/java/java_extension.cc              \
   google/protobuf/compiler/java/java_extension.h               \
@@ -278,22 +282,38 @@ libprotoc_la_SOURCES =                                         \
   google/protobuf/compiler/java/java_helpers.h                 \
   google/protobuf/compiler/java/java_lazy_message_field.cc     \
   google/protobuf/compiler/java/java_lazy_message_field.h      \
+  google/protobuf/compiler/java/java_lazy_message_field_lite.cc\
+  google/protobuf/compiler/java/java_lazy_message_field_lite.h \
   google/protobuf/compiler/java/java_map_field.cc              \
   google/protobuf/compiler/java/java_map_field.h               \
+  google/protobuf/compiler/java/java_map_field_lite.cc         \
+  google/protobuf/compiler/java/java_map_field_lite.h          \
   google/protobuf/compiler/java/java_message.cc                \
+  google/protobuf/compiler/java/java_message_lite.cc           \
+  google/protobuf/compiler/java/java_message_builder.cc        \
+  google/protobuf/compiler/java/java_message_builder_lite.cc   \
   google/protobuf/compiler/java/java_message_field.cc          \
   google/protobuf/compiler/java/java_message_field.h           \
+  google/protobuf/compiler/java/java_message_field_lite.cc     \
+  google/protobuf/compiler/java/java_message_field_lite.h      \
   google/protobuf/compiler/java/java_message.h                 \
+  google/protobuf/compiler/java/java_message_lite.h            \
+  google/protobuf/compiler/java/java_message_builder.h         \
+  google/protobuf/compiler/java/java_message_builder_lite.h    \
   google/protobuf/compiler/java/java_name_resolver.cc          \
   google/protobuf/compiler/java/java_name_resolver.h           \
   google/protobuf/compiler/java/java_primitive_field.cc        \
   google/protobuf/compiler/java/java_primitive_field.h         \
+  google/protobuf/compiler/java/java_primitive_field_lite.cc   \
+  google/protobuf/compiler/java/java_primitive_field_lite.h    \
   google/protobuf/compiler/java/java_shared_code_generator.cc  \
   google/protobuf/compiler/java/java_shared_code_generator.h   \
   google/protobuf/compiler/java/java_service.cc                \
   google/protobuf/compiler/java/java_service.h                 \
   google/protobuf/compiler/java/java_string_field.cc           \
   google/protobuf/compiler/java/java_string_field.h            \
+  google/protobuf/compiler/java/java_string_field_lite.cc      \
+  google/protobuf/compiler/java/java_string_field_lite.h       \
   google/protobuf/compiler/java/java_doc_comment.cc            \
   google/protobuf/compiler/java/java_doc_comment.h             \
   google/protobuf/compiler/javanano/javanano_enum.cc           \
@@ -381,6 +401,7 @@ protoc_SOURCES = google/protobuf/compiler/main.cc
 # Tests ==============================================================
 
 protoc_inputs =                                                \
+  google/protobuf/any_test.proto                               \
   google/protobuf/map_lite_unittest.proto                      \
   google/protobuf/map_proto2_unittest.proto                    \
   google/protobuf/map_unittest.proto                           \
@@ -419,6 +440,7 @@ EXTRA_DIST =                                                   \
   google/protobuf/testdata/golden_message_proto3               \
   google/protobuf/testdata/golden_packed_fields_message        \
   google/protobuf/testdata/bad_utf8_string                     \
+  google/protobuf/testdata/map_test_data.txt                   \
   google/protobuf/testdata/text_format_unittest_data.txt       \
   google/protobuf/testdata/text_format_unittest_data_oneof_implemented.txt  \
   google/protobuf/testdata/text_format_unittest_data_pointy.txt             \
@@ -443,6 +465,8 @@ protoc_lite_outputs =                                          \
 
 protoc_outputs =                                               \
   $(protoc_lite_outputs)                                       \
+  google/protobuf/any_test.pb.cc                               \
+  google/protobuf/any_test.pb.h                                \
   google/protobuf/map_proto2_unittest.pb.cc                    \
   google/protobuf/map_proto2_unittest.pb.h                     \
   google/protobuf/map_unittest.pb.cc                           \
@@ -543,6 +567,7 @@ protobuf_test_SOURCES =                                        \
   google/protobuf/stubs/stringprintf_unittest.cc               \
   google/protobuf/stubs/template_util_unittest.cc              \
   google/protobuf/stubs/type_traits_unittest.cc                \
+  google/protobuf/any_test.cc                                  \
   google/protobuf/arenastring_unittest.cc                      \
   google/protobuf/arena_unittest.cc                            \
   google/protobuf/descriptor_database_unittest.cc              \
@@ -604,6 +629,8 @@ nodist_protobuf_lazy_descriptor_test_SOURCES = $(protoc_outputs)
 protobuf_lite_test_LDADD = $(PTHREAD_LIBS) libprotobuf-lite.la
 protobuf_lite_test_CXXFLAGS = $(NO_OPT_CXXFLAGS)
 protobuf_lite_test_SOURCES =                                           \
+  google/protobuf/arena_test_util.cc                                   \
+  google/protobuf/arena_test_util.h                                    \
   google/protobuf/lite_unittest.cc                                     \
   google/protobuf/map_lite_test_util.cc                                \
   google/protobuf/map_lite_test_util.h                                 \

src/google/protobuf/any.cc

@@ -0,0 +1,100 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <google/protobuf/any.h>
+
+namespace google {
+namespace protobuf {
+namespace internal {
+
+namespace {
+string GetTypeUrl(const Descriptor* message) {
+  return string(kTypeGoogleApisComPrefix) + message->full_name();
+}
+
+}  // namespace
+
+const char kAnyFullTypeName[] = "google.protobuf.Any";
+const char kTypeGoogleApisComPrefix[] = "type.googleapis.com/";
+
+AnyMetadata::AnyMetadata(UrlType* type_url, ValueType* value)
+    : type_url_(type_url), value_(value) {
+}
+
+void AnyMetadata::PackFrom(const Message& message) {
+  type_url_->SetNoArena(&::google::protobuf::internal::GetEmptyString(),
+      GetTypeUrl(message.GetDescriptor()));
+  message.SerializeToString(value_->MutableNoArena(
+      &::google::protobuf::internal::GetEmptyStringAlreadyInited()));
+}
+
+bool AnyMetadata::UnpackTo(Message* message) const {
+  if (!InternalIs(message->GetDescriptor())) {
+    return false;
+  }
+  return message->ParseFromString(
+      value_->GetNoArena(&::google::protobuf::internal::GetEmptyString()));
+}
+
+bool AnyMetadata::InternalIs(const Descriptor* descriptor) const {
+  return type_url_->GetNoArena(
+             &::google::protobuf::internal::GetEmptyString()) ==
+         GetTypeUrl(descriptor);
+}
+
+bool ParseAnyTypeUrl(const string& type_url, string* full_type_name) {
+  const int prefix_len = strlen(kTypeGoogleApisComPrefix);
+  if (strncmp(type_url.c_str(), kTypeGoogleApisComPrefix, prefix_len) == 0) {
+    full_type_name->assign(type_url.data() + prefix_len,
+                           type_url.size() - prefix_len);
+    return true;
+  }
+  return true;
+}
+
+
+bool GetAnyFieldDescriptors(const Message& message,
+                            const FieldDescriptor** type_url_field,
+                            const FieldDescriptor** value_field) {
+    const Descriptor* descriptor = message.GetDescriptor();
+    if (descriptor->full_name() != kAnyFullTypeName) {
+      return false;
+    }
+    *type_url_field = descriptor->FindFieldByNumber(1);
+    *value_field = descriptor->FindFieldByNumber(2);
+    return (*type_url_field != NULL &&
+            (*type_url_field)->type() == FieldDescriptor::TYPE_STRING &&
+            *value_field != NULL &&
+            (*value_field)->type() == FieldDescriptor::TYPE_BYTES);
+}
+
+}  // namespace internal
+}  // namespace protobuf
+}  // namespace google

src/google/protobuf/any.h

@@ -0,0 +1,90 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef GOOGLE_PROTOBUF_ANY_H__
+#define GOOGLE_PROTOBUF_ANY_H__
+
+#include <string>
+
+#include <google/protobuf/stubs/common.h>
+#include <google/protobuf/arenastring.h>
+#include <google/protobuf/descriptor.h>
+#include <google/protobuf/message.h>
+
+namespace google {
+namespace protobuf {
+namespace internal {
+
+// Helper class used to implement google::protobuf::Any.
+class AnyMetadata {
+  typedef ArenaStringPtr UrlType;
+  typedef ArenaStringPtr ValueType;
+ public:
+  // AnyMetadata does not take ownership of "type_url" and "value".
+  AnyMetadata(UrlType* type_url, ValueType* value);
+
+  void PackFrom(const Message& message);
+
+  bool UnpackTo(Message* message) const;
+
+  template<typename T>
+  bool Is() const {
+    return InternalIs(T::default_instance().GetDescriptor());
+  }
+
+ private:
+  bool InternalIs(const Descriptor* message) const;
+
+  UrlType* type_url_;
+  ValueType* value_;
+
+  GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(AnyMetadata);
+};
+
+extern const char kAnyFullTypeName[];          // "google.protobuf.Any".
+extern const char kTypeGoogleApisComPrefix[];  // "type.googleapis.com/".
+
+// Get the proto type name from Any::type_url value. For example, passing
+// "type.googleapis.com/rpc.QueryOrigin" will return "rpc.QueryOrigin" in
+// *full_type_name. Returns false if type_url does not start with
+// "type.googleapis.com".
+bool ParseAnyTypeUrl(const string& type_url, string* full_type_name);
+
+// See if message is of type google.protobuf.Any, if so, return the descriptors
+// for "type_url" and "value" fields.
+bool GetAnyFieldDescriptors(const Message& message,
+                            const FieldDescriptor** type_url_field,
+                            const FieldDescriptor** value_field);
+
+}  // namespace internal
+}  // namespace protobuf
+
+}  // namespace google
+#endif  // GOOGLE_PROTOBUF_ANY_H__

src/google/protobuf/any.pb.cc

@@ -111,13 +111,21 @@ static void MergeFromFail(int line) {
 
 // ===================================================================
 
+void Any::PackFrom(const ::google::protobuf::Message& message) {
+  _any_metadata_.PackFrom(message);
+}
+
+bool Any::UnpackTo(::google::protobuf::Message* message) const {
+  return _any_metadata_.UnpackTo(message);
+}
+
 #ifndef _MSC_VER
 const int Any::kTypeUrlFieldNumber;
 const int Any::kValueFieldNumber;
 #endif  // !_MSC_VER
 
 Any::Any()
-  : ::google::protobuf::Message() , _internal_metadata_(NULL)  {
+  : ::google::protobuf::Message(), _internal_metadata_(NULL), _any_metadata_(&type_url_, &value_) {
   SharedCtor();
   // @@protoc_insertion_point(constructor:google.protobuf.Any)
 }
@@ -128,7 +136,8 @@ void Any::InitAsDefaultInstance() {
 
 Any::Any(const Any& from)
   : ::google::protobuf::Message(),
-    _internal_metadata_(NULL) {
+    _internal_metadata_(NULL),
+    _any_metadata_(&type_url_, &value_) {
   SharedCtor();
   MergeFrom(from);
   // @@protoc_insertion_point(copy_constructor:google.protobuf.Any)
@@ -316,9 +325,9 @@ int Any::ByteSize() const {
 
 void Any::MergeFrom(const ::google::protobuf::Message& from) {
   if (GOOGLE_PREDICT_FALSE(&from == this)) MergeFromFail(__LINE__);
-  const Any* source =
-    ::google::protobuf::internal::dynamic_cast_if_available<const Any*>(
-      &from);
+  const Any* source = 
+      ::google::protobuf::internal::DynamicCastToGenerated<const Any>(
+          &from);
   if (source == NULL) {
     ::google::protobuf::internal::ReflectionOps::Merge(from, this);
   } else {
@@ -378,7 +387,7 @@ void Any::InternalSwap(Any* other) {
 // Any
 
 // optional string type_url = 1;
- void Any::clear_type_url() {
+void Any::clear_type_url() {
   type_url_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
 }
  const ::std::string& Any::type_url() const {
@@ -421,7 +430,7 @@ void Any::InternalSwap(Any* other) {
 }
 
 // optional bytes value = 2;
- void Any::clear_value() {
+void Any::clear_value() {
   value_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
 }
  const ::std::string& Any::value() const {

src/google/protobuf/any.pb.h

@@ -27,6 +27,7 @@
 #include <google/protobuf/repeated_field.h>
 #include <google/protobuf/extension_set.h>
 #include <google/protobuf/unknown_field_set.h>
+#include "google/protobuf/any.h"
 // @@protoc_insertion_point(includes)
 
 namespace google {
@@ -56,6 +57,14 @@ class LIBPROTOBUF_EXPORT Any : public ::google::protobuf::Message {
   static const ::google::protobuf::Descriptor* descriptor();
   static const Any& default_instance();
 
+  // implements Any -----------------------------------------------
+
+  void PackFrom(const ::google::protobuf::Message& message);
+  bool UnpackTo(::google::protobuf::Message* message) const;
+  template<typename T> bool Is() const {
+    return _any_metadata_.Is<T>();
+  }
+
   void Swap(Any* other);
 
   // implements Message ----------------------------------------------
@@ -127,6 +136,7 @@ class LIBPROTOBUF_EXPORT Any : public ::google::protobuf::Message {
   ::google::protobuf::internal::ArenaStringPtr type_url_;
   ::google::protobuf::internal::ArenaStringPtr value_;
   mutable int _cached_size_;
+  ::google::protobuf::internal::AnyMetadata _any_metadata_;
   friend void LIBPROTOBUF_EXPORT protobuf_AddDesc_google_2fprotobuf_2fany_2eproto();
   friend void protobuf_AssignDesc_google_2fprotobuf_2fany_2eproto();
   friend void protobuf_ShutdownFile_google_2fprotobuf_2fany_2eproto();

src/google/protobuf/any_test.cc

@@ -0,0 +1,89 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <google/protobuf/any_test.pb.h>
+#include <gtest/gtest.h>
+
+namespace google {
+namespace protobuf {
+namespace {
+
+TEST(AnyTest, TestPackAndUnpack) {
+  protobuf_unittest::TestAny submessage;
+  submessage.set_int32_value(12345);
+  protobuf_unittest::TestAny message;
+  message.mutable_any_value()->PackFrom(submessage);
+
+  string data = message.SerializeAsString();
+
+  ASSERT_TRUE(message.ParseFromString(data));
+  EXPECT_TRUE(message.has_any_value());
+  ASSERT_TRUE(message.any_value().UnpackTo(&submessage));
+  EXPECT_EQ(12345, submessage.int32_value());
+}
+
+TEST(AnyTest, TestPackAndUnpackAny) {
+  // We can pack a Any message inside another Any message.
+  protobuf_unittest::TestAny submessage;
+  submessage.set_int32_value(12345);
+  google::protobuf::Any any;
+  any.PackFrom(submessage);
+  protobuf_unittest::TestAny message;
+  message.mutable_any_value()->PackFrom(any);
+
+  string data = message.SerializeAsString();
+
+  ASSERT_TRUE(message.ParseFromString(data));
+  EXPECT_TRUE(message.has_any_value());
+  ASSERT_TRUE(message.any_value().UnpackTo(&any));
+  ASSERT_TRUE(any.UnpackTo(&submessage));
+  EXPECT_EQ(12345, submessage.int32_value());
+}
+
+TEST(AnyTest, TestIs) {
+  protobuf_unittest::TestAny submessage;
+  submessage.set_int32_value(12345);
+  google::protobuf::Any any;
+  any.PackFrom(submessage);
+  ASSERT_TRUE(any.ParseFromString(any.SerializeAsString()));
+  EXPECT_TRUE(any.Is<protobuf_unittest::TestAny>());
+  EXPECT_FALSE(any.Is<google::protobuf::Any>());
+
+  protobuf_unittest::TestAny message;
+  message.mutable_any_value()->PackFrom(any);
+  ASSERT_TRUE(message.ParseFromString(message.SerializeAsString()));
+  EXPECT_FALSE(message.any_value().Is<protobuf_unittest::TestAny>());
+  EXPECT_TRUE(message.any_value().Is<google::protobuf::Any>());
+}
+
+}  // namespace
+}  // namespace protobuf
+
+}  // namespace google

src/google/protobuf/any_test.proto

@@ -0,0 +1,41 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+syntax = "proto3";
+
+package protobuf_unittest;
+
+import "google/protobuf/any.proto";
+
+message TestAny {
+  int32 int32_value = 1;
+  google.protobuf.Any any_value = 2;
+  repeated google.protobuf.Any repeated_any_value = 3;
+}

src/google/protobuf/api.pb.cc

@@ -162,7 +162,7 @@ const int Api::kSourceContextFieldNumber;
 #endif  // !_MSC_VER
 
 Api::Api()
-  : ::google::protobuf::Message() , _internal_metadata_(NULL)  {
+  : ::google::protobuf::Message(), _internal_metadata_(NULL) {
   SharedCtor();
   // @@protoc_insertion_point(constructor:google.protobuf.Api)
 }
@@ -230,7 +230,7 @@ Api* Api::New(::google::protobuf::Arena* arena) const {
 void Api::Clear() {
   name_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
   version_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-  if (source_context_ != NULL) delete source_context_;
+  if (GetArenaNoVirtual() == NULL && source_context_ != NULL) delete source_context_;
   source_context_ = NULL;
   methods_.Clear();
   options_.Clear();
@@ -266,26 +266,31 @@ bool Api::MergePartialFromCodedStream(
       case 2: {
         if (tag == 18) {
          parse_methods:
-          DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual(
+          DO_(input->IncrementRecursionDepth());
+         parse_loop_methods:
+          DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtualNoRecursionDepth(
                 input, add_methods()));
         } else {
           goto handle_unusual;
         }
-        if (input->ExpectTag(18)) goto parse_methods;
-        if (input->ExpectTag(26)) goto parse_options;
+        if (input->ExpectTag(18)) goto parse_loop_methods;
+        if (input->ExpectTag(26)) goto parse_loop_options;
+        input->UnsafeDecrementRecursionDepth();
         break;
       }
 
       // repeated .google.protobuf.Option options = 3;
       case 3: {
         if (tag == 26) {
-         parse_options:
-          DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual(
+          DO_(input->IncrementRecursionDepth());
+         parse_loop_options:
+          DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtualNoRecursionDepth(
                 input, add_options()));
         } else {
           goto handle_unusual;
         }
-        if (input->ExpectTag(26)) goto parse_options;
+        if (input->ExpectTag(26)) goto parse_loop_options;
+        input->UnsafeDecrementRecursionDepth();
         if (input->ExpectTag(34)) goto parse_version;
         break;
       }
@@ -483,9 +488,9 @@ int Api::ByteSize() const {
 
 void Api::MergeFrom(const ::google::protobuf::Message& from) {
   if (GOOGLE_PREDICT_FALSE(&from == this)) MergeFromFail(__LINE__);
-  const Api* source =
-    ::google::protobuf::internal::dynamic_cast_if_available<const Api*>(
-      &from);
+  const Api* source = 
+      ::google::protobuf::internal::DynamicCastToGenerated<const Api>(
+          &from);
   if (source == NULL) {
     ::google::protobuf::internal::ReflectionOps::Merge(from, this);
   } else {
@@ -553,7 +558,7 @@ void Api::InternalSwap(Api* other) {
 // Api
 
 // optional string name = 1;
- void Api::clear_name() {
+void Api::clear_name() {
   name_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
 }
  const ::std::string& Api::name() const {
@@ -596,10 +601,10 @@ void Api::InternalSwap(Api* other) {
 }
 
 // repeated .google.protobuf.Method methods = 2;
- int Api::methods_size() const {
+int Api::methods_size() const {
   return methods_.size();
 }
- void Api::clear_methods() {
+void Api::clear_methods() {
   methods_.Clear();
 }
  const ::google::protobuf::Method& Api::methods(int index) const {
@@ -626,10 +631,10 @@ Api::mutable_methods() {
 }
 
 // repeated .google.protobuf.Option options = 3;
- int Api::options_size() const {
+int Api::options_size() const {
   return options_.size();
 }
- void Api::clear_options() {
+void Api::clear_options() {
   options_.Clear();
 }
  const ::google::protobuf::Option& Api::options(int index) const {
@@ -656,7 +661,7 @@ Api::mutable_options() {
 }
 
 // optional string version = 4;
- void Api::clear_version() {
+void Api::clear_version() {
   version_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
 }
  const ::std::string& Api::version() const {
@@ -699,11 +704,11 @@ Api::mutable_options() {
 }
 
 // optional .google.protobuf.SourceContext source_context = 5;
- bool Api::has_source_context() const {
+bool Api::has_source_context() const {
   return !_is_default_instance_ && source_context_ != NULL;
 }
- void Api::clear_source_context() {
-  if (source_context_ != NULL) delete source_context_;
+void Api::clear_source_context() {
+  if (GetArenaNoVirtual() == NULL && source_context_ != NULL) delete source_context_;
   source_context_ = NULL;
 }
  const ::google::protobuf::SourceContext& Api::source_context() const {
@@ -749,7 +754,7 @@ const int Method::kOptionsFieldNumber;
 #endif  // !_MSC_VER
 
 Method::Method()
-  : ::google::protobuf::Message() , _internal_metadata_(NULL)  {
+  : ::google::protobuf::Message(), _internal_metadata_(NULL) {
   SharedCtor();
   // @@protoc_insertion_point(constructor:google.protobuf.Method)
 }
@@ -929,12 +934,15 @@ bool Method::MergePartialFromCodedStream(
       case 6: {
         if (tag == 50) {
          parse_options:
-          DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual(
+          DO_(input->IncrementRecursionDepth());
+         parse_loop_options:
+          DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtualNoRecursionDepth(
                 input, add_options()));
         } else {
           goto handle_unusual;
         }
-        if (input->ExpectTag(50)) goto parse_options;
+        if (input->ExpectTag(50)) goto parse_loop_options;
+        input->UnsafeDecrementRecursionDepth();
         if (input->ExpectAtEnd()) goto success;
         break;
       }
@@ -1119,9 +1127,9 @@ int Method::ByteSize() const {
 
 void Method::MergeFrom(const ::google::protobuf::Message& from) {
   if (GOOGLE_PREDICT_FALSE(&from == this)) MergeFromFail(__LINE__);
-  const Method* source =
-    ::google::protobuf::internal::dynamic_cast_if_available<const Method*>(
-      &from);
+  const Method* source = 
+      ::google::protobuf::internal::DynamicCastToGenerated<const Method>(
+          &from);
   if (source == NULL) {
     ::google::protobuf::internal::ReflectionOps::Merge(from, this);
   } else {
@@ -1196,7 +1204,7 @@ void Method::InternalSwap(Method* other) {
 // Method
 
 // optional string name = 1;
- void Method::clear_name() {
+void Method::clear_name() {
   name_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
 }
  const ::std::string& Method::name() const {
@@ -1239,7 +1247,7 @@ void Method::InternalSwap(Method* other) {
 }
 
 // optional string request_type_url = 2;
- void Method::clear_request_type_url() {
+void Method::clear_request_type_url() {
   request_type_url_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
 }
  const ::std::string& Method::request_type_url() const {
@@ -1282,7 +1290,7 @@ void Method::InternalSwap(Method* other) {
 }
 
 // optional bool request_streaming = 3;
- void Method::clear_request_streaming() {
+void Method::clear_request_streaming() {
   request_streaming_ = false;
 }
  bool Method::request_streaming() const {
@@ -1296,7 +1304,7 @@ void Method::InternalSwap(Method* other) {
 }
 
 // optional string response_type_url = 4;
- void Method::clear_response_type_url() {
+void Method::clear_response_type_url() {
   response_type_url_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
 }
  const ::std::string& Method::response_type_url() const {
@@ -1339,7 +1347,7 @@ void Method::InternalSwap(Method* other) {
 }
 
 // optional bool response_streaming = 5;
- void Method::clear_response_streaming() {
+void Method::clear_response_streaming() {
   response_streaming_ = false;
 }
  bool Method::response_streaming() const {
@@ -1353,10 +1361,10 @@ void Method::InternalSwap(Method* other) {
 }
 
 // repeated .google.protobuf.Option options = 6;
- int Method::options_size() const {
+int Method::options_size() const {
   return options_.size();
 }
- void Method::clear_options() {
+void Method::clear_options() {
   options_.Clear();
 }
  const ::google::protobuf::Option& Method::options(int index) const {

src/google/protobuf/api.pb.h

@@ -466,7 +466,7 @@ inline bool Api::has_source_context() const {
   return !_is_default_instance_ && source_context_ != NULL;
 }
 inline void Api::clear_source_context() {
-  if (source_context_ != NULL) delete source_context_;
+  if (GetArenaNoVirtual() == NULL && source_context_ != NULL) delete source_context_;
   source_context_ = NULL;
 }
 inline const ::google::protobuf::SourceContext& Api::source_context() const {
@@ -690,6 +690,8 @@ Method::mutable_options() {
 }
 
 #endif  // !PROTOBUF_INLINE_NOT_IN_HEADERS
+// -------------------------------------------------------------------
+
 
 // @@protoc_insertion_point(namespace_scope)
 

src/google/protobuf/arena.cc

@@ -29,7 +29,6 @@
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <google/protobuf/arena.h>
-#include <google/protobuf/stubs/common.h>
 
 #ifdef ADDRESS_SANITIZER
 #include <sanitizer/asan_interface.h>
@@ -155,10 +154,16 @@ void Arena::AddListNode(void* elem, void (*cleanup)(void*)) {
             reinterpret_cast<google::protobuf::internal::AtomicWord>(node)));
 }
 
-void* Arena::AllocateAligned(size_t n) {
+void* Arena::AllocateAligned(const std::type_info* allocated, size_t n) {
   // Align n to next multiple of 8 (from Hacker's Delight, Chapter 3.)
   n = (n + 7) & -8;
 
+  // Monitor allocation if needed.
+  if (GOOGLE_PREDICT_FALSE(hooks_cookie_ != NULL) &&
+      options_.on_arena_allocation != NULL) {
+    options_.on_arena_allocation(allocated, n, hooks_cookie_);
+  }
+
   // If this thread already owns a block in this arena then try to use that.
   // This fast path optimizes the case where multiple threads allocate from the
   // same arena.

src/google/protobuf/arena.h

@@ -28,12 +28,12 @@
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-// This header is logically internal, but is made public because it is used
-// from protocol-compiler-generated code, which may reside in other components.
-
 #ifndef GOOGLE_PROTOBUF_ARENA_H__
 #define GOOGLE_PROTOBUF_ARENA_H__
 
+#if __cplusplus >= 201103L
+#include <google/protobuf/stubs/type_traits.h>
+#endif
 #include <typeinfo>
 
 #include <google/protobuf/stubs/atomic_sequence_num.h>
@@ -113,10 +113,13 @@ struct ArenaOptions {
   void (*on_arena_reset)(Arena* arena, void* cookie, uint64 space_used);
   void (*on_arena_destruction)(Arena* arena, void* cookie, uint64 space_used);
 
-  // type_name is promised to be a static string - its lifetime extends to
-  // match program's lifetime.
-  void (*on_arena_allocation)(const char* type_name, uint64 alloc_size,
-      Arena* arena, void* cookie);
+  // type_info is promised to be static - its lifetime extends to
+  // match program's lifetime (It is given by typeid operator).
+  // Note: typeid(void) will be passed as allocated_type every time we
+  // intentionally want to avoid monitoring an allocation. (i.e. internal
+  // allocations for managing the arena)
+  void (*on_arena_allocation)(const std::type_info* allocated_type,
+      uint64 alloc_size, void* cookie);
 
   ArenaOptions()
       : start_block_size(kDefaultStartBlockSize),
@@ -137,6 +140,14 @@ struct ArenaOptions {
   static const size_t kDefaultMaxBlockSize   = 8192;
 };
 
+// Support for non-RTTI environments. (The metrics hooks API uses type
+// information.)
+#ifndef GOOGLE_PROTOBUF_NO_RTTI
+#define RTTI_TYPE_ID(type) (&typeid(type))
+#else
+#define RTTI_TYPE_ID(type) (NULL)
+#endif
+
 // Arena allocator. Arena allocation replaces ordinary (heap-based) allocation
 // with new/delete, and improves performance by aggregating allocations into
 // larger blocks and freeing allocations all at once. Protocol messages are
@@ -146,6 +157,44 @@ struct ArenaOptions {
 // This is a thread-safe implementation: multiple threads may allocate from the
 // arena concurrently. Destruction is not thread-safe and the destructing
 // thread must synchronize with users of the arena first.
+//
+// An arena provides two allocation interfaces: CreateMessage<T>, which works
+// for arena-enabled proto2 message types as well as other types that satisfy
+// the appropriate protocol (described below), and Create<T>, which works for
+// any arbitrary type T. CreateMessage<T> is better when the type T supports it,
+// because this interface (i) passes the arena pointer to the created object so
+// that its sub-objects and internal allocations can use the arena too, and (ii)
+// elides the object's destructor call when possible. Create<T> does not place
+// any special requirements on the type T, and will invoke the object's
+// destructor when the arena is destroyed.
+//
+// The arena message allocation protocol, required by CreateMessage<T>, is as
+// follows:
+//
+// - The type T must have (at least) two constructors: a constructor with no
+//   arguments, called when a T is allocated on the heap; and a constructor with
+//   a google::protobuf::Arena* argument, called when a T is allocated on an arena. If the
+//   second constructor is called with a NULL arena pointer, it must be
+//   equivalent to invoking the first (no-argument) constructor.
+//
+// - The type T must have a particular type trait: a nested type
+//   |InternalArenaConstructable_|. This is usually a typedef to |void|. If no
+//   such type trait exists, then the instantiation CreateMessage<T> will fail
+//   to compile.
+//
+// - The type T *may* have the type trait |DestructorSkippable_|. If this type
+//   trait is present in the type, then its destructor will not be called if and
+//   only if it was passed a non-NULL arena pointer. If this type trait is not
+//   present on the type, then its destructor is always called when the
+//   containing arena is destroyed.
+//
+// - One- and two-user-argument forms of CreateMessage<T>() also exist that
+//   forward these constructor arguments to T's constructor: for example,
+//   CreateMessage<T>(Arena*, arg1, arg2) forwards to a constructor T(Arena*,
+//   arg1, arg2).
+//
+// This protocol is implemented by all arena-enabled proto2 message classes as
+// well as RepeatedPtrField.
 class LIBPROTOBUF_EXPORT Arena {
  public:
   // Arena constructor taking custom options. See ArenaOptions below for
@@ -172,8 +221,10 @@ class LIBPROTOBUF_EXPORT Arena {
   // compilation error will occur.
   //
   // RepeatedField and RepeatedPtrField may also be instantiated directly on an
-  // arena with this method: they act as "arena-capable message types" for the
-  // purposes of the Arena API.
+  // arena with this method.
+  //
+  // This function also accepts any type T that satisfies the arena message
+  // allocation protocol, documented above.
   template <typename T> GOOGLE_ATTRIBUTE_ALWAYS_INLINE
   static T* CreateMessage(::google::protobuf::Arena* arena) {
     if (arena == NULL) {
@@ -183,17 +234,55 @@ class LIBPROTOBUF_EXPORT Arena {
     }
   }
 
+  // One-argument form of CreateMessage. This is useful for constructing objects
+  // that implement the arena message construction protocol described above but
+  // take additional constructor arguments.
+  template <typename T, typename Arg> GOOGLE_ATTRIBUTE_ALWAYS_INLINE
+  static T* CreateMessage(::google::protobuf::Arena* arena, const Arg& arg) {
+    if (arena == NULL) {
+      return new T(NULL, arg);
+    } else {
+      return arena->CreateMessageInternal<T>(static_cast<T*>(0),
+                                             arg);
+    }
+  }
+
+  // Two-argument form of CreateMessage. This is useful for constructing objects
+  // that implement the arena message construction protocol described above but
+  // take additional constructor arguments.
+  template <typename T, typename Arg1, typename Arg2> GOOGLE_ATTRIBUTE_ALWAYS_INLINE
+  static T* CreateMessage(::google::protobuf::Arena* arena,
+                          const Arg1& arg1,
+                          const Arg2& arg2) {
+    if (arena == NULL) {
+      return new T(NULL, arg1, arg2);
+    } else {
+      return arena->CreateMessageInternal<T>(static_cast<T*>(0),
+                                             arg1, arg2);
+    }
+  }
+
   // API to create any objects on the arena. Note that only the object will
   // be created on the arena; the underlying ptrs (in case of a proto2 message)
   // will be still heap allocated. Proto messages should usually be allocated
   // with CreateMessage<T>() instead.
+  //
+  // Note that even if T satisfies the arena message construction protocol
+  // (InternalArenaConstructable_ trait and optional DestructorSkippable_
+  // trait), as described above, this function does not follow the protocol;
+  // instead, it treats T as a black-box type, just as if it did not have these
+  // traits. Specifically, T's constructor arguments will always be only those
+  // passed to Create<T>() -- no additional arena pointer is implicitly added.
+  // Furthermore, the destructor will always be called at arena destruction time
+  // (unless the destructor is trivial). Hence, from T's point of view, it is as
+  // if the object were allocated on the heap (except that the underlying memory
+  // is obtained from the arena).
   template <typename T> GOOGLE_ATTRIBUTE_ALWAYS_INLINE
   static T* Create(::google::protobuf::Arena* arena) {
     if (arena == NULL) {
       return new T();
     } else {
-      return arena->CreateInternal<T>(
-          SkipDeleteList<T>(static_cast<T*>(0)));
+      return arena->CreateInternal<T>(google::protobuf::internal::has_trivial_destructor<T>::value);
     }
   }
 
@@ -203,7 +292,7 @@ class LIBPROTOBUF_EXPORT Arena {
     if (arena == NULL) {
       return new T(arg);
     } else {
-      return arena->CreateInternal<T>(SkipDeleteList<T>(static_cast<T*>(0)),
+      return arena->CreateInternal<T>(google::protobuf::internal::has_trivial_destructor<T>::value,
                                       arg);
     }
   }
@@ -214,9 +303,8 @@ class LIBPROTOBUF_EXPORT Arena {
     if (arena == NULL) {
       return new T(arg1, arg2);
     } else {
-      return arena->CreateInternal<T>(SkipDeleteList<T>(static_cast<T*>(0)),
-                                      arg1,
-                                      arg2);
+      return arena->CreateInternal<T>(google::protobuf::internal::has_trivial_destructor<T>::value,
+                                      arg1, arg2);
     }
   }
 
@@ -229,10 +317,8 @@ class LIBPROTOBUF_EXPORT Arena {
     if (arena == NULL) {
       return new T(arg1, arg2, arg3);
     } else {
-      return arena->CreateInternal<T>(SkipDeleteList<T>(static_cast<T*>(0)),
-                                      arg1,
-                                      arg2,
-                                      arg3);
+      return arena->CreateInternal<T>(google::protobuf::internal::has_trivial_destructor<T>::value,
+                                      arg1, arg2, arg3);
     }
   }
 
@@ -246,20 +332,95 @@ class LIBPROTOBUF_EXPORT Arena {
     if (arena == NULL) {
       return new T(arg1, arg2, arg3, arg4);
     } else {
-      return arena->CreateInternal<T>(SkipDeleteList<T>(static_cast<T*>(0)),
-                                      arg1,
-                                      arg2,
-                                      arg3,
-                                      arg4);
+      return arena->CreateInternal<T>(google::protobuf::internal::has_trivial_destructor<T>::value,
+                                      arg1, arg2, arg3, arg4);
     }
   }
 
-  // Create an array of object type T on the arena. Type T must have a trivial
-  // constructor, as it will not be invoked when created on the arena.
+  // Version of the above with five constructor arguments for the created
+  // object.
+  template <typename T, typename Arg1, typename Arg2, typename Arg3,
+            typename Arg4, typename Arg5>
+  GOOGLE_ATTRIBUTE_ALWAYS_INLINE static T* Create(::google::protobuf::Arena* arena,
+                                           const Arg1& arg1, const Arg2& arg2,
+                                           const Arg3& arg3, const Arg4& arg4,
+                                           const Arg5& arg5) {
+    if (arena == NULL) {
+      return new T(arg1, arg2, arg3, arg4, arg5);
+    } else {
+      return arena->CreateInternal<T>(google::protobuf::internal::has_trivial_destructor<T>::value,
+                                      arg1, arg2, arg3, arg4, arg5);
+    }
+  }
+
+  // Version of the above with six constructor arguments for the created
+  // object.
+  template <typename T, typename Arg1, typename Arg2, typename Arg3,
+            typename Arg4, typename Arg5, typename Arg6>
+  GOOGLE_ATTRIBUTE_ALWAYS_INLINE static T* Create(::google::protobuf::Arena* arena,
+                                           const Arg1& arg1, const Arg2& arg2,
+                                           const Arg3& arg3, const Arg4& arg4,
+                                           const Arg5& arg5, const Arg6& arg6) {
+    if (arena == NULL) {
+      return new T(arg1, arg2, arg3, arg4, arg5, arg6);
+    } else {
+      return arena->CreateInternal<T>(google::protobuf::internal::has_trivial_destructor<T>::value,
+                                      arg1, arg2, arg3, arg4, arg5, arg6);
+    }
+  }
+
+  // Version of the above with seven constructor arguments for the created
+  // object.
+  template <typename T, typename Arg1, typename Arg2, typename Arg3,
+            typename Arg4, typename Arg5, typename Arg6, typename Arg7>
+  GOOGLE_ATTRIBUTE_ALWAYS_INLINE static T* Create(::google::protobuf::Arena* arena,
+                                           const Arg1& arg1, const Arg2& arg2,
+                                           const Arg3& arg3, const Arg4& arg4,
+                                           const Arg5& arg5, const Arg6& arg6,
+                                           const Arg7& arg7) {
+    if (arena == NULL) {
+      return new T(arg1, arg2, arg3, arg4, arg5, arg6, arg7);
+    } else {
+      return arena->CreateInternal<T>(google::protobuf::internal::has_trivial_destructor<T>::value,
+                                      arg1, arg2, arg3, arg4, arg5, arg6, arg7);
+    }
+  }
+
+  // Version of the above with eight constructor arguments for the created
+  // object.
+  template <typename T, typename Arg1, typename Arg2, typename Arg3,
+            typename Arg4, typename Arg5, typename Arg6, typename Arg7,
+            typename Arg8>
+  GOOGLE_ATTRIBUTE_ALWAYS_INLINE static T* Create(::google::protobuf::Arena* arena,
+                                           const Arg1& arg1, const Arg2& arg2,
+                                           const Arg3& arg3, const Arg4& arg4,
+                                           const Arg5& arg5, const Arg6& arg6,
+                                           const Arg7& arg7, const Arg8& arg8) {
+    if (arena == NULL) {
+      return new T(arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8);
+    } else {
+      return arena->CreateInternal<T>(
+          google::protobuf::internal::has_trivial_destructor<T>::value,
+          arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8);
+    }
+  }
+
+  // Create an array of object type T on the arena *without* invoking the
+  // constructor of T. If `arena` is null, then the return value should be freed
+  // with `delete[] x;` (or `::operator delete[](x);`).
+  // To ensure safe uses, this function checks at compile time
+  // (when compiled as C++11) that T is trivially default-constructible and
+  // trivially destructible.
   template <typename T> GOOGLE_ATTRIBUTE_ALWAYS_INLINE
   static T* CreateArray(::google::protobuf::Arena* arena, size_t num_elements) {
+#if __cplusplus >= 201103L
+    static_assert(std::is_trivially_default_constructible<T>::value,
+                  "CreateArray requires a trivially constructible type");
+    static_assert(std::is_trivially_destructible<T>::value,
+                  "CreateArray requires a trivially destructible type");
+#endif
     if (arena == NULL) {
-      return new T[num_elements];
+      return static_cast<T*>(::operator new[](num_elements * sizeof(T)));
     } else {
       return arena->CreateInternalRawArray<T>(num_elements);
     }
@@ -374,27 +535,26 @@ class LIBPROTOBUF_EXPORT Arena {
   // wrap them in static functions.
   static ThreadCache& thread_cache();
 #elif defined(GOOGLE_PROTOBUF_OS_ANDROID) || defined(GOOGLE_PROTOBUF_OS_IPHONE)
-  // Android ndk does not support __thread keyword so we use a custom thread
+  // Android ndk does not support GOOGLE_THREAD_LOCAL keyword so we use a custom thread
   // local storage class we implemented.
-  // iOS also does not support the __thread keyword.
+  // iOS also does not support the GOOGLE_THREAD_LOCAL keyword.
   static ThreadCache& thread_cache();
 #else
   static GOOGLE_THREAD_LOCAL ThreadCache thread_cache_;
   static ThreadCache& thread_cache() { return thread_cache_; }
 #endif
 
-  // SFINAE for skipping addition to delete list for a Type. This is mainly to
-  // skip proto2/proto1 message objects with cc_enable_arenas=true from being
-  // part of the delete list. Also, note, compiler will optimize out the branch
-  // in CreateInternal<T>.
-  //
+  // SFINAE for skipping addition to delete list for a message type when created
+  // with CreateMessage. This is mainly to skip proto2/proto1 message objects
+  // with cc_enable_arenas=true from being part of the delete list. Also, note,
+  // compiler will optimize out the branch in CreateInternal<T>.
   template<typename T>
   static inline bool SkipDeleteList(typename T::DestructorSkippable_*) {
     return true;
   }
 
-  // For non message objects, we skip addition to delete list if the object has
-  // a trivial destructor.
+  // For message objects that don't have the DestructorSkippable_ trait, we
+  // always add to the delete list.
   template<typename T>
   static inline bool SkipDeleteList(...) {
     return google::protobuf::internal::has_trivial_destructor<T>::value;
@@ -419,14 +579,15 @@ class LIBPROTOBUF_EXPORT Arena {
   // Just allocate the required size for the given type assuming the
   // type has a trivial constructor.
   template<typename T> GOOGLE_ATTRIBUTE_ALWAYS_INLINE
-  inline T* CreateInternalRawArray(uint32 num_elements) {
-    return static_cast<T*>(AllocateAligned(sizeof(T) * num_elements));
+  inline T* CreateInternalRawArray(size_t num_elements) {
+    return static_cast<T*>(
+        AllocateAligned(RTTI_TYPE_ID(T), sizeof(T) * num_elements));
   }
 
   template <typename T> GOOGLE_ATTRIBUTE_ALWAYS_INLINE
   inline T* CreateInternal(
       bool skip_explicit_ownership) {
-    T* t = new (AllocateAligned(sizeof(T))) T();
+    T* t = new (AllocateAligned(RTTI_TYPE_ID(T), sizeof(T))) T();
     if (!skip_explicit_ownership) {
       AddListNode(t, &internal::arena_destruct_object<T>);
     }
@@ -436,7 +597,7 @@ class LIBPROTOBUF_EXPORT Arena {
   template <typename T, typename Arg> GOOGLE_ATTRIBUTE_ALWAYS_INLINE
   inline T* CreateInternal(
       bool skip_explicit_ownership, const Arg& arg) {
-    T* t = new (AllocateAligned(sizeof(T))) T(arg);
+    T* t = new (AllocateAligned(RTTI_TYPE_ID(T), sizeof(T))) T(arg);
     if (!skip_explicit_ownership) {
       AddListNode(t, &internal::arena_destruct_object<T>);
     }
@@ -446,7 +607,7 @@ class LIBPROTOBUF_EXPORT Arena {
   template <typename T, typename Arg1, typename Arg2> GOOGLE_ATTRIBUTE_ALWAYS_INLINE
   inline T* CreateInternal(
       bool skip_explicit_ownership, const Arg1& arg1, const Arg2& arg2) {
-    T* t = new (AllocateAligned(sizeof(T))) T(arg1, arg2);
+    T* t = new (AllocateAligned(RTTI_TYPE_ID(T), sizeof(T))) T(arg1, arg2);
     if (!skip_explicit_ownership) {
       AddListNode(t, &internal::arena_destruct_object<T>);
     }
@@ -458,7 +619,8 @@ class LIBPROTOBUF_EXPORT Arena {
                                                    const Arg1& arg1,
                                                    const Arg2& arg2,
                                                    const Arg3& arg3) {
-    T* t = new (AllocateAligned(sizeof(T))) T(arg1, arg2, arg3);
+    T* t = new (AllocateAligned(RTTI_TYPE_ID(T), sizeof(T)))
+        T(arg1, arg2, arg3);
     if (!skip_explicit_ownership) {
       AddListNode(t, &internal::arena_destruct_object<T>);
     }
@@ -472,7 +634,79 @@ class LIBPROTOBUF_EXPORT Arena {
                                                    const Arg2& arg2,
                                                    const Arg3& arg3,
                                                    const Arg4& arg4) {
-    T* t = new (AllocateAligned(sizeof(T))) T(arg1, arg2, arg3, arg4);
+    T* t = new (AllocateAligned(RTTI_TYPE_ID(T), sizeof(T)))
+        T(arg1, arg2, arg3, arg4);
+    if (!skip_explicit_ownership) {
+      AddListNode(t, &internal::arena_destruct_object<T>);
+    }
+    return t;
+  }
+
+  template <typename T, typename Arg1, typename Arg2, typename Arg3,
+            typename Arg4, typename Arg5>
+  GOOGLE_ATTRIBUTE_ALWAYS_INLINE inline T* CreateInternal(bool skip_explicit_ownership,
+                                                   const Arg1& arg1,
+                                                   const Arg2& arg2,
+                                                   const Arg3& arg3,
+                                                   const Arg4& arg4,
+                                                   const Arg5& arg5) {
+    T* t = new (AllocateAligned(RTTI_TYPE_ID(T), sizeof(T)))
+        T(arg1, arg2, arg3, arg4, arg5);
+    if (!skip_explicit_ownership) {
+      AddListNode(t, &internal::arena_destruct_object<T>);
+    }
+    return t;
+  }
+
+  template <typename T, typename Arg1, typename Arg2, typename Arg3,
+            typename Arg4, typename Arg5, typename Arg6>
+  GOOGLE_ATTRIBUTE_ALWAYS_INLINE inline T* CreateInternal(bool skip_explicit_ownership,
+                                                   const Arg1& arg1,
+                                                   const Arg2& arg2,
+                                                   const Arg3& arg3,
+                                                   const Arg4& arg4,
+                                                   const Arg5& arg5,
+                                                   const Arg6& arg6) {
+    T* t = new (AllocateAligned(RTTI_TYPE_ID(T), sizeof(T)))
+        T(arg1, arg2, arg3, arg4, arg5, arg6);
+    if (!skip_explicit_ownership) {
+      AddListNode(t, &internal::arena_destruct_object<T>);
+    }
+    return t;
+  }
+
+  template <typename T, typename Arg1, typename Arg2, typename Arg3,
+            typename Arg4, typename Arg5, typename Arg6, typename Arg7>
+  GOOGLE_ATTRIBUTE_ALWAYS_INLINE inline T* CreateInternal(bool skip_explicit_ownership,
+                                                   const Arg1& arg1,
+                                                   const Arg2& arg2,
+                                                   const Arg3& arg3,
+                                                   const Arg4& arg4,
+                                                   const Arg5& arg5,
+                                                   const Arg6& arg6,
+                                                   const Arg7& arg7) {
+    T* t = new (AllocateAligned(RTTI_TYPE_ID(T), sizeof(T)))
+        T(arg1, arg2, arg3, arg4, arg5, arg6, arg7);
+    if (!skip_explicit_ownership) {
+      AddListNode(t, &internal::arena_destruct_object<T>);
+    }
+    return t;
+  }
+
+  template <typename T, typename Arg1, typename Arg2, typename Arg3,
+            typename Arg4, typename Arg5, typename Arg6, typename Arg7,
+            typename Arg8>
+  GOOGLE_ATTRIBUTE_ALWAYS_INLINE inline T* CreateInternal(bool skip_explicit_ownership,
+                                                   const Arg1& arg1,
+                                                   const Arg2& arg2,
+                                                   const Arg3& arg3,
+                                                   const Arg4& arg4,
+                                                   const Arg5& arg5,
+                                                   const Arg6& arg6,
+                                                   const Arg7& arg7,
+                                                   const Arg8& arg8) {
+    T* t = new (AllocateAligned(RTTI_TYPE_ID(T), sizeof(T)))
+        T(arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8);
     if (!skip_explicit_ownership) {
       AddListNode(t, &internal::arena_destruct_object<T>);
     }
@@ -485,6 +719,20 @@ class LIBPROTOBUF_EXPORT Arena {
                                      this);
   }
 
+  template <typename T, typename Arg> GOOGLE_ATTRIBUTE_ALWAYS_INLINE
+  inline T* CreateMessageInternal(typename T::InternalArenaConstructable_*,
+                                  const Arg& arg) {
+    return CreateInternal<T, Arena*>(SkipDeleteList<T>(static_cast<T*>(0)),
+                                     this, arg);
+  }
+
+  template <typename T, typename Arg1, typename Arg2> GOOGLE_ATTRIBUTE_ALWAYS_INLINE
+  inline T* CreateMessageInternal(typename T::InternalArenaConstructable_*,
+                                  const Arg1& arg1, const Arg2& arg2) {
+    return CreateInternal<T, Arena*>(SkipDeleteList<T>(static_cast<T*>(0)),
+                                     this, arg1, arg2);
+  }
+
   // CreateInArenaStorage is used to implement map field. Without it,
   // google::protobuf::Map need to call generated message's protected arena constructor,
   // which needs to declare google::protobuf::Map as friend of generated message.
@@ -536,8 +784,15 @@ class LIBPROTOBUF_EXPORT Arena {
     return NULL;
   }
 
+  // Allocate and also optionally call on_arena_allocation callback with the
+  // allocated type info when the hooks are in place in ArenaOptions and
+  // the cookie is not null.
+  void* AllocateAligned(const std::type_info* allocated, size_t n);
 
-  void* AllocateAligned(size_t size);
+  // Allocate an internal allocation, avoiding optional typed monitoring.
+  GOOGLE_ATTRIBUTE_ALWAYS_INLINE inline void* AllocateAligned(size_t n) {
+    return AllocateAligned(NULL, n);
+  }
 
   void Init();
 
@@ -596,6 +851,9 @@ class LIBPROTOBUF_EXPORT Arena {
   GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(Arena);
 };
 
+// Defined above for supporting environments without RTTI.
+#undef RTTI_TYPE_ID
+
 template<typename T>
 const typename Arena::is_arena_constructable<T>::type
     Arena::is_arena_constructable<T>::value =

src/google/protobuf/arena_nc_test.py

@@ -35,6 +35,7 @@
 import unittest
 
 from google3.testing.pybase import fake_target_util
+import unittest
 
 
 class ArenaNcTest(unittest.TestCase):

src/google/protobuf/arena_test_util.h

@@ -31,6 +31,7 @@
 #ifndef GOOGLE_PROTOBUF_ARENA_TEST_UTIL_H__
 #define GOOGLE_PROTOBUF_ARENA_TEST_UTIL_H__
 
+
 namespace google {
 namespace protobuf {
 namespace internal {

src/google/protobuf/arena_unittest.cc

@@ -39,6 +39,7 @@
 #include <google/protobuf/stubs/shared_ptr.h>
 #endif
 #include <string>
+#include <typeinfo>
 #include <vector>
 
 #include <google/protobuf/stubs/common.h>
@@ -47,11 +48,14 @@
 #include <google/protobuf/unittest.pb.h>
 #include <google/protobuf/unittest_arena.pb.h>
 #include <google/protobuf/unittest_no_arena.pb.h>
+#include <google/protobuf/io/coded_stream.h>
+#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
 #include <google/protobuf/descriptor.h>
 #include <google/protobuf/extension_set.h>
 #include <google/protobuf/message.h>
 #include <google/protobuf/message_lite.h>
 #include <google/protobuf/repeated_field.h>
+#include <google/protobuf/wire_format_lite.h>
 #include <google/protobuf/unknown_field_set.h>
 #include <gtest/gtest.h>
 
@@ -125,6 +129,29 @@ class MustBeConstructedWithOneThroughFour {
   GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(MustBeConstructedWithOneThroughFour);
 };
 
+// A class that takes eight different types as constructor arguments.
+class MustBeConstructedWithOneThroughEight {
+ public:
+  MustBeConstructedWithOneThroughEight(
+      int one, const char* two, const string& three,
+      const PleaseDontCopyMe* four, int five, const char* six,
+      const string& seven, const string& eight)
+      : one_(one), two_(two), three_(three), four_(four), five_(five),
+        six_(six), seven_(seven), eight_(eight) {}
+
+  int one_;
+  const char* const two_;
+  string three_;
+  const PleaseDontCopyMe* four_;
+  int five_;
+  const char* const six_;
+  string seven_;
+  string eight_;
+
+ private:
+  GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(MustBeConstructedWithOneThroughEight);
+};
+
 }  // namespace
 
 TEST(ArenaTest, ArenaConstructable) {
@@ -156,7 +183,7 @@ TEST(ArenaTest, BasicCreate) {
   EXPECT_EQ(2, notifier.GetCount());
 }
 
-TEST(ArenaTest, CreateWithManyConstructorArguments) {
+TEST(ArenaTest, CreateWithFourConstructorArguments) {
   Arena arena;
   const string three("3");
   const PleaseDontCopyMe four(4);
@@ -170,6 +197,26 @@ TEST(ArenaTest, CreateWithManyConstructorArguments) {
   ASSERT_EQ(4, new_object->four_->value());
 }
 
+TEST(ArenaTest, CreateWithEightConstructorArguments) {
+  Arena arena;
+  const string three("3");
+  const PleaseDontCopyMe four(4);
+  const string seven("7");
+  const string eight("8");
+  const MustBeConstructedWithOneThroughEight* new_object =
+      Arena::Create<MustBeConstructedWithOneThroughEight>(
+          &arena, 1, "2", three, &four, 5, "6", seven, eight);
+  EXPECT_TRUE(new_object != NULL);
+  ASSERT_EQ(1, new_object->one_);
+  ASSERT_STREQ("2", new_object->two_);
+  ASSERT_EQ("3", new_object->three_);
+  ASSERT_EQ(4, new_object->four_->value());
+  ASSERT_EQ(5, new_object->five_);
+  ASSERT_STREQ("6", new_object->six_);
+  ASSERT_EQ("7", new_object->seven_);
+  ASSERT_EQ("8", new_object->eight_);
+}
+
 TEST(ArenaTest, InitialBlockTooSmall) {
   // Construct a small (64 byte) initial block of memory to be used by the
   // arena allocator; then, allocate an object which will not fit in the
@@ -1113,6 +1160,19 @@ TEST(ArenaTest, GetArenaShouldReturnNullForNonArenaAllocatedMessages) {
   EXPECT_EQ(NULL, Arena::GetArena(const_pointer_to_message));
 }
 
+TEST(ArenaTest, UnsafeSetAllocatedOnArena) {
+  ::google::protobuf::Arena arena;
+  TestAllTypes* message = Arena::CreateMessage<TestAllTypes>(&arena);
+  EXPECT_FALSE(message->has_optional_string());
+
+  string owned_string = "test with long enough content to heap-allocate";
+  message->unsafe_arena_set_allocated_optional_string(&owned_string);
+  EXPECT_TRUE(message->has_optional_string());
+
+  message->unsafe_arena_set_allocated_optional_string(NULL);
+  EXPECT_FALSE(message->has_optional_string());
+}
+
 // A helper utility class to only contain static hook functions, some
 // counters to be used to verify the counters have been called and a cookie
 // value to be verified.
@@ -1124,6 +1184,13 @@ class ArenaHooksTestUtil {
     return static_cast<void*>(cookie);
   }
 
+  static void on_allocation(const std::type_info* /*unused*/, uint64 alloc_size,
+                            void* cookie) {
+    ++num_allocations;
+    int cookie_value = *static_cast<int*>(cookie);
+    EXPECT_EQ(kCookieValue, cookie_value);
+  }
+
   static void on_reset(::google::protobuf::Arena* arena, void* cookie,
                        uint64 space_used) {
     ++num_reset;
@@ -1141,10 +1208,12 @@ class ArenaHooksTestUtil {
 
   static const int kCookieValue = 999;
   static uint32 num_init;
+  static uint32 num_allocations;
   static uint32 num_reset;
   static uint32 num_destruct;
 };
 uint32 ArenaHooksTestUtil::num_init = 0;
+uint32 ArenaHooksTestUtil::num_allocations = 0;
 uint32 ArenaHooksTestUtil::num_reset = 0;
 uint32 ArenaHooksTestUtil::num_destruct = 0;
 const int ArenaHooksTestUtil::kCookieValue;
@@ -1153,6 +1222,7 @@ const int ArenaHooksTestUtil::kCookieValue;
 TEST(ArenaTest, ArenaHooksSanity) {
   ::google::protobuf::ArenaOptions options;
   options.on_arena_init = ArenaHooksTestUtil::on_init;
+  options.on_arena_allocation = ArenaHooksTestUtil::on_allocation;
   options.on_arena_reset = ArenaHooksTestUtil::on_reset;
   options.on_arena_destruction = ArenaHooksTestUtil::on_destruction;
 
@@ -1160,6 +1230,9 @@ TEST(ArenaTest, ArenaHooksSanity) {
   {
     ::google::protobuf::Arena arena(options);
     EXPECT_EQ(1, ArenaHooksTestUtil::num_init);
+    EXPECT_EQ(0, ArenaHooksTestUtil::num_allocations);
+    ::google::protobuf::Arena::Create<uint64>(&arena);
+    EXPECT_EQ(1, ArenaHooksTestUtil::num_allocations);
 
     arena.Reset();
     arena.Reset();

src/google/protobuf/compiler/code_generator.h

@@ -79,6 +79,37 @@ class LIBPROTOC_EXPORT CodeGenerator {
                         GeneratorContext* generator_context,
                         string* error) const = 0;
 
+  // Generates code for all given proto files, generating one or more files in
+  // the given output directory.
+  //
+  // This method should be called instead of |Generate()| when
+  // |HasGenerateAll()| returns |true|. It is used to emulate legacy semantics
+  // when more than one `.proto` file is specified on one compiler invocation.
+  //
+  // WARNING: Please do not use unless legacy semantics force the code generator
+  // to produce a single output file for all input files, or otherwise require
+  // an examination of all input files first. The canonical code generator
+  // design produces one output file per input .proto file, and we do not wish
+  // to encourage alternate designs.
+  //
+  // A parameter is given as passed on the command line, as in |Generate()|
+  // above.
+  //
+  // Returns true if successful.  Otherwise, sets *error to a description of
+  // the problem (e.g. "invalid parameter") and returns false.
+  virtual bool GenerateAll(const vector<const FileDescriptor*>& files,
+                           const string& parameter,
+                           GeneratorContext* generator_context,
+                           string* error) const {
+    *error = "Unimplemented GenerateAll() method.";
+    return false;
+  }
+
+  // Returns true if the code generator expects to receive all FileDescriptors
+  // at once (via |GenerateAll()|), rather than one at a time (via
+  // |Generate()|). This is required to implement legacy semantics.
+  virtual bool HasGenerateAll() const { return false; }
+
  private:
   GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(CodeGenerator);
 };

src/google/protobuf/compiler/command_line_interface.cc

@@ -898,12 +898,14 @@ CommandLineInterface::ParseArguments(int argc, const char* const argv[]) {
     return PARSE_ARGUMENT_FAIL;
   }
   if (mode_ != MODE_COMPILE && !dependency_out_name_.empty()) {
-    cerr << "Can only use --dependency_out=FILE when generating code." << endl;
+    std::cerr << "Can only use --dependency_out=FILE when generating code."
+              << std::endl;
     return PARSE_ARGUMENT_FAIL;
   }
   if (!dependency_out_name_.empty() && input_files_.size() > 1) {
-    cerr << "Can only process one input file when using --dependency_out=FILE."
-         << endl;
+    std::cerr
+        << "Can only process one input file when using --dependency_out=FILE."
+        << std::endl;
     return PARSE_ARGUMENT_FAIL;
   }
   if (imports_in_descriptor_set_ && descriptor_set_name_.empty()) {
@@ -1054,11 +1056,11 @@ CommandLineInterface::InterpretArgument(const string& name,
 
   } else if (name == "--dependency_out") {
     if (!dependency_out_name_.empty()) {
-      cerr << name << " may only be passed once." << endl;
+      std::cerr << name << " may only be passed once." << std::endl;
       return PARSE_ARGUMENT_FAIL;
     }
     if (value.empty()) {
-      cerr << name << " requires a non-empty value." << endl;
+      std::cerr << name << " requires a non-empty value." << std::endl;
       return PARSE_ARGUMENT_FAIL;
     }
     dependency_out_name_ = value;
@@ -1272,7 +1274,8 @@ void CommandLineInterface::PrintHelpText() {
 "                              defined in the given proto files. Groups share\n"
 "                              the same field number space with the parent \n"
 "                              message. Extension ranges are counted as \n"
-"                              occupied fields numbers."  << std::endl;
+"                              occupied fields numbers."
+      << std::endl;
   if (!plugin_prefix_.empty()) {
     std::cerr <<
 "  --plugin=EXECUTABLE         Specifies a plugin executable to use.\n"
@@ -1327,13 +1330,23 @@ bool CommandLineInterface::GenerateOutput(
       }
       parameters.append(generator_parameters_[output_directive.name]);
     }
-    for (int i = 0; i < parsed_files.size(); i++) {
-      if (!output_directive.generator->Generate(parsed_files[i], parameters,
-                                                generator_context, &error)) {
-        // Generator returned an error.
-        std::cerr << output_directive.name << ": " << parsed_files[i]->name()
-                  << ": " << error << std::endl;
-        return false;
+    if (output_directive.generator->HasGenerateAll()) {
+      if (!output_directive.generator->GenerateAll(
+          parsed_files, parameters, generator_context, &error)) {
+          // Generator returned an error.
+          std::cerr << output_directive.name << ": "
+                    << ": " << error << std::endl;
+          return false;
+      }
+    } else {
+      for (int i = 0; i < parsed_files.size(); i++) {
+        if (!output_directive.generator->Generate(parsed_files[i], parameters,
+                                                  generator_context, &error)) {
+          // Generator returned an error.
+          std::cerr << output_directive.name << ": " << parsed_files[i]->name()
+                    << ": " << error << std::endl;
+          return false;
+        }
       }
     }
   }
@@ -1403,7 +1416,8 @@ bool CommandLineInterface::GenerateDependencyManifestFile(
       printer.Print(" $disk_file$", "disk_file", disk_file);
       if (i < file_set.file_size() - 1) printer.Print("\\\n");
     } else {
-      cerr << "Unable to identify path for file " << virtual_file << endl;
+      std::cerr << "Unable to identify path for file " << virtual_file
+                << std::endl;
       return false;
     }
   }
@@ -1673,6 +1687,10 @@ void GatherOccupiedFieldRanges(const Descriptor* descriptor,
     ranges->insert(FieldRange(descriptor->extension_range(i)->start,
                               descriptor->extension_range(i)->end));
   }
+  for (int i = 0; i < descriptor->reserved_range_count(); ++i) {
+    ranges->insert(FieldRange(descriptor->reserved_range(i)->start,
+                              descriptor->reserved_range(i)->end));
+  }
   // Handle the nested messages/groups in declaration order to make it
   // post-order strict.
   for (int i = 0; i < descriptor->nested_type_count(); ++i) {