Down-integrate from internal code base.

Makefile.am

@@ -79,6 +79,7 @@ EXTRA_DIST =                                                                 \
   java/src/main/java/com/google/protobuf/Descriptors.java                    \
   java/src/main/java/com/google/protobuf/DynamicMessage.java                 \
   java/src/main/java/com/google/protobuf/Extension.java                      \
+  java/src/main/java/com/google/protobuf/ExtensionLite.java                  \
   java/src/main/java/com/google/protobuf/ExtensionRegistry.java              \
   java/src/main/java/com/google/protobuf/ExtensionRegistryLite.java          \
   java/src/main/java/com/google/protobuf/FieldSet.java                       \
@@ -91,6 +92,10 @@ EXTRA_DIST =                                                                 \
   java/src/main/java/com/google/protobuf/LazyStringArrayList.java            \
   java/src/main/java/com/google/protobuf/LazyStringList.java                 \
   java/src/main/java/com/google/protobuf/LiteralByteString.java              \
+  java/src/main/java/com/google/protobuf/MapEntry.java                       \
+  java/src/main/java/com/google/protobuf/MapEntryLite.java                   \
+  java/src/main/java/com/google/protobuf/MapField.java                       \
+  java/src/main/java/com/google/protobuf/MapFieldLite.java                   \
   java/src/main/java/com/google/protobuf/Message.java                        \
   java/src/main/java/com/google/protobuf/MessageLite.java                    \
   java/src/main/java/com/google/protobuf/MessageLiteOrBuilder.java           \
@@ -112,6 +117,7 @@ EXTRA_DIST =                                                                 \
   java/src/main/java/com/google/protobuf/TextFormat.java                     \
   java/src/main/java/com/google/protobuf/UninitializedMessageException.java  \
   java/src/main/java/com/google/protobuf/UnknownFieldSet.java                \
+  java/src/main/java/com/google/protobuf/UnknownFieldSetLite.java            \
   java/src/main/java/com/google/protobuf/UnmodifiableLazyStringList.java     \
   java/src/main/java/com/google/protobuf/Utf8.java                           \
   java/src/main/java/com/google/protobuf/WireFormat.java                     \
@@ -124,18 +130,22 @@ EXTRA_DIST =                                                                 \
   java/src/test/java/com/google/protobuf/DeprecatedFieldTest.java            \
   java/src/test/java/com/google/protobuf/DescriptorsTest.java                \
   java/src/test/java/com/google/protobuf/DynamicMessageTest.java             \
+  java/src/test/java/com/google/protobuf/FieldPresenceTest.java              \
   java/src/test/java/com/google/protobuf/ForceFieldBuildersPreRun.java       \
   java/src/test/java/com/google/protobuf/GeneratedMessageTest.java           \
   java/src/test/java/com/google/protobuf/IsValidUtf8Test.java                \
   java/src/test/java/com/google/protobuf/IsValidUtf8TestUtil.java            \
-  java/src/test/java/com/google/protobuf/LazyFieldTest.java                  \
   java/src/test/java/com/google/protobuf/LazyFieldLiteTest.java              \
+  java/src/test/java/com/google/protobuf/LazyFieldTest.java                  \
   java/src/test/java/com/google/protobuf/LazyMessageLiteTest.java            \
   java/src/test/java/com/google/protobuf/LazyStringArrayListTest.java        \
   java/src/test/java/com/google/protobuf/LazyStringEndToEndTest.java         \
   java/src/test/java/com/google/protobuf/LiteEqualsAndHashTest.java          \
   java/src/test/java/com/google/protobuf/LiteralByteStringTest.java          \
   java/src/test/java/com/google/protobuf/LiteTest.java                       \
+  java/src/test/java/com/google/protobuf/MapForProto2LiteTest.java           \
+  java/src/test/java/com/google/protobuf/MapForProto2Test.java               \
+  java/src/test/java/com/google/protobuf/MapTest.java                        \
   java/src/test/java/com/google/protobuf/MessageTest.java                    \
   java/src/test/java/com/google/protobuf/NestedBuildersTest.java             \
   java/src/test/java/com/google/protobuf/ParserTest.java                     \
@@ -148,20 +158,25 @@ EXTRA_DIST =                                                                 \
   java/src/test/java/com/google/protobuf/TestBadIdentifiers.java             \
   java/src/test/java/com/google/protobuf/TestUtil.java                       \
   java/src/test/java/com/google/protobuf/TextFormatTest.java                 \
+  java/src/test/java/com/google/protobuf/UnknownEnumValueTest.java           \
+  java/src/test/java/com/google/protobuf/UnknownFieldSetLiteTest.java        \
   java/src/test/java/com/google/protobuf/UnknownFieldSetTest.java            \
   java/src/test/java/com/google/protobuf/UnmodifiableLazyStringListTest.java \
   java/src/test/java/com/google/protobuf/WireFormatTest.java                 \
+  java/src/test/java/com/google/protobuf/field_presence_test.proto           \
   java/src/test/java/com/google/protobuf/lazy_fields_lite.proto              \
   java/src/test/java/com/google/protobuf/lite_equals_and_hash.proto          \
+  java/src/test/java/com/google/protobuf/map_for_proto2_test.proto           \
+  java/src/test/java/com/google/protobuf/map_test.proto                      \
   java/src/test/java/com/google/protobuf/multiple_files_test.proto           \
   java/src/test/java/com/google/protobuf/nested_builders_test.proto          \
   java/src/test/java/com/google/protobuf/nested_extension_lite.proto         \
   java/src/test/java/com/google/protobuf/nested_extension.proto              \
   java/src/test/java/com/google/protobuf/non_nested_extension_lite.proto     \
   java/src/test/java/com/google/protobuf/non_nested_extension.proto          \
-  java/src/test/java/com/google/protobuf/outer_class_name_test.proto         \
   java/src/test/java/com/google/protobuf/outer_class_name_test2.proto        \
   java/src/test/java/com/google/protobuf/outer_class_name_test3.proto        \
+  java/src/test/java/com/google/protobuf/outer_class_name_test.proto         \
   java/src/test/java/com/google/protobuf/test_bad_identifiers.proto          \
   java/src/test/java/com/google/protobuf/test_check_utf8.proto               \
   java/src/test/java/com/google/protobuf/test_check_utf8_size.proto          \
@@ -194,6 +209,7 @@ EXTRA_DIST =                                                                 \
   python/google/protobuf/internal/more_extensions.proto                      \
   python/google/protobuf/internal/more_extensions_dynamic.proto              \
   python/google/protobuf/internal/more_messages.proto                        \
+  python/google/protobuf/internal/proto_builder_test.py                      \
   python/google/protobuf/internal/python_message.py                          \
   python/google/protobuf/internal/reflection_test.py                         \
   python/google/protobuf/internal/service_reflection_test.py                 \
@@ -207,6 +223,10 @@ EXTRA_DIST =                                                                 \
   python/google/protobuf/internal/wire_format.py                             \
   python/google/protobuf/internal/wire_format_test.py                        \
   python/google/protobuf/internal/__init__.py                                \
+  python/google/protobuf/internal/import_test_package/BUILD                  \
+  python/google/protobuf/internal/import_test_package/__init__.py            \
+  python/google/protobuf/internal/import_test_package/inner.proto            \
+  python/google/protobuf/internal/import_test_package/outer.proto            \
   python/google/protobuf/pyext/README                                        \
   python/google/protobuf/pyext/cpp_message.py                                \
   python/google/protobuf/pyext/descriptor.h                                  \
@@ -232,6 +252,7 @@ EXTRA_DIST =                                                                 \
   python/google/protobuf/descriptor_pool.py                                  \
   python/google/protobuf/message.py                                          \
   python/google/protobuf/message_factory.py                                  \
+  python/google/protobuf/proto_builder.py                                    \
   python/google/protobuf/reflection.py                                       \
   python/google/protobuf/service.py                                          \
   python/google/protobuf/service_reflection.py                               \

java/pom.xml

@@ -130,6 +130,10 @@
                   <arg value="../src/google/protobuf/unittest_lite_imports_nonlite.proto" />
                   <arg value="../src/google/protobuf/unittest_enormous_descriptor.proto" />
                   <arg value="../src/google/protobuf/unittest_no_generic_services.proto" />
+                  <arg value="src/test/java/com/google/protobuf/field_presence_test.proto" />
+                  <arg value="src/test/java/com/google/protobuf/map_for_proto2_lite_test.proto" />
+                  <arg value="src/test/java/com/google/protobuf/map_for_proto2_test.proto" />
+                  <arg value="src/test/java/com/google/protobuf/map_test.proto" />
                 </exec>
               </tasks>
               <testSourceRoot>target/generated-test-sources</testSourceRoot>
@@ -164,34 +168,37 @@
             <configuration>
               <includes>
                 <include>**/AbstractMessageLite.java</include>
+                <include>**/AbstractParser.java</include>
+                <include>**/BoundedByteString.java</include>
                 <include>**/ByteString.java</include>
                 <include>**/CodedInputStream.java</include>
                 <include>**/CodedOutputStream.java</include>
+                <include>**/ExtensionLite.java</include>
                 <include>**/ExtensionRegistryLite.java</include>
                 <include>**/FieldSet.java</include>
                 <include>**/GeneratedMessageLite.java</include>
                 <include>**/Internal.java</include>
                 <include>**/InvalidProtocolBufferException.java</include>
+                <include>**/LazyFieldLite.java</include>
                 <include>**/LazyStringArrayList.java</include>
                 <include>**/LazyStringList.java</include>
+                <include>**/LiteralByteString.java</include>
+                <include>**/MapEntryLite.java</include>
+                <include>**/MapFieldLite.java</include>
                 <include>**/MessageLite.java</include>
                 <include>**/MessageLiteOrBuilder.java</include>
+                <include>**/Parser.java</include>
+                <include>**/ProtocolStringList.java</include>
+                <include>**/RopeByteString.java</include>
                 <include>**/SmallSortedMap.java</include>
                 <include>**/UninitializedMessageException.java</include>
+                <include>**/UnknownFieldSetLite.java</include>
                 <include>**/UnmodifiableLazyStringList.java</include>
-                <include>**/WireFormat.java</include>
-                <include>**/Parser.java</include>
-                <include>**/AbstractParser.java</include>
-                <include>**/BoundedByteString.java</include>
-                <include>**/LiteralByteString.java</include>
-                <include>**/RopeByteString.java</include>
                 <include>**/Utf8.java</include>
-                <include>**/LazyField.java</include>
-                <include>**/LazyFieldLite.java</include>
-                <include>**/ProtocolStringList.java</include>
+                <include>**/WireFormat.java</include>
               </includes>
               <testIncludes>
-                <testInclude>**/LiteTest.java</testInclude>
+                <testInclude>**/**LiteTest.java</testInclude>
                 <testInclude>**/*Lite.java</testInclude>
               </testIncludes>
             </configuration>
@@ -200,7 +207,7 @@
             <artifactId>maven-surefire-plugin</artifactId>
             <configuration>
               <includes>
-                <include>**/LiteTest.java</include>
+                <include>**/**LiteTest.java</include>
               </includes>
             </configuration>
           </plugin>

java/src/main/java/com/google/protobuf/AbstractMessage.java

@@ -37,6 +37,9 @@ import com.google.protobuf.Internal.EnumLite;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
@@ -143,6 +146,40 @@ public abstract class AbstractMessage extends AbstractMessageLite
     return toByteString(a).equals(toByteString(b));
   }
   
+  /**
+   * Converts a list of MapEntry messages into a Map used for equals() and
+   * hashCode().
+   */
+  @SuppressWarnings({"rawtypes", "unchecked"})
+  private static Map convertMapEntryListToMap(List list) {
+    if (list.isEmpty()) {
+      return Collections.emptyMap();
+    }
+    Map result = new HashMap();
+    Iterator iterator = list.iterator();
+    Message entry = (Message) iterator.next();
+    Descriptors.Descriptor descriptor = entry.getDescriptorForType();
+    Descriptors.FieldDescriptor key = descriptor.findFieldByName("key");
+    Descriptors.FieldDescriptor value = descriptor.findFieldByName("value");
+    result.put(entry.getField(key), entry.getField(value));
+    while (iterator.hasNext()) {
+      entry = (Message) iterator.next();
+      result.put(entry.getField(key), entry.getField(value));
+    }
+    return result;
+  }
+  
+  /**
+   * Compares two map fields. The parameters must be a list of MapEntry
+   * messages.
+   */
+  @SuppressWarnings({"rawtypes", "unchecked"})
+  private static boolean compareMapField(Object a, Object b) {
+    Map ma = convertMapEntryListToMap((List) a);
+    Map mb = convertMapEntryListToMap((List) b);
+    return MapFieldLite.equals(ma, mb);
+  }
+  
   /**
    * Compares two set of fields.
    * This method is used to implement {@link AbstractMessage#equals(Object)}
@@ -181,6 +218,10 @@ public abstract class AbstractMessage extends AbstractMessageLite
             return false;
           }
         }
+      } else if (descriptor.isMapField()) {
+        if (!compareMapField(value1, value2)) {
+          return false;
+        }
       } else {
         // Compare non-bytes fields.
         if (!value1.equals(value2)) {
@@ -190,6 +231,15 @@ public abstract class AbstractMessage extends AbstractMessageLite
     }
     return true;
   }
+  
+  /**
+   * Calculates the hash code of a map field. {@code value} must be a list of
+   * MapEntry messages.
+   */
+  @SuppressWarnings("unchecked")
+  private static int hashMapField(Object value) {
+    return MapFieldLite.calculateHashCodeForMap(convertMapEntryListToMap((List) value));
+  }
 
   /** Get a hash code for given fields and values, using the given seed. */
   @SuppressWarnings("unchecked")
@@ -198,7 +248,9 @@ public abstract class AbstractMessage extends AbstractMessageLite
       FieldDescriptor field = entry.getKey();
       Object value = entry.getValue();
       hash = (37 * hash) + field.getNumber();
-      if (field.getType() != FieldDescriptor.Type.ENUM){
+      if (field.isMapField()) {
+        hash = (53 * hash) + hashMapField(value);
+      } else if (field.getType() != FieldDescriptor.Type.ENUM){
         hash = (53 * hash) + value.hashCode();
       } else if (field.isRepeated()) {
         List<? extends EnumLite> list = (List<? extends EnumLite>) value;
@@ -359,6 +411,12 @@ public abstract class AbstractMessage extends AbstractMessageLite
           "getFieldBuilder() called on an unsupported message type.");
     }
 
+    public Message.Builder getRepeatedFieldBuilder(final FieldDescriptor field,
+        int index) {
+      throw new UnsupportedOperationException(
+          "getRepeatedFieldBuilder() called on an unsupported message type.");
+    }
+
     public String toString() {
       return TextFormat.printToString(this);
     }

java/src/main/java/com/google/protobuf/CodedInputStream.java

@@ -612,16 +612,16 @@ public final class CodedInputStream {
         return x;
       } else if (bufferSize - pos < 9) {
         break fastpath;
-      } else if ((x ^= (buffer[pos++] << 7)) < 0L) {
-        x ^= (~0L << 7);
-      } else if ((x ^= (buffer[pos++] << 14)) >= 0L) {
-        x ^= (~0L << 7) ^ (~0L << 14);
-      } else if ((x ^= (buffer[pos++] << 21)) < 0L) {
-        x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21);
+      } else if ((x ^= (buffer[pos++] << 7)) < 0) {
+        x ^= (~0 << 7);
+      } else if ((x ^= (buffer[pos++] << 14)) >= 0) {
+        x ^= (~0 << 7) ^ (~0 << 14);
+      } else if ((x ^= (buffer[pos++] << 21)) < 0) {
+        x ^= (~0 << 7) ^ (~0 << 14) ^ (~0 << 21);
       } else {
         int y = buffer[pos++];
         x ^= y << 28;
-        x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28);
+        x ^= (~0 << 7) ^ (~0 << 14) ^ (~0 << 21) ^ (~0 << 28);
         if (y < 0 &&
             buffer[pos++] < 0 &&
             buffer[pos++] < 0 &&
@@ -739,13 +739,13 @@ public final class CodedInputStream {
         return y;
       } else if (bufferSize - pos < 9) {
         break fastpath;
-      } else if ((x = y ^ (buffer[pos++] << 7)) < 0L) {
-        x ^= (~0L << 7);
-      } else if ((x ^= (buffer[pos++] << 14)) >= 0L) {
-        x ^= (~0L << 7) ^ (~0L << 14);
-      } else if ((x ^= (buffer[pos++] << 21)) < 0L) {
-        x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21);
-      } else if ((x ^= ((long) buffer[pos++] << 28)) >= 0L) {
+      } else if ((y ^= (buffer[pos++] << 7)) < 0) {
+        x = y ^ (~0 << 7);
+      } else if ((y ^= (buffer[pos++] << 14)) >= 0) {
+        x = y ^ ((~0 << 7) ^ (~0 << 14));
+      } else if ((y ^= (buffer[pos++] << 21)) < 0) {
+        x = y ^ ((~0 << 7) ^ (~0 << 14) ^ (~0 << 21));
+      } else if ((x = ((long) y) ^ ((long) buffer[pos++] << 28)) >= 0L) {
         x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28);
       } else if ((x ^= ((long) buffer[pos++] << 35)) < 0L) {
         x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28) ^ (~0L << 35);
@@ -882,7 +882,7 @@ public final class CodedInputStream {
   /** See setSizeLimit() */
   private int sizeLimit = DEFAULT_SIZE_LIMIT;
 
-  private static final int DEFAULT_RECURSION_LIMIT = 64;
+  private static final int DEFAULT_RECURSION_LIMIT = 100;
   private static final int DEFAULT_SIZE_LIMIT = 64 << 20;  // 64MB
   private static final int BUFFER_SIZE = 4096;
 

java/src/main/java/com/google/protobuf/Descriptors.java

@@ -32,6 +32,7 @@ package com.google.protobuf;
 
 import com.google.protobuf.DescriptorProtos.*;
 
+import java.lang.ref.WeakReference;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -40,6 +41,7 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.WeakHashMap;
 import java.util.logging.Logger;
 import java.io.UnsupportedEncodingException;
 
@@ -123,6 +125,26 @@ public final class Descriptors {
       return Collections.unmodifiableList(Arrays.asList(publicDependencies));
     }
 
+    /** The syntax of the .proto file. */
+    public enum Syntax {
+      UNKNOWN("unknown"),
+      PROTO2("proto2"),
+      PROTO3("proto3");
+
+      Syntax(String name) {
+        this.name = name;
+      }
+      private final String name;
+    }
+
+    /** Get the syntax of the .proto file. */
+    public Syntax getSyntax() {
+      if (Syntax.PROTO3.name.equals(proto.getSyntax())) {
+        return Syntax.PROTO3;
+      }
+      return Syntax.PROTO2;
+    }
+
     /**
      * Find a message type in the file by name.  Does not find nested types.
      *
@@ -539,6 +561,10 @@ public final class Descriptors {
         extensions[i].setProto(proto.getExtension(i));
       }
     }
+    
+    boolean supportsUnknownEnumValue() {
+      return getSyntax() == Syntax.PROTO3;
+    }
   }
 
   // =================================================================
@@ -871,6 +897,11 @@ public final class Descriptors {
       return (type == Type.STRING) && (getFile().getOptions().getJavaStringCheckUtf8());
     }
 
+    boolean isMapField() {
+      return getType() == Type.MESSAGE && isRepeated()
+          && getMessageType().getOptions().getMapEntry();
+    }
+
     // I'm pretty sure values() constructs a new array every time, since there
     // is nothing stopping the caller from mutating the array.  Therefore we
     // make a static copy here.
@@ -1001,6 +1032,11 @@ public final class Descriptors {
       return getNumber() - other.getNumber();
     }
 
+    @Override
+    public String toString() {
+      return getFullName();
+    }
+
     private final int index;
 
     private FieldDescriptorProto proto;
@@ -1420,6 +1456,64 @@ public final class Descriptors {
       return file.pool.enumValuesByNumber.get(
         new DescriptorPool.DescriptorIntPair(this, number));
     }
+    
+    /**
+     * Get the enum value for a number. If no enum value has this number,
+     * construct an EnumValueDescriptor for it.
+     */
+    public EnumValueDescriptor findValueByNumberCreatingIfUnknown(final int number) {
+      EnumValueDescriptor result = findValueByNumber(number);
+      if (result != null) {
+        return result;
+      }
+      // The number represents an unknown enum value.
+      synchronized (this) {
+        // Descriptors are compared by object identity so for the same number
+        // we need to return the same EnumValueDescriptor object. This means
+        // we have to store created EnumValueDescriptors. However, as there
+        // are potentially 2G unknown enum values, storing all of these
+        // objects persistently will consume lots of memory for long-running
+        // services and it's also unnecessary as not many EnumValueDescriptors
+        // will be used at the same time.
+        //
+        // To solve the problem we take advantage of Java's weak references and
+        // rely on gc to release unused descriptors.
+        //
+        // Here is how it works:
+        //   * We store unknown EnumValueDescriptors in a WeakHashMap with the
+        //     value being a weak reference to the descriptor.
+        //   * The descriptor holds a strong reference to the key so as long
+        //     as the EnumValueDescriptor is in use, the key will be there
+        //     and the corresponding map entry will be there. Following-up
+        //     queries with the same number will return the same descriptor.
+        //   * If the user no longer uses an unknown EnumValueDescriptor,
+        //     it will be gc-ed since we only hold a weak reference to it in
+        //     the map. The key in the corresponding map entry will also be
+        //     gc-ed as the only strong reference to it is in the descriptor
+        //     which is just gc-ed. With the key being gone WeakHashMap will
+        //     then remove the whole entry. This way unknown descriptors will
+        //     be freed automatically and we don't need to do anything to
+        //     clean-up unused map entries.
+        
+        // Note: We must use "new Integer(number)" here because we don't want
+        // these Integer objects to be cached.
+        Integer key = new Integer(number);
+        WeakReference<EnumValueDescriptor> reference = unknownValues.get(key);
+        if (reference != null) {
+          result = reference.get();
+        }
+        if (result == null) {
+          result = new EnumValueDescriptor(file, this, key);
+          unknownValues.put(key, new WeakReference<EnumValueDescriptor>(result));
+        }
+      }
+      return result;
+    }
+    
+    // Used in tests only.
+    int getUnknownEnumValueDescriptorCount() {
+      return unknownValues.size();
+    }
 
     private final int index;
     private EnumDescriptorProto proto;
@@ -1427,6 +1521,8 @@ public final class Descriptors {
     private final FileDescriptor file;
     private final Descriptor containingType;
     private EnumValueDescriptor[] values;
+    private final WeakHashMap<Integer, WeakReference<EnumValueDescriptor>> unknownValues =
+        new WeakHashMap<Integer, WeakReference<EnumValueDescriptor>>();
 
     private EnumDescriptor(final EnumDescriptorProto proto,
                            final FileDescriptor file,
@@ -1531,6 +1627,25 @@ public final class Descriptors {
       file.pool.addSymbol(this);
       file.pool.addEnumValueByNumber(this);
     }
+    
+    private Integer number;
+    // Create an unknown enum value.
+    private EnumValueDescriptor(
+        final FileDescriptor file,
+        final EnumDescriptor parent,
+        final Integer number) {
+      String name = "UNKNOWN_ENUM_VALUE_" + parent.getName() + "_" + number;
+      EnumValueDescriptorProto proto = EnumValueDescriptorProto
+          .newBuilder().setName(name).setNumber(number).build();
+      this.index = -1;
+      this.proto = proto;
+      this.file = file;
+      this.type = parent;
+      this.fullName = parent.getFullName() + '.' + proto.getName();
+      this.number = number;
+      
+      // Don't add this descriptor into pool.
+    }
 
     /** See {@link FileDescriptor#setProto}. */
     private void setProto(final EnumValueDescriptorProto proto) {

java/src/main/java/com/google/protobuf/DynamicMessage.java

@@ -549,12 +549,22 @@ public final class DynamicMessage extends AbstractMessage {
     }
 
     public Builder setUnknownFields(UnknownFieldSet unknownFields) {
+      if (getDescriptorForType().getFile().getSyntax()
+          == Descriptors.FileDescriptor.Syntax.PROTO3) {
+        // Proto3 discards unknown fields.
+        return this;
+      }
       this.unknownFields = unknownFields;
       return this;
     }
 
     @Override
     public Builder mergeUnknownFields(UnknownFieldSet unknownFields) {
+      if (getDescriptorForType().getFile().getSyntax()
+          == Descriptors.FileDescriptor.Syntax.PROTO3) {
+        // Proto3 discards unknown fields.
+        return this;
+      }
       this.unknownFields =
         UnknownFieldSet.newBuilder(this.unknownFields)
                        .mergeFrom(unknownFields)
@@ -588,10 +598,12 @@ public final class DynamicMessage extends AbstractMessage {
         throw new IllegalArgumentException(
           "DynamicMessage should use EnumValueDescriptor to set Enum Value.");
       }
-      if (field.getEnumType() != ((EnumValueDescriptor) value).getType()) {
-        throw new IllegalArgumentException(
-          "EnumValueDescriptor doesn't much Enum Field.");
-      }
+      // TODO(xiaofeng): Re-enable this check after Orgstore is fixed to not
+      // set incorrect EnumValueDescriptors.
+      // if (field.getEnumType() != ((EnumValueDescriptor) value).getType()) {
+      //   throw new IllegalArgumentException(
+      //     "EnumValueDescriptor doesn't match Enum Field.");
+      // }
     }
 
     /** Verifies the value for an enum field. */
@@ -618,5 +630,12 @@ public final class DynamicMessage extends AbstractMessage {
       throw new UnsupportedOperationException(
         "getFieldBuilder() called on a dynamic message type.");
     }
+
+    @Override
+    public com.google.protobuf.Message.Builder getRepeatedFieldBuilder(FieldDescriptor field,
+        int index) {
+      throw new UnsupportedOperationException(
+        "getRepeatedFieldBuilder() called on a dynamic message type.");
+    }
   }
 }

java/src/main/java/com/google/protobuf/Extension.java

@@ -35,27 +35,16 @@ package com.google.protobuf;
  *
  * @author liujisi@google.com (Jisi Liu)
  */
-public abstract class Extension<ContainingType extends MessageLite, Type> {
-  /** Returns the field number of the extension. */
-  public abstract int getNumber();
-
-  /** Returns the type of the field. */
-  public abstract WireFormat.FieldType getLiteType();
-
-  /** Returns whether it is a repeated field. */
-  public abstract boolean isRepeated();
+public abstract class Extension<ContainingType extends MessageLite, Type>
+    extends ExtensionLite<ContainingType, Type> {
 
   /** Returns the descriptor of the extension. */
   public abstract Descriptors.FieldDescriptor getDescriptor();
 
-  /** Returns the default value of the extension field. */
-  public abstract Type getDefaultValue();
-
-  /**
-   * Returns the default instance of the extension field, if it's a message
-   * extension.
-   */
-  public abstract MessageLite getMessageDefaultInstance();
+  /** Returns whether or not this extension is a Lite Extension. */
+  final boolean isLite() {
+    return false;
+  }
 
   // All the methods below are extension implementation details.
 

java/src/main/java/com/google/protobuf/ExtensionLite.java

@@ -0,0 +1,63 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package com.google.protobuf;
+
+/**
+ * Lite interface that generated extensions implement.
+ * <p>
+ * Methods are for use by generated code only. You can hold a reference to
+ * extensions using this type name.
+ */
+public abstract class ExtensionLite<ContainingType extends MessageLite, Type> {
+
+  /** Returns the field number of the extension. */
+  public abstract int getNumber();
+
+  /** Returns the type of the field. */
+  public abstract WireFormat.FieldType getLiteType();
+
+  /** Returns whether it is a repeated field. */
+  public abstract boolean isRepeated();
+
+  /** Returns the default value of the extension field. */
+  public abstract Type getDefaultValue();
+
+  /**
+   * Returns the default instance of the extension field, if it's a message
+   * extension.
+   */
+  public abstract MessageLite getMessageDefaultInstance();
+  
+  /** Returns whether or not this extension is a Lite Extension. */
+  boolean isLite() {
+    return true;
+  }
+}

java/src/main/java/com/google/protobuf/FieldSet.java

@@ -672,7 +672,7 @@ final class FieldSet<FieldDescriptorType extends
    *               {@link Message#getField(Descriptors.FieldDescriptor)} for
    *               this field.
    */
-  private static void writeElementNoTag(
+  static void writeElementNoTag(
       final CodedOutputStream output,
       final WireFormat.FieldType type,
       final Object value) throws IOException {
@@ -830,7 +830,7 @@ final class FieldSet<FieldDescriptorType extends
    *               {@link Message#getField(Descriptors.FieldDescriptor)} for
    *               this field.
    */
-  private static int computeElementSizeNoTag(
+  static int computeElementSizeNoTag(
       final WireFormat.FieldType type, final Object value) {
     switch (type) {
       // Note:  Minor violation of 80-char limit rule here because this would

java/src/main/java/com/google/protobuf/GeneratedMessage.java

@@ -31,16 +31,20 @@
 package com.google.protobuf;
 
 import com.google.protobuf.Descriptors.Descriptor;
+import com.google.protobuf.Descriptors.EnumDescriptor;
 import com.google.protobuf.Descriptors.EnumValueDescriptor;
 import com.google.protobuf.Descriptors.FieldDescriptor;
 import com.google.protobuf.Descriptors.FileDescriptor;
 import com.google.protobuf.Descriptors.OneofDescriptor;
+import com.google.protobuf.GeneratedMessageLite.ExtendableMessage;
+import com.google.protobuf.GeneratedMessageLite.GeneratedExtension;
 
 import java.io.IOException;
 import java.io.ObjectStreamException;
 import java.io.Serializable;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
+import java.lang.reflect.Type;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Iterator;
@@ -67,10 +71,15 @@ public abstract class GeneratedMessage extends AbstractMessage
    */
   protected static boolean alwaysUseFieldBuilders = false;
 
+  /** For use by generated code only.  */
+  protected UnknownFieldSet unknownFields;
+  
   protected GeneratedMessage() {
+    unknownFields = UnknownFieldSet.getDefaultInstance();
   }
 
   protected GeneratedMessage(Builder<?> builder) {
+    unknownFields = builder.getUnknownFields();
   }
 
   public Parser<? extends GeneratedMessage> getParserForType() {
@@ -291,13 +300,12 @@ public abstract class GeneratedMessage extends AbstractMessage
       return isClean;
     }
 
-    // This is implemented here only to work around an apparent bug in the
-    // Java compiler and/or build system.  See bug #1898463.  The mere presence
-    // of this dummy clone() implementation makes it go away.
     @Override
     public BuilderType clone() {
-      throw new UnsupportedOperationException(
-          "This is supposed to be overridden by subclasses.");
+      BuilderType builder =
+          (BuilderType) getDefaultInstanceForType().newBuilderForType();
+      builder.mergeFrom(buildPartial());
+      return builder;
     }
 
     /**
@@ -357,6 +365,13 @@ public abstract class GeneratedMessage extends AbstractMessage
       return internalGetFieldAccessorTable().getField(field).getBuilder(this);
     }
 
+    //@Override (Java 1.6 override semantics, but we must support 1.5)
+    public Message.Builder getRepeatedFieldBuilder(final FieldDescriptor field,
+        int index) {
+      return internalGetFieldAccessorTable().getField(field).getRepeatedBuilder(
+          this, index);
+    }
+
     //@Override (Java 1.6 override semantics, but we must support 1.5)
     public boolean hasOneof(final OneofDescriptor oneof) {
       return internalGetFieldAccessorTable().getOneof(oneof).has(this);
@@ -428,7 +443,7 @@ public abstract class GeneratedMessage extends AbstractMessage
       return (BuilderType) this;
     }
 
-    public final BuilderType setUnknownFields(
+    public BuilderType setUnknownFields(
         final UnknownFieldSet unknownFields) {
       this.unknownFields = unknownFields;
       onChanged();
@@ -436,7 +451,7 @@ public abstract class GeneratedMessage extends AbstractMessage
     }
 
     @Override
-    public final BuilderType mergeUnknownFields(
+    public BuilderType mergeUnknownFields(
         final UnknownFieldSet unknownFields) {
       this.unknownFields =
         UnknownFieldSet.newBuilder(this.unknownFields)
@@ -529,6 +544,25 @@ public abstract class GeneratedMessage extends AbstractMessage
         isClean = false;
       }
     }
+
+    /**
+     * Gets the map field with the given field number. This method should be
+     * overridden in the generated message class if the message contains map
+     * fields.
+     * 
+     * Unlike other field types, reflection support for map fields can't be
+     * implemented based on generated public API because we need to access a
+     * map field as a list in reflection API but the generated API only allows
+     * us to access it as a map. This method returns the underlying map field
+     * directly and thus enables us to access the map field as a list. 
+     */
+    @SuppressWarnings({"unused", "rawtypes"})
+    protected MapField internalGetMapField(int fieldNumber) {
+      // Note that we can't use descriptor names here because this method will
+      // be called when descriptor is being initialized.
+      throw new RuntimeException(
+          "No map fields found in " + getClass().getName());
+    }
   }
 
   // =================================================================
@@ -541,19 +575,19 @@ public abstract class GeneratedMessage extends AbstractMessage
 
     /** Check if a singular extension is present. */
     <Type> boolean hasExtension(
-        Extension<MessageType, Type> extension);
+        ExtensionLite<MessageType, Type> extension);
 
     /** Get the number of elements in a repeated extension. */
     <Type> int getExtensionCount(
-        Extension<MessageType, List<Type>> extension);
+        ExtensionLite<MessageType, List<Type>> extension);
 
     /** Get the value of an extension. */
     <Type> Type getExtension(
-        Extension<MessageType, Type> extension);
+        ExtensionLite<MessageType, Type> extension);
 
     /** Get one element of a repeated extension. */
     <Type> Type getExtension(
-        Extension<MessageType, List<Type>> extension,
+        ExtensionLite<MessageType, List<Type>> extension,
         int index);
   }
 
@@ -625,7 +659,9 @@ public abstract class GeneratedMessage extends AbstractMessage
     /** Check if a singular extension is present. */
     //@Override (Java 1.6 override semantics, but we must support 1.5)
     public final <Type> boolean hasExtension(
-        final Extension<MessageType, Type> extension) {
+        final ExtensionLite<MessageType, Type> extensionLite) {
+      Extension<MessageType, Type> extension = checkNotLite(extensionLite);
+
       verifyExtensionContainingType(extension);
       return extensions.hasField(extension.getDescriptor());
     }
@@ -633,7 +669,9 @@ public abstract class GeneratedMessage extends AbstractMessage
     /** Get the number of elements in a repeated extension. */
     //@Override (Java 1.6 override semantics, but we must support 1.5)
     public final <Type> int getExtensionCount(
-        final Extension<MessageType, List<Type>> extension) {
+        final ExtensionLite<MessageType, List<Type>> extensionLite) {
+      Extension<MessageType, List<Type>> extension = checkNotLite(extensionLite);
+
       verifyExtensionContainingType(extension);
       final FieldDescriptor descriptor = extension.getDescriptor();
       return extensions.getRepeatedFieldCount(descriptor);
@@ -643,7 +681,9 @@ public abstract class GeneratedMessage extends AbstractMessage
     //@Override (Java 1.6 override semantics, but we must support 1.5)
     @SuppressWarnings("unchecked")
     public final <Type> Type getExtension(
-        final Extension<MessageType, Type> extension) {
+        final ExtensionLite<MessageType, Type> extensionLite) {
+      Extension<MessageType, Type> extension = checkNotLite(extensionLite);
+      
       verifyExtensionContainingType(extension);
       FieldDescriptor descriptor = extension.getDescriptor();
       final Object value = extensions.getField(descriptor);
@@ -666,8 +706,10 @@ public abstract class GeneratedMessage extends AbstractMessage
     //@Override (Java 1.6 override semantics, but we must support 1.5)
     @SuppressWarnings("unchecked")
     public final <Type> Type getExtension(
-        final Extension<MessageType, List<Type>> extension,
+        final ExtensionLite<MessageType, List<Type>> extensionLite,
         final int index) {
+      Extension<MessageType, List<Type>> extension = checkNotLite(extensionLite);
+
       verifyExtensionContainingType(extension);
       FieldDescriptor descriptor = extension.getDescriptor();
       return (Type) extension.singularFromReflectionType(
@@ -914,11 +956,10 @@ public abstract class GeneratedMessage extends AbstractMessage
 
     // This is implemented here only to work around an apparent bug in the
     // Java compiler and/or build system.  See bug #1898463.  The mere presence
-    // of this dummy clone() implementation makes it go away.
+    // of this clone() implementation makes it go away.
     @Override
     public BuilderType clone() {
-      throw new UnsupportedOperationException(
-          "This is supposed to be overridden by subclasses.");
+      return super.clone();
     }
 
     private void ensureExtensionsIsMutable() {
@@ -943,7 +984,9 @@ public abstract class GeneratedMessage extends AbstractMessage
     /** Check if a singular extension is present. */
     //@Override (Java 1.6 override semantics, but we must support 1.5)
     public final <Type> boolean hasExtension(
-        final Extension<MessageType, Type> extension) {
+        final ExtensionLite<MessageType, Type> extensionLite) {
+      Extension<MessageType, Type> extension = checkNotLite(extensionLite);
+
       verifyExtensionContainingType(extension);
       return extensions.hasField(extension.getDescriptor());
     }
@@ -951,7 +994,9 @@ public abstract class GeneratedMessage extends AbstractMessage
     /** Get the number of elements in a repeated extension. */
     //@Override (Java 1.6 override semantics, but we must support 1.5)
     public final <Type> int getExtensionCount(
-        final Extension<MessageType, List<Type>> extension) {
+        final ExtensionLite<MessageType, List<Type>> extensionLite) {
+      Extension<MessageType, List<Type>> extension = checkNotLite(extensionLite);
+
       verifyExtensionContainingType(extension);
       final FieldDescriptor descriptor = extension.getDescriptor();
       return extensions.getRepeatedFieldCount(descriptor);
@@ -960,7 +1005,9 @@ public abstract class GeneratedMessage extends AbstractMessage
     /** Get the value of an extension. */
     //@Override (Java 1.6 override semantics, but we must support 1.5)
     public final <Type> Type getExtension(
-        final Extension<MessageType, Type> extension) {
+        final ExtensionLite<MessageType, Type> extensionLite) {
+      Extension<MessageType, Type> extension = checkNotLite(extensionLite);
+
       verifyExtensionContainingType(extension);
       FieldDescriptor descriptor = extension.getDescriptor();
       final Object value = extensions.getField(descriptor);
@@ -982,8 +1029,10 @@ public abstract class GeneratedMessage extends AbstractMessage
     /** Get one element of a repeated extension. */
     //@Override (Java 1.6 override semantics, but we must support 1.5)
     public final <Type> Type getExtension(
-        final Extension<MessageType, List<Type>> extension,
+        final ExtensionLite<MessageType, List<Type>> extensionLite,
         final int index) {
+      Extension<MessageType, List<Type>> extension = checkNotLite(extensionLite);
+
       verifyExtensionContainingType(extension);
       FieldDescriptor descriptor = extension.getDescriptor();
       return (Type) extension.singularFromReflectionType(
@@ -992,8 +1041,10 @@ public abstract class GeneratedMessage extends AbstractMessage
 
     /** Set the value of an extension. */
     public final <Type> BuilderType setExtension(
-        final Extension<MessageType, Type> extension,
+        final ExtensionLite<MessageType, Type> extensionLite,
         final Type value) {
+      Extension<MessageType, Type> extension = checkNotLite(extensionLite);
+
       verifyExtensionContainingType(extension);
       ensureExtensionsIsMutable();
       final FieldDescriptor descriptor = extension.getDescriptor();
@@ -1004,8 +1055,10 @@ public abstract class GeneratedMessage extends AbstractMessage
 
     /** Set the value of one element of a repeated extension. */
     public final <Type> BuilderType setExtension(
-        final Extension<MessageType, List<Type>> extension,
+        final ExtensionLite<MessageType, List<Type>> extensionLite,
         final int index, final Type value) {
+      Extension<MessageType, List<Type>> extension = checkNotLite(extensionLite);
+
       verifyExtensionContainingType(extension);
       ensureExtensionsIsMutable();
       final FieldDescriptor descriptor = extension.getDescriptor();
@@ -1018,8 +1071,10 @@ public abstract class GeneratedMessage extends AbstractMessage
 
     /** Append a value to a repeated extension. */
     public final <Type> BuilderType addExtension(
-        final Extension<MessageType, List<Type>> extension,
+        final ExtensionLite<MessageType, List<Type>> extensionLite,
         final Type value) {
+      Extension<MessageType, List<Type>> extension = checkNotLite(extensionLite);
+
       verifyExtensionContainingType(extension);
       ensureExtensionsIsMutable();
       final FieldDescriptor descriptor = extension.getDescriptor();
@@ -1031,7 +1086,9 @@ public abstract class GeneratedMessage extends AbstractMessage
 
     /** Clear an extension. */
     public final <Type> BuilderType clearExtension(
-        final Extension<MessageType, ?> extension) {
+        final ExtensionLite<MessageType, ?> extensionLite) {
+      Extension<MessageType, ?> extension = checkNotLite(extensionLite);
+
       verifyExtensionContainingType(extension);
       ensureExtensionsIsMutable();
       extensions.clearField(extension.getDescriptor());
@@ -1594,6 +1651,25 @@ public abstract class GeneratedMessage extends AbstractMessage
       }
     }
   }
+  
+  /**
+   * Gets the map field with the given field number. This method should be
+   * overridden in the generated message class if the message contains map
+   * fields.
+   * 
+   * Unlike other field types, reflection support for map fields can't be
+   * implemented based on generated public API because we need to access a
+   * map field as a list in reflection API but the generated API only allows
+   * us to access it as a map. This method returns the underlying map field
+   * directly and thus enables us to access the map field as a list. 
+   */
+  @SuppressWarnings({"rawtypes", "unused"})
+  protected MapField internalGetMapField(int fieldNumber) {
+    // Note that we can't use descriptor names here because this method will
+    // be called when descriptor is being initialized.
+    throw new RuntimeException(
+        "No map fields found in " + getClass().getName());
+  }
 
   /**
    * Users should ignore this class.  This class provides the implementation
@@ -1633,6 +1709,11 @@ public abstract class GeneratedMessage extends AbstractMessage
       oneofs = new OneofAccessor[descriptor.getOneofs().size()];
       initialized = false;
     }
+    
+    private boolean isMapFieldEnabled(FieldDescriptor field) {
+      boolean result = true;
+      return result;
+    }
 
     /**
      * Ensures the field accessors are initialized. This method is thread-safe.
@@ -1657,8 +1738,13 @@ public abstract class GeneratedMessage extends AbstractMessage
           }
           if (field.isRepeated()) {
             if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) {
-              fields[i] = new RepeatedMessageFieldAccessor(
-                  field, camelCaseNames[i], messageClass, builderClass);
+              if (field.isMapField() && isMapFieldEnabled(field)) {
+                fields[i] = new MapFieldAccessor(
+                    field, camelCaseNames[i], messageClass, builderClass);
+              } else {
+                fields[i] = new RepeatedMessageFieldAccessor(
+                    field, camelCaseNames[i], messageClass, builderClass);
+              }
             } else if (field.getJavaType() == FieldDescriptor.JavaType.ENUM) {
               fields[i] = new RepeatedEnumFieldAccessor(
                   field, camelCaseNames[i], messageClass, builderClass);
@@ -1744,6 +1830,8 @@ public abstract class GeneratedMessage extends AbstractMessage
       void clear(Builder builder);
       Message.Builder newBuilder();
       Message.Builder getBuilder(GeneratedMessage.Builder builder);
+      Message.Builder getRepeatedBuilder(GeneratedMessage.Builder builder,
+                                         int index);
     }
 
     /** OneofAccessor provides access to a single oneof. */
@@ -1799,9 +1887,9 @@ public abstract class GeneratedMessage extends AbstractMessage
         invokeOrDie(clearMethod, builder);
       }
     }
-    
+
     private static boolean supportFieldPresence(FileDescriptor file) {
-      return true;
+      return file.getSyntax() == FileDescriptor.Syntax.PROTO2;
     }
 
     // ---------------------------------------------------------------
@@ -1919,6 +2007,11 @@ public abstract class GeneratedMessage extends AbstractMessage
         throw new UnsupportedOperationException(
           "getFieldBuilder() called on a non-Message type.");
       }
+      public Message.Builder getRepeatedBuilder(GeneratedMessage.Builder builder,
+          int index) {
+        throw new UnsupportedOperationException(
+          "getRepeatedFieldBuilder() called on a non-Message type.");
+      }
     }
 
     private static class RepeatedFieldAccessor implements FieldAccessor {
@@ -2014,6 +2107,113 @@ public abstract class GeneratedMessage extends AbstractMessage
         throw new UnsupportedOperationException(
           "getFieldBuilder() called on a non-Message type.");
       }
+      public Message.Builder getRepeatedBuilder(GeneratedMessage.Builder builder,
+          int index) {
+        throw new UnsupportedOperationException(
+          "getRepeatedFieldBuilder() called on a non-Message type.");
+      }
+    }
+
+    private static class MapFieldAccessor implements FieldAccessor {
+      MapFieldAccessor(
+          final FieldDescriptor descriptor, final String camelCaseName,
+          final Class<? extends GeneratedMessage> messageClass,
+          final Class<? extends Builder> builderClass) {
+        field = descriptor;
+        Method getDefaultInstanceMethod =
+            getMethodOrDie(messageClass, "getDefaultInstance");
+        MapField defaultMapField = getMapField(
+            (GeneratedMessage) invokeOrDie(getDefaultInstanceMethod, null));
+        mapEntryMessageDefaultInstance =
+            defaultMapField.getMapEntryMessageDefaultInstance();
+      }
+
+      private final FieldDescriptor field;
+      private final Message mapEntryMessageDefaultInstance;
+      
+      private MapField<?, ?> getMapField(GeneratedMessage message) {
+        return (MapField<?, ?>) message.internalGetMapField(field.getNumber());
+      }
+      
+      private MapField<?, ?> getMapField(GeneratedMessage.Builder builder) {
+        return (MapField<?, ?>) builder.internalGetMapField(field.getNumber());
+      }
+      
+      public Object get(GeneratedMessage message) {
+        List result = new ArrayList();
+        for (int i = 0; i < getRepeatedCount(message); i++) {
+          result.add(getRepeated(message, i));
+        }
+        return Collections.unmodifiableList(result);
+      }
+
+      public Object get(Builder builder) {
+        List result = new ArrayList();
+        for (int i = 0; i < getRepeatedCount(builder); i++) {
+          result.add(getRepeated(builder, i));
+        }
+        return Collections.unmodifiableList(result);
+      }
+
+      public void set(Builder builder, Object value) {
+        clear(builder);
+        for (Object entry : (List) value) {
+          addRepeated(builder, entry);
+        }
+      }
+
+      public Object getRepeated(GeneratedMessage message, int index) {
+        return getMapField(message).getList().get(index);
+      }
+
+      public Object getRepeated(Builder builder, int index) {
+        return getMapField(builder).getList().get(index);
+      }
+
+      public void setRepeated(Builder builder, int index, Object value) {
+        getMapField(builder).getMutableList().set(index, (Message) value);
+      }
+
+      public void addRepeated(Builder builder, Object value) {
+        getMapField(builder).getMutableList().add((Message) value);
+      }
+
+      public boolean has(GeneratedMessage message) {
+        throw new UnsupportedOperationException(
+            "hasField() is not supported for repeated fields.");
+      }
+
+      public boolean has(Builder builder) {
+        throw new UnsupportedOperationException(
+            "hasField() is not supported for repeated fields.");
+      }
+
+      public int getRepeatedCount(GeneratedMessage message) {
+        return getMapField(message).getList().size();
+      }
+
+      public int getRepeatedCount(Builder builder) {
+        return getMapField(builder).getList().size();
+      }
+
+      public void clear(Builder builder) {
+        getMapField(builder).getMutableList().clear();
+      }
+
+      public com.google.protobuf.Message.Builder newBuilder() {
+        return mapEntryMessageDefaultInstance.newBuilderForType();
+      }
+
+      public com.google.protobuf.Message.Builder getBuilder(Builder builder) {
+        throw new UnsupportedOperationException(
+            "Nested builder not supported for map fields.");
+      }
+      
+      public com.google.protobuf.Message.Builder getRepeatedBuilder(
+          Builder builder, int index) {
+        throw new UnsupportedOperationException(
+            "Nested builder not supported for map fields.");
+      }
     }
 
     // ---------------------------------------------------------------
@@ -2026,28 +2226,60 @@ public abstract class GeneratedMessage extends AbstractMessage
           final Class<? extends Builder> builderClass,
           final String containingOneofCamelCaseName) {
         super(descriptor, camelCaseName, messageClass, builderClass, containingOneofCamelCaseName);
+        
+        enumDescriptor = descriptor.getEnumType();
 
         valueOfMethod = getMethodOrDie(type, "valueOf",
                                        EnumValueDescriptor.class);
         getValueDescriptorMethod =
           getMethodOrDie(type, "getValueDescriptor");
+
+        supportUnknownEnumValue = descriptor.getFile().supportsUnknownEnumValue();
+        if (supportUnknownEnumValue) {
+          getValueMethod =
+              getMethodOrDie(messageClass, "get" + camelCaseName + "Value");
+          getValueMethodBuilder =
+              getMethodOrDie(builderClass, "get" + camelCaseName + "Value");
+          setValueMethod =
+              getMethodOrDie(builderClass, "set" + camelCaseName + "Value", int.class);
+        }
       }
+      
+      private EnumDescriptor enumDescriptor;
 
       private Method valueOfMethod;
       private Method getValueDescriptorMethod;
+      
+      private boolean supportUnknownEnumValue;
+      private Method getValueMethod;
+      private Method getValueMethodBuilder;
+      private Method setValueMethod;
 
       @Override
       public Object get(final GeneratedMessage message) {
+        if (supportUnknownEnumValue) {
+          int value = (Integer) invokeOrDie(getValueMethod, message);
+          return enumDescriptor.findValueByNumberCreatingIfUnknown(value);
+        }
         return invokeOrDie(getValueDescriptorMethod, super.get(message));
       }
 
       @Override
       public Object get(final GeneratedMessage.Builder builder) {
+        if (supportUnknownEnumValue) {
+          int value = (Integer) invokeOrDie(getValueMethodBuilder, builder);
+          return enumDescriptor.findValueByNumberCreatingIfUnknown(value);
+        }
         return invokeOrDie(getValueDescriptorMethod, super.get(builder));
       }
 
       @Override
       public void set(final Builder builder, final Object value) {
+        if (supportUnknownEnumValue) {
+          invokeOrDie(setValueMethod, builder,
+              ((EnumValueDescriptor) value).getNumber());
+          return;
+        }
         super.set(builder, invokeOrDie(valueOfMethod, null, value));
       }
     }
@@ -2059,22 +2291,44 @@ public abstract class GeneratedMessage extends AbstractMessage
           final Class<? extends GeneratedMessage> messageClass,
           final Class<? extends Builder> builderClass) {
         super(descriptor, camelCaseName, messageClass, builderClass);
+        
+        enumDescriptor = descriptor.getEnumType();
 
         valueOfMethod = getMethodOrDie(type, "valueOf",
                                        EnumValueDescriptor.class);
         getValueDescriptorMethod =
           getMethodOrDie(type, "getValueDescriptor");
+
+        supportUnknownEnumValue = descriptor.getFile().supportsUnknownEnumValue();
+        if (supportUnknownEnumValue) {
+          getRepeatedValueMethod =
+              getMethodOrDie(messageClass, "get" + camelCaseName + "Value", int.class);
+          getRepeatedValueMethodBuilder =
+              getMethodOrDie(builderClass, "get" + camelCaseName + "Value", int.class);
+          setRepeatedValueMethod =
+              getMethodOrDie(builderClass, "set" + camelCaseName + "Value", int.class, int.class);
+          addRepeatedValueMethod =
+              getMethodOrDie(builderClass, "add" + camelCaseName + "Value", int.class);
+        }
       }
+      private EnumDescriptor enumDescriptor;
 
       private final Method valueOfMethod;
       private final Method getValueDescriptorMethod;
+      
+      private boolean supportUnknownEnumValue;
+      private Method getRepeatedValueMethod;
+      private Method getRepeatedValueMethodBuilder;
+      private Method setRepeatedValueMethod;
+      private Method addRepeatedValueMethod;
 
       @Override
       @SuppressWarnings("unchecked")
       public Object get(final GeneratedMessage message) {
         final List newList = new ArrayList();
-        for (final Object element : (List) super.get(message)) {
-          newList.add(invokeOrDie(getValueDescriptorMethod, element));
+        final int size = getRepeatedCount(message);
+        for (int i = 0; i < size; i++) {
+          newList.add(getRepeated(message, i));
         }
         return Collections.unmodifiableList(newList);
       }
@@ -2083,8 +2337,9 @@ public abstract class GeneratedMessage extends AbstractMessage
       @SuppressWarnings("unchecked")
       public Object get(final GeneratedMessage.Builder builder) {
         final List newList = new ArrayList();
-        for (final Object element : (List) super.get(builder)) {
-          newList.add(invokeOrDie(getValueDescriptorMethod, element));
+        final int size = getRepeatedCount(builder);
+        for (int i = 0; i < size; i++) {
+          newList.add(getRepeated(builder, i));
         }
         return Collections.unmodifiableList(newList);
       }
@@ -2092,23 +2347,41 @@ public abstract class GeneratedMessage extends AbstractMessage
       @Override
       public Object getRepeated(final GeneratedMessage message,
                                 final int index) {
+        if (supportUnknownEnumValue) {
+          int value = (Integer) invokeOrDie(getRepeatedValueMethod, message, index);
+          return enumDescriptor.findValueByNumberCreatingIfUnknown(value);
+        }
         return invokeOrDie(getValueDescriptorMethod,
           super.getRepeated(message, index));
       }
       @Override
       public Object getRepeated(final GeneratedMessage.Builder builder,
                                 final int index) {
+        if (supportUnknownEnumValue) {
+          int value = (Integer) invokeOrDie(getRepeatedValueMethodBuilder, builder, index);
+          return enumDescriptor.findValueByNumberCreatingIfUnknown(value);
+        }
         return invokeOrDie(getValueDescriptorMethod,
           super.getRepeated(builder, index));
       }
       @Override
       public void setRepeated(final Builder builder,
                               final int index, final Object value) {
+        if (supportUnknownEnumValue) {
+          invokeOrDie(setRepeatedValueMethod, builder, index,
+              ((EnumValueDescriptor) value).getNumber());
+          return;
+        }
         super.setRepeated(builder, index, invokeOrDie(valueOfMethod, null,
                           value));
       }
       @Override
       public void addRepeated(final Builder builder, final Object value) {
+        if (supportUnknownEnumValue) {
+          invokeOrDie(addRepeatedValueMethod, builder,
+              ((EnumValueDescriptor) value).getNumber());
+          return;
+        }
         super.addRepeated(builder, invokeOrDie(valueOfMethod, null, value));
       }
     }
@@ -2168,9 +2441,12 @@ public abstract class GeneratedMessage extends AbstractMessage
         super(descriptor, camelCaseName, messageClass, builderClass);
 
         newBuilderMethod = getMethodOrDie(type, "newBuilder");
+        getBuilderMethodBuilder = getMethodOrDie(builderClass,
+            "get" + camelCaseName + "Builder", Integer.TYPE);
       }
 
       private final Method newBuilderMethod;
+      private final Method getBuilderMethodBuilder;
 
       private Object coerceType(final Object value) {
         if (type.isInstance(value)) {
@@ -2198,6 +2474,12 @@ public abstract class GeneratedMessage extends AbstractMessage
       public Message.Builder newBuilder() {
         return (Message.Builder) invokeOrDie(newBuilderMethod, null);
       }
+      @Override
+      public Message.Builder getRepeatedBuilder(
+          final GeneratedMessage.Builder builder, final int index) {
+        return (Message.Builder) invokeOrDie(
+            getBuilderMethodBuilder, builder, index);
+      }
     }
   }
 
@@ -2210,4 +2492,18 @@ public abstract class GeneratedMessage extends AbstractMessage
   protected Object writeReplace() throws ObjectStreamException {
     return new GeneratedMessageLite.SerializedForm(this);
   }
+  
+  /**
+   * Checks that the {@link Extension} is non-Lite and returns it as a
+   * {@link GeneratedExtension}.
+   */
+  private static <MessageType extends ExtendableMessage<MessageType>, T>
+    Extension<MessageType, T> checkNotLite(
+        ExtensionLite<MessageType, T> extension) {
+    if (extension.isLite()) {
+      throw new IllegalArgumentException("Expected non-lite extension.");
+    }
+    
+    return (Extension<MessageType, T>) extension;
+  }
 }

java/src/main/java/com/google/protobuf/GeneratedMessageLite.java

@@ -30,6 +30,8 @@
 
 package com.google.protobuf;
 
+import com.google.protobuf.WireFormat.FieldType;
+
 import java.io.IOException;
 import java.io.ObjectStreamException;
 import java.io.Serializable;
@@ -50,10 +52,15 @@ public abstract class GeneratedMessageLite extends AbstractMessageLite
     implements Serializable {
   private static final long serialVersionUID = 1L;
 
+  /** For use by generated code only.  */
+  protected UnknownFieldSetLite unknownFields;
+  
   protected GeneratedMessageLite() {
+    unknownFields = UnknownFieldSetLite.getDefaultInstance();
   }
 
   protected GeneratedMessageLite(Builder builder) {
+    unknownFields = builder.unknownFields;
   }
 
   public Parser<? extends MessageLite> getParserForType() {
@@ -62,15 +69,16 @@ public abstract class GeneratedMessageLite extends AbstractMessageLite
   }
 
   /**
-   * Called by subclasses to parse an unknown field.
+   * Called by subclasses to parse an unknown field. For use by generated code
+   * only.
    * @return {@code true} unless the tag is an end-group tag.
    */
-  protected boolean parseUnknownField(
+  protected static boolean parseUnknownField(
       CodedInputStream input,
-      CodedOutputStream unknownFieldsCodedOutput,
+      UnknownFieldSetLite.Builder unknownFields,
       ExtensionRegistryLite extensionRegistry,
       int tag) throws IOException {
-    return input.skipField(tag, unknownFieldsCodedOutput);
+    return unknownFields.mergeFieldFrom(tag, input);
   }
 
   /**
@@ -84,22 +92,28 @@ public abstract class GeneratedMessageLite extends AbstractMessageLite
   public abstract static class Builder<MessageType extends GeneratedMessageLite,
                                        BuilderType extends Builder>
       extends AbstractMessageLite.Builder<BuilderType> {
+
+    private UnknownFieldSetLite unknownFields =
+        UnknownFieldSetLite.getDefaultInstance();
+
     protected Builder() {}
 
     //@Override (Java 1.6 override semantics, but we must support 1.5)
     public BuilderType clear() {
-      unknownFields = ByteString.EMPTY;
+      unknownFields = UnknownFieldSetLite.getDefaultInstance();
       return (BuilderType) this;
     }
 
-    // This is implemented here only to work around an apparent bug in the
-    // Java compiler and/or build system.  See bug #1898463.  The mere presence
-    // of this dummy clone() implementation makes it go away.
-    @Override
+    //@Override (Java 1.6 override semantics, but we must support 1.5)
     public BuilderType clone() {
-      throw new UnsupportedOperationException(
-          "This is supposed to be overridden by subclasses.");
+      BuilderType builder =
+          (BuilderType) getDefaultInstanceForType().newBuilderForType();
+      builder.mergeFrom(buildPartial());
+      return builder;
     }
+    
+    /** All subclasses implement this. */
+    public abstract MessageType buildPartial();
 
     /** All subclasses implement this. */
     public abstract BuilderType mergeFrom(MessageType message);
@@ -113,22 +127,43 @@ public abstract class GeneratedMessageLite extends AbstractMessageLite
      */
     protected boolean parseUnknownField(
         CodedInputStream input,
-        CodedOutputStream unknownFieldsCodedOutput,
+        UnknownFieldSetLite.Builder unknownFields,
         ExtensionRegistryLite extensionRegistry,
         int tag) throws IOException {
-      return input.skipField(tag, unknownFieldsCodedOutput);
+      return unknownFields.mergeFieldFrom(tag, input);
     }
 
-    public final ByteString getUnknownFields() {
-      return unknownFields;
+    /**
+     * Merge some unknown fields into the {@link UnknownFieldSetLite} for this
+     * message.
+     *
+     * <p>For use by generated code only.
+     */
+    protected final BuilderType mergeUnknownFields(
+        final UnknownFieldSetLite unknownFields) {
+      this.unknownFields = UnknownFieldSetLite.concat(this.unknownFields, unknownFields);
+      return (BuilderType) this;
     }
-
-    public final BuilderType setUnknownFields(final ByteString unknownFields) {
-      this.unknownFields = unknownFields;
+    
+    public BuilderType mergeFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      MessageType parsedMessage = null;
+      try {
+        parsedMessage =
+            (MessageType) getDefaultInstanceForType().getParserForType().parsePartialFrom(
+                input, extensionRegistry);
+      } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+        parsedMessage = (MessageType) e.getUnfinishedMessage();
+        throw e;
+      } finally {
+        if (parsedMessage != null) {
+          mergeFrom(parsedMessage);
+        }
+      }
       return (BuilderType) this;
     }
-
-    private ByteString unknownFields = ByteString.EMPTY;
   }
 
 
@@ -143,18 +178,18 @@ public abstract class GeneratedMessageLite extends AbstractMessageLite
 
     /** Check if a singular extension is present. */
     <Type> boolean hasExtension(
-        GeneratedExtension<MessageType, Type> extension);
+        ExtensionLite<MessageType, Type> extension);
 
     /** Get the number of elements in a repeated extension. */
     <Type> int getExtensionCount(
-        GeneratedExtension<MessageType, List<Type>> extension);
+        ExtensionLite<MessageType, List<Type>> extension);
 
     /** Get the value of an extension. */
-    <Type> Type getExtension(GeneratedExtension<MessageType, Type> extension);
+    <Type> Type getExtension(ExtensionLite<MessageType, Type> extension);
 
     /** Get one element of a repeated extension. */
     <Type> Type getExtension(
-        GeneratedExtension<MessageType, List<Type>> extension,
+        ExtensionLite<MessageType, List<Type>> extension,
         int index);
   }
 
@@ -166,7 +201,11 @@ public abstract class GeneratedMessageLite extends AbstractMessageLite
       extends GeneratedMessageLite
       implements ExtendableMessageOrBuilder<MessageType> {
 
-    private final FieldSet<ExtensionDescriptor> extensions;
+    /**
+     * Represents the set of extensions on this message. For use by generated
+     * code only.
+     */
+    protected final FieldSet<ExtensionDescriptor> extensions;
 
     protected ExtendableMessage() {
       this.extensions = FieldSet.newFieldSet();
@@ -190,30 +229,39 @@ public abstract class GeneratedMessageLite extends AbstractMessageLite
     /** Check if a singular extension is present. */
     //@Override (Java 1.6 override semantics, but we must support 1.5)
     public final <Type> boolean hasExtension(
-        final GeneratedExtension<MessageType, Type> extension) {
-      verifyExtensionContainingType(extension);
-      return extensions.hasField(extension.descriptor);
+        final ExtensionLite<MessageType, Type> extension) {
+      GeneratedExtension<MessageType, Type> extensionLite =
+          checkIsLite(extension);
+      
+      verifyExtensionContainingType(extensionLite);
+      return extensions.hasField(extensionLite.descriptor);
     }
 
     /** Get the number of elements in a repeated extension. */
     //@Override (Java 1.6 override semantics, but we must support 1.5)
     public final <Type> int getExtensionCount(
-        final GeneratedExtension<MessageType, List<Type>> extension) {
-      verifyExtensionContainingType(extension);
-      return extensions.getRepeatedFieldCount(extension.descriptor);
+        final ExtensionLite<MessageType, List<Type>> extension) {
+      GeneratedExtension<MessageType, List<Type>> extensionLite =
+          checkIsLite(extension);
+      
+      verifyExtensionContainingType(extensionLite);
+      return extensions.getRepeatedFieldCount(extensionLite.descriptor);
     }
 
     /** Get the value of an extension. */
     //@Override (Java 1.6 override semantics, but we must support 1.5)
     @SuppressWarnings("unchecked")
     public final <Type> Type getExtension(
-        final GeneratedExtension<MessageType, Type> extension) {
-      verifyExtensionContainingType(extension);
-      final Object value = extensions.getField(extension.descriptor);
+        final ExtensionLite<MessageType, Type> extension) {
+      GeneratedExtension<MessageType, Type> extensionLite =
+          checkIsLite(extension);
+      
+      verifyExtensionContainingType(extensionLite);
+      final Object value = extensions.getField(extensionLite.descriptor);
       if (value == null) {
-        return extension.defaultValue;
+        return extensionLite.defaultValue;
       } else {
-        return (Type) extension.fromFieldSetType(value);
+        return (Type) extensionLite.fromFieldSetType(value);
       }
     }
 
@@ -221,11 +269,14 @@ public abstract class GeneratedMessageLite extends AbstractMessageLite
     //@Override (Java 1.6 override semantics, but we must support 1.5)
     @SuppressWarnings("unchecked")
     public final <Type> Type getExtension(
-        final GeneratedExtension<MessageType, List<Type>> extension,
+        final ExtensionLite<MessageType, List<Type>> extension,
         final int index) {
-      verifyExtensionContainingType(extension);
-      return (Type) extension.singularFromFieldSetType(
-          extensions.getRepeatedField(extension.descriptor, index));
+      GeneratedExtension<MessageType, List<Type>> extensionLite =
+          checkIsLite(extension);
+
+      verifyExtensionContainingType(extensionLite);
+      return (Type) extensionLite.singularFromFieldSetType(
+          extensions.getRepeatedField(extensionLite.descriptor, index));
     }
 
     /** Called by subclasses to check if all extensions are initialized. */
@@ -233,25 +284,6 @@ public abstract class GeneratedMessageLite extends AbstractMessageLite
       return extensions.isInitialized();
     }
 
-    /**
-     * Called by subclasses to parse an unknown field or an extension.
-     * @return {@code true} unless the tag is an end-group tag.
-     */
-    @Override
-    protected boolean parseUnknownField(
-        CodedInputStream input,
-        CodedOutputStream unknownFieldsCodedOutput,
-        ExtensionRegistryLite extensionRegistry,
-        int tag) throws IOException {
-      return GeneratedMessageLite.parseUnknownField(
-          extensions,
-          getDefaultInstanceForType(),
-          input,
-          unknownFieldsCodedOutput,
-          extensionRegistry,
-          tag);
-    }
-
 
     /**
      * Used by parsing constructors in generated classes.
@@ -377,30 +409,39 @@ public abstract class GeneratedMessageLite extends AbstractMessageLite
     /** Check if a singular extension is present. */
     //@Override (Java 1.6 override semantics, but we must support 1.5)
     public final <Type> boolean hasExtension(
-        final GeneratedExtension<MessageType, Type> extension) {
-      verifyExtensionContainingType(extension);
-      return extensions.hasField(extension.descriptor);
+        final ExtensionLite<MessageType, Type> extension) {
+      GeneratedExtension<MessageType, Type> extensionLite =
+          checkIsLite(extension);
+      
+      verifyExtensionContainingType(extensionLite);
+      return extensions.hasField(extensionLite.descriptor);
     }
 
     /** Get the number of elements in a repeated extension. */
     //@Override (Java 1.6 override semantics, but we must support 1.5)
     public final <Type> int getExtensionCount(
-        final GeneratedExtension<MessageType, List<Type>> extension) {
-      verifyExtensionContainingType(extension);
-      return extensions.getRepeatedFieldCount(extension.descriptor);
+        final ExtensionLite<MessageType, List<Type>> extension) {
+      GeneratedExtension<MessageType, List<Type>> extensionLite =
+          checkIsLite(extension);
+      
+      verifyExtensionContainingType(extensionLite);
+      return extensions.getRepeatedFieldCount(extensionLite.descriptor);
     }
 
     /** Get the value of an extension. */
     //@Override (Java 1.6 override semantics, but we must support 1.5)
     @SuppressWarnings("unchecked")
     public final <Type> Type getExtension(
-        final GeneratedExtension<MessageType, Type> extension) {
-      verifyExtensionContainingType(extension);
-      final Object value = extensions.getField(extension.descriptor);
+        final ExtensionLite<MessageType, Type> extension) {
+      GeneratedExtension<MessageType, Type> extensionLite =
+          checkIsLite(extension);
+      
+      verifyExtensionContainingType(extensionLite);
+      final Object value = extensions.getField(extensionLite.descriptor);
       if (value == null) {
-        return extension.defaultValue;
+        return extensionLite.defaultValue;
       } else {
-        return (Type) extension.fromFieldSetType(value);
+        return (Type) extensionLite.fromFieldSetType(value);
       }
     }
 
@@ -408,11 +449,14 @@ public abstract class GeneratedMessageLite extends AbstractMessageLite
     @SuppressWarnings("unchecked")
     //@Override (Java 1.6 override semantics, but we must support 1.5)
     public final <Type> Type getExtension(
-        final GeneratedExtension<MessageType, List<Type>> extension,
+        final ExtensionLite<MessageType, List<Type>> extension,
         final int index) {
-      verifyExtensionContainingType(extension);
-      return (Type) extension.singularFromFieldSetType(
-          extensions.getRepeatedField(extension.descriptor, index));
+      GeneratedExtension<MessageType, List<Type>> extensionLite =
+          checkIsLite(extension);
+      
+      verifyExtensionContainingType(extensionLite);
+      return (Type) extensionLite.singularFromFieldSetType(
+          extensions.getRepeatedField(extensionLite.descriptor, index));
     }
 
     // This is implemented here only to work around an apparent bug in the
@@ -423,46 +467,57 @@ public abstract class GeneratedMessageLite extends AbstractMessageLite
       throw new UnsupportedOperationException(
           "This is supposed to be overridden by subclasses.");
     }
-
+    
     /** Set the value of an extension. */
     public final <Type> BuilderType setExtension(
-        final GeneratedExtension<MessageType, Type> extension,
+        final ExtensionLite<MessageType, Type> extension,
         final Type value) {
-      verifyExtensionContainingType(extension);
+      GeneratedExtension<MessageType, Type> extensionLite =
+          checkIsLite(extension);
+      
+      verifyExtensionContainingType(extensionLite);
       ensureExtensionsIsMutable();
-      extensions.setField(extension.descriptor,
-                          extension.toFieldSetType(value));
+      extensions.setField(extensionLite.descriptor,
+                          extensionLite.toFieldSetType(value));
       return (BuilderType) this;
     }
 
     /** Set the value of one element of a repeated extension. */
     public final <Type> BuilderType setExtension(
-        final GeneratedExtension<MessageType, List<Type>> extension,
+        final ExtensionLite<MessageType, List<Type>> extension,
         final int index, final Type value) {
-      verifyExtensionContainingType(extension);
+      GeneratedExtension<MessageType, List<Type>> extensionLite =
+          checkIsLite(extension);
+      
+      verifyExtensionContainingType(extensionLite);
       ensureExtensionsIsMutable();
-      extensions.setRepeatedField(extension.descriptor, index,
-                                  extension.singularToFieldSetType(value));
+      extensions.setRepeatedField(extensionLite.descriptor, index,
+                                  extensionLite.singularToFieldSetType(value));
       return (BuilderType) this;
     }
 
     /** Append a value to a repeated extension. */
     public final <Type> BuilderType addExtension(
-        final GeneratedExtension<MessageType, List<Type>> extension,
+        final ExtensionLite<MessageType, List<Type>> extension,
         final Type value) {
-      verifyExtensionContainingType(extension);
+      GeneratedExtension<MessageType, List<Type>> extensionLite =
+          checkIsLite(extension);
+      
+      verifyExtensionContainingType(extensionLite);
       ensureExtensionsIsMutable();
-      extensions.addRepeatedField(extension.descriptor,
-                                  extension.singularToFieldSetType(value));
+      extensions.addRepeatedField(extensionLite.descriptor,
+                                  extensionLite.singularToFieldSetType(value));
       return (BuilderType) this;
     }
 
     /** Clear an extension. */
     public final <Type> BuilderType clearExtension(
-        final GeneratedExtension<MessageType, ?> extension) {
-      verifyExtensionContainingType(extension);
+        final ExtensionLite<MessageType, ?> extension) {
+      GeneratedExtension<MessageType, ?> extensionLite = checkIsLite(extension);
+      
+      verifyExtensionContainingType(extensionLite);
       ensureExtensionsIsMutable();
-      extensions.clearField(extension.descriptor);
+      extensions.clearField(extensionLite.descriptor);
       return (BuilderType) this;
     }
 
@@ -471,44 +526,24 @@ public abstract class GeneratedMessageLite extends AbstractMessageLite
       return extensions.isInitialized();
     }
 
-    /**
-     * Called by subclasses to parse an unknown field or an extension.
-     * @return {@code true} unless the tag is an end-group tag.
-     */
-    @Override
-    protected boolean parseUnknownField(
-        CodedInputStream input,
-        CodedOutputStream unknownFieldsCodedOutput,
-        ExtensionRegistryLite extensionRegistry,
-        int tag) throws IOException {
-      ensureExtensionsIsMutable();
-      return GeneratedMessageLite.parseUnknownField(
-          extensions,
-          getDefaultInstanceForType(),
-          input,
-          unknownFieldsCodedOutput,
-          extensionRegistry,
-          tag);
-    }
-
     protected final void mergeExtensionFields(final MessageType other) {
       ensureExtensionsIsMutable();
       extensions.mergeFrom(((ExtendableMessage) other).extensions);
     }
   }
 
-  // -----------------------------------------------------------------
+  //-----------------------------------------------------------------
 
   /**
-   * Parse an unknown field or an extension.
+   * Parse an unknown field or an extension. For use by generated code only.
    * @return {@code true} unless the tag is an end-group tag.
    */
-  private static <MessageType extends MessageLite>
+  protected static <MessageType extends MessageLite>
       boolean parseUnknownField(
           FieldSet<ExtensionDescriptor> extensions,
           MessageType defaultInstance,
           CodedInputStream input,
-          CodedOutputStream unknownFieldsCodedOutput,
+          UnknownFieldSetLite.Builder unknownFields,
           ExtensionRegistryLite extensionRegistry,
           int tag) throws IOException {
     int wireType = WireFormat.getTagWireType(tag);
@@ -537,7 +572,7 @@ public abstract class GeneratedMessageLite extends AbstractMessageLite
     }
 
     if (unknown) {  // Unknown field or wrong wire type.  Skip.
-      return input.skipField(tag, unknownFieldsCodedOutput);
+      return unknownFields.mergeFieldFrom(tag, input);
     }
 
     if (packed) {
@@ -599,8 +634,7 @@ public abstract class GeneratedMessageLite extends AbstractMessageLite
           // If the number isn't recognized as a valid value for this enum,
           // write it to unknown fields object.
           if (value == null) {
-            unknownFieldsCodedOutput.writeRawVarint32(tag);
-            unknownFieldsCodedOutput.writeUInt32NoTag(rawValue);
+            unknownFields.mergeVarintField(fieldNumber, rawValue);
             return true;
           }
           break;
@@ -768,7 +802,8 @@ public abstract class GeneratedMessageLite extends AbstractMessageLite
    * this type as parameters to extension accessors and ExtensionRegistry.add().
    */
   public static class GeneratedExtension<
-      ContainingType extends MessageLite, Type> {
+      ContainingType extends MessageLite, Type>
+          extends ExtensionLite<ContainingType, Type> {
 
     /**
      * Create a new isntance with the given parameters.
@@ -888,6 +923,18 @@ public abstract class GeneratedMessageLite extends AbstractMessageLite
         return value;
       }
     }
+
+    public FieldType getLiteType() {
+      return descriptor.getLiteType();
+    }
+
+    public boolean isRepeated() {
+      return descriptor.isRepeated;
+    }
+
+    public Type getDefaultValue() {
+      return defaultValue;
+    }
   }
 
   /**
@@ -897,8 +944,8 @@ public abstract class GeneratedMessageLite extends AbstractMessageLite
   static final class SerializedForm implements Serializable {
     private static final long serialVersionUID = 0L;
 
-    private String messageClassName;
-    private byte[] asBytes;
+    private final String messageClassName;
+    private final byte[] asBytes;
 
     /**
      * Creates the serialized form by calling {@link com.google.protobuf.MessageLite#toByteArray}.
@@ -918,19 +965,17 @@ public abstract class GeneratedMessageLite extends AbstractMessageLite
     protected Object readResolve() throws ObjectStreamException {
       try {
         Class messageClass = Class.forName(messageClassName);
-        Method newBuilder = messageClass.getMethod("newBuilder");
-        MessageLite.Builder builder =
-            (MessageLite.Builder) newBuilder.invoke(null);
-        builder.mergeFrom(asBytes);
-        return builder.buildPartial();
+        Parser<?> parser =
+            (Parser<?>) messageClass.getField("PARSER").get(null);
+        return parser.parsePartialFrom(asBytes);
       } catch (ClassNotFoundException e) {
         throw new RuntimeException("Unable to find proto buffer class", e);
-      } catch (NoSuchMethodException e) {
-        throw new RuntimeException("Unable to find newBuilder method", e);
+      } catch (NoSuchFieldException e) {
+        throw new RuntimeException("Unable to find PARSER", e);
+      } catch (SecurityException e) {
+        throw new RuntimeException("Unable to call PARSER", e);
       } catch (IllegalAccessException e) {
-        throw new RuntimeException("Unable to call newBuilder method", e);
-      } catch (InvocationTargetException e) {
-        throw new RuntimeException("Error calling newBuilder", e.getCause());
+        throw new RuntimeException("Unable to call parseFrom method", e);
       } catch (InvalidProtocolBufferException e) {
         throw new RuntimeException("Unable to understand proto buffer", e);
       }
@@ -946,4 +991,18 @@ public abstract class GeneratedMessageLite extends AbstractMessageLite
   protected Object writeReplace() throws ObjectStreamException {
     return new SerializedForm(this);
   }
+  
+  /**
+   * Checks that the {@link Extension} is Lite and returns it as a
+   * {@link GeneratedExtension}.
+   */
+  private static <MessageType extends ExtendableMessage<MessageType>, T>
+    GeneratedExtension<MessageType, T> checkIsLite(
+        ExtensionLite<MessageType, T> extension) {
+    if (!extension.isLite()) {
+      throw new IllegalArgumentException("Expected a lite extension.");
+    }
+    
+    return (GeneratedExtension<MessageType, T>) extension;
+  }
 }

java/src/main/java/com/google/protobuf/Internal.java

@@ -33,8 +33,14 @@ package com.google.protobuf;
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
 import java.nio.ByteBuffer;
+import java.util.AbstractList;
+import java.util.AbstractMap;
+import java.util.AbstractSet;
 import java.util.Arrays;
+import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
 
 /**
  * The classes contained within are used internally by the Protocol Buffer
@@ -388,4 +394,164 @@ public class Internal {
   public static final ByteBuffer EMPTY_BYTE_BUFFER =
       ByteBuffer.wrap(EMPTY_BYTE_ARRAY);
 
+
+  /**
+   * Provides an immutable view of List<T> around a List<F>.
+   *
+   * Protobuf internal. Used in protobuf generated code only.
+   */
+  public static class ListAdapter<F, T> extends AbstractList<T> {
+    /**
+     * Convert individual elements of the List from F to T.
+     */
+    public interface Converter<F, T> {
+      T convert(F from);
+    }
+
+    private final List<F> fromList;
+    private final Converter<F, T> converter;
+
+    public ListAdapter(List<F> fromList, Converter<F, T> converter) {
+      this.fromList = fromList;
+      this.converter = converter;
+    }
+
+    @Override
+    public T get(int index) {
+      return converter.convert(fromList.get(index));
+    }
+
+    @Override
+    public int size() {
+      return fromList.size();
+    }
+  }
+
+  /**
+   * Wrap around a Map<K, RealValue> and provide a Map<K, V> interface.
+   */
+  public static class MapAdapter<K, V, RealValue> extends AbstractMap<K, V> {
+    /**
+     * An interface used to convert between two types.
+     */
+    public interface Converter<A, B> {
+      B doForward(A object);
+      A doBackward(B object);
+    }
+
+    public static <T extends EnumLite> Converter<Integer, T> newEnumConverter(
+        final EnumLiteMap<T> enumMap, final T unrecognizedValue) {
+      return new Converter<Integer, T>() {
+        public T doForward(Integer value) {
+          T result = enumMap.findValueByNumber(value);
+          return result == null ? unrecognizedValue : result;
+        }
+        public Integer doBackward(T value) {
+          return value.getNumber();
+        }
+      };
+    }
+
+    private final Map<K, RealValue> realMap;
+    private final Converter<RealValue, V> valueConverter;
+    
+    public MapAdapter(Map<K, RealValue> realMap,
+        Converter<RealValue, V> valueConverter) {
+      this.realMap = realMap;
+      this.valueConverter = valueConverter;
+    }
+    
+    @SuppressWarnings("unchecked")
+    @Override
+    public V get(Object key) {
+      RealValue result = realMap.get(key);
+      if (result == null) {
+        return null;
+      }
+      return valueConverter.doForward(result);
+    }
+    
+    @Override
+    public V put(K key, V value) {
+      RealValue oldValue = realMap.put(key, valueConverter.doBackward(value));
+      if (oldValue == null) {
+        return null;
+      }
+      return valueConverter.doForward(oldValue);
+    }
+
+    @Override
+    public Set<java.util.Map.Entry<K, V>> entrySet() {
+      return new SetAdapter(realMap.entrySet());
+    }
+    
+    private class SetAdapter extends AbstractSet<Map.Entry<K, V>> {
+      private final Set<Map.Entry<K, RealValue>> realSet;
+      public SetAdapter(Set<Map.Entry<K, RealValue>> realSet) {
+        this.realSet = realSet;
+      }
+      
+      @Override
+      public Iterator<java.util.Map.Entry<K, V>> iterator() {
+        return new IteratorAdapter(realSet.iterator());
+      }
+
+      @Override
+      public int size() {
+        return realSet.size();
+      } 
+    }
+    
+    private class IteratorAdapter implements Iterator<Map.Entry<K, V>> {
+      private final Iterator<Map.Entry<K, RealValue>> realIterator;
+      
+      public IteratorAdapter(
+          Iterator<Map.Entry<K, RealValue>> realIterator) {
+        this.realIterator = realIterator;
+      }
+      
+      @Override
+      public boolean hasNext() {
+        return realIterator.hasNext();
+      }
+
+      @Override
+      public java.util.Map.Entry<K, V> next() {
+        return new EntryAdapter(realIterator.next());
+      }
+
+      @Override
+      public void remove() {
+        realIterator.remove();
+      }
+    }
+    
+    private class EntryAdapter implements Map.Entry<K, V> {
+      private final Map.Entry<K, RealValue> realEntry;
+      
+      public EntryAdapter(Map.Entry<K, RealValue> realEntry) {
+        this.realEntry = realEntry;
+      }
+      
+      @Override
+      public K getKey() {
+        return realEntry.getKey();
+      }
+
+      @Override
+      public V getValue() {
+        return valueConverter.doForward(realEntry.getValue());
+      }
+
+      @Override
+      public V setValue(V value) {
+        RealValue oldValue = realEntry.setValue(
+            valueConverter.doBackward(value));
+        if (oldValue == null) {
+          return null;
+        }
+        return valueConverter.doForward(oldValue);
+      }
+    }
+  }
 }

java/src/main/java/com/google/protobuf/LazyStringArrayList.java

@@ -74,6 +74,10 @@ public class LazyStringArrayList extends AbstractList<String>
     list = new ArrayList<Object>();
   }
 
+  public LazyStringArrayList(int intialCapacity) {
+    list = new ArrayList<Object>(intialCapacity);
+  }
+
   public LazyStringArrayList(LazyStringList from) {
     list = new ArrayList<Object>(from.size());
     addAll(from);

java/src/main/java/com/google/protobuf/LiteralByteString.java

@@ -190,6 +190,15 @@ class LiteralByteString extends ByteString {
     }
 
     if (other instanceof LiteralByteString) {
+      LiteralByteString otherAsLiteral = (LiteralByteString) other;
+      // If we know the hash codes and they are not equal, we know the byte
+      // strings are not equal.
+      if (hash != 0 
+          && otherAsLiteral.hash != 0 
+          && hash != otherAsLiteral.hash) {
+        return false;
+      }
+      
       return equalsRange((LiteralByteString) other, 0, size());
     } else if (other instanceof RopeByteString) {
       return other.equals(this);

java/src/main/java/com/google/protobuf/MapEntry.java

@@ -0,0 +1,433 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package com.google.protobuf;
+
+import com.google.protobuf.Descriptors.Descriptor;
+import com.google.protobuf.Descriptors.EnumValueDescriptor;
+import com.google.protobuf.Descriptors.FieldDescriptor;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+import java.util.TreeMap;
+
+/**
+ * Implements MapEntry messages.
+ * 
+ * In reflection API, map fields will be treated as repeated message fields and
+ * each map entry is accessed as a message. This MapEntry class is used to
+ * represent these map entry messages in reflection API.
+ * 
+ * Protobuf internal. Users shouldn't use this class.
+ */
+public final class MapEntry<K, V> extends AbstractMessage {
+  private static class Metadata<K, V> {
+    public final Descriptor descriptor;  
+    public final MapEntry<K, V> defaultInstance;
+    public final AbstractParser<MapEntry<K, V>> parser;
+    
+    public Metadata(
+        final Descriptor descriptor, final MapEntry<K, V> defaultInstance) {
+      this.descriptor = descriptor;
+      this.defaultInstance = defaultInstance;
+      final Metadata<K, V> thisMetadata = this;
+      this.parser = new AbstractParser<MapEntry<K, V>>() {
+        private final Parser<MapEntryLite<K, V>> dataParser =
+            defaultInstance.data.getParserForType();
+        @Override
+        public MapEntry<K, V> parsePartialFrom(
+            CodedInputStream input, ExtensionRegistryLite extensionRegistry)
+            throws InvalidProtocolBufferException {
+          MapEntryLite<K, V> data =
+              dataParser.parsePartialFrom(input, extensionRegistry);
+          return new MapEntry<K, V>(thisMetadata, data);
+        }
+        
+      };
+    }
+  }
+  
+  private final Metadata<K, V> metadata;
+  private final MapEntryLite<K, V> data;
+  
+  /** Create a default MapEntry instance. */
+  private MapEntry(Descriptor descriptor,
+      WireFormat.FieldType keyType, K defaultKey,
+      WireFormat.FieldType valueType, V defaultValue) {
+    this.data = MapEntryLite.newDefaultInstance(
+        keyType, defaultKey, valueType, defaultValue);
+    this.metadata = new Metadata<K, V>(descriptor, this); 
+  }
+  
+  /** Create a new MapEntry message. */
+  private MapEntry(Metadata<K, V> metadata, MapEntryLite<K, V> data) {
+    this.metadata = metadata;
+    this.data = data;
+  }
+  
+  /**
+   * Create a default MapEntry instance. A default MapEntry instance should be
+   * created only once for each map entry message type. Generated code should
+   * store the created default instance and use it later to create new MapEntry
+   * messages of the same type. 
+   */
+  public static <K, V> MapEntry<K, V> newDefaultInstance(
+      Descriptor descriptor,
+      WireFormat.FieldType keyType, K defaultKey,
+      WireFormat.FieldType valueType, V defaultValue) {
+    return new MapEntry<K, V>(
+        descriptor, keyType, defaultKey, valueType, defaultValue);
+  }
+  
+  public K getKey() {
+    return data.getKey();
+  }
+  
+  public V getValue() {
+    return data.getValue();
+  }
+  
+  @Override
+  public int getSerializedSize() {
+    return data.getSerializedSize();
+  }
+  
+  @Override
+  public void writeTo(CodedOutputStream output) throws IOException {
+    data.writeTo(output);
+  }
+  
+  @Override
+  public boolean isInitialized() {
+    return data.isInitialized();
+  }
+  
+  @Override
+  public Parser<MapEntry<K, V>> getParserForType() {
+    return metadata.parser;
+  }
+
+  @Override
+  public Builder<K, V> newBuilderForType() {
+    return new Builder<K, V>(metadata);
+  }
+  
+  @Override
+  public Builder<K, V> toBuilder() {
+    return new Builder<K, V>(metadata, data);
+  }
+
+  @Override
+  public MapEntry<K, V> getDefaultInstanceForType() {
+    return metadata.defaultInstance;
+  }
+
+  @Override
+  public Descriptor getDescriptorForType() {
+    return metadata.descriptor;
+  }
+
+  @Override
+  public Map<FieldDescriptor, Object> getAllFields() {
+    final TreeMap<FieldDescriptor, Object> result =
+        new TreeMap<FieldDescriptor, Object>();
+    for (final FieldDescriptor field : metadata.descriptor.getFields()) {
+      if (hasField(field)) {
+        result.put(field, getField(field));
+      }
+    }
+    return Collections.unmodifiableMap(result);
+  }
+  
+  private void checkFieldDescriptor(FieldDescriptor field) {
+    if (field.getContainingType() != metadata.descriptor) {
+      throw new RuntimeException(
+          "Wrong FieldDescriptor \"" + field.getFullName()
+          + "\" used in message \"" + metadata.descriptor.getFullName()); 
+    }
+  }
+
+  @Override
+  public boolean hasField(FieldDescriptor field) {
+    checkFieldDescriptor(field);;
+    // A MapEntry always contains two fields.
+    return true;
+  }
+
+  @Override
+  public Object getField(FieldDescriptor field) {
+    checkFieldDescriptor(field);
+    Object result = field.getNumber() == 1 ? getKey() : getValue();
+    // Convert enums to EnumValueDescriptor.
+    if (field.getType() == FieldDescriptor.Type.ENUM) {
+      result = field.getEnumType().findValueByNumberCreatingIfUnknown(
+          (java.lang.Integer) result);
+    }
+    return result;
+  }
+
+  @Override
+  public int getRepeatedFieldCount(FieldDescriptor field) {
+    throw new RuntimeException(
+        "There is no repeated field in a map entry message.");
+  }
+
+  @Override
+  public Object getRepeatedField(FieldDescriptor field, int index) {
+    throw new RuntimeException(
+        "There is no repeated field in a map entry message.");
+  }
+
+  @Override
+  public UnknownFieldSet getUnknownFields() {
+    return UnknownFieldSet.getDefaultInstance();
+  }
+
+  /**
+   * Builder to create {@link MapEntry} messages.
+   */
+  public static class Builder<K, V>
+      extends AbstractMessage.Builder<Builder<K, V>> {
+    private final Metadata<K, V> metadata;
+    private MapEntryLite<K, V> data;
+    private MapEntryLite.Builder<K, V> dataBuilder;
+    
+    private Builder(Metadata<K, V> metadata) {
+      this.metadata = metadata;
+      this.data = metadata.defaultInstance.data;
+      this.dataBuilder = null;
+    }
+    
+    private Builder(Metadata<K, V> metadata, MapEntryLite<K, V> data) {
+      this.metadata = metadata;
+      this.data = data;
+      this.dataBuilder = null;
+    }
+    
+    public K getKey() {
+      return dataBuilder == null ? data.getKey() : dataBuilder.getKey();
+    }
+    
+    public V getValue() {
+      return dataBuilder == null ? data.getValue() : dataBuilder.getValue();
+    }
+    
+    private void ensureMutable() {
+      if (dataBuilder == null) {
+        dataBuilder = data.toBuilder();
+      }
+    }
+    
+    public Builder<K, V> setKey(K key) {
+      ensureMutable();
+      dataBuilder.setKey(key);
+      return this;
+    }
+    
+    public Builder<K, V> clearKey() {
+      ensureMutable();
+      dataBuilder.clearKey();
+      return this;
+    }
+    
+    public Builder<K, V> setValue(V value) {
+      ensureMutable();
+      dataBuilder.setValue(value);
+      return this;
+    }
+    
+    public Builder<K, V> clearValue() {
+      ensureMutable();
+      dataBuilder.clearValue();
+      return this;
+    }
+
+    @Override
+    public MapEntry<K, V> build() {
+      MapEntry<K, V> result = buildPartial();
+      if (!result.isInitialized()) {
+        throw newUninitializedMessageException(result);
+      }
+      return result;
+    }
+
+    @Override
+    public MapEntry<K, V> buildPartial() {
+      if (dataBuilder != null) {
+        data = dataBuilder.build();
+        dataBuilder = null;
+      }
+      return new MapEntry<K, V>(metadata, data);
+    }
+
+    @Override
+    public Descriptor getDescriptorForType() {
+      return metadata.descriptor;
+    }
+    
+    private void checkFieldDescriptor(FieldDescriptor field) {
+      if (field.getContainingType() != metadata.descriptor) {
+        throw new RuntimeException(
+            "Wrong FieldDescriptor \"" + field.getFullName()
+            + "\" used in message \"" + metadata.descriptor.getFullName()); 
+      }
+    }
+
+    @Override
+    public com.google.protobuf.Message.Builder newBuilderForField(
+        FieldDescriptor field) {
+      checkFieldDescriptor(field);;
+      // This method should be called for message fields and in a MapEntry
+      // message only the value field can possibly be a message field.
+      if (field.getNumber() != 2
+          || field.getJavaType() != FieldDescriptor.JavaType.MESSAGE) {
+        throw new RuntimeException(
+            "\"" + field.getFullName() + "\" is not a message value field.");
+      }
+      return ((Message) data.getValue()).newBuilderForType();
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public Builder<K, V> setField(FieldDescriptor field, Object value) {
+      checkFieldDescriptor(field);
+      if (field.getNumber() == 1) {
+        setKey((K) value);
+      } else {
+        if (field.getType() == FieldDescriptor.Type.ENUM) {
+          value = ((EnumValueDescriptor) value).getNumber();
+        }
+        setValue((V) value);
+      }
+      return this;
+    }
+
+    @Override
+    public Builder<K, V> clearField(FieldDescriptor field) {
+      checkFieldDescriptor(field);
+      if (field.getNumber() == 1) {
+        clearKey();
+      } else {
+        clearValue();
+      }
+      return this;
+    }
+
+    @Override
+    public Builder<K, V> setRepeatedField(FieldDescriptor field, int index,
+        Object value) {
+      throw new RuntimeException(
+          "There is no repeated field in a map entry message.");
+    }
+
+    @Override
+    public Builder<K, V> addRepeatedField(FieldDescriptor field, Object value) {
+      throw new RuntimeException(
+          "There is no repeated field in a map entry message.");
+    }
+
+    @Override
+    public Builder<K, V> setUnknownFields(UnknownFieldSet unknownFields) {
+      // Unknown fields are discarded for MapEntry message.
+      return this;
+    }
+
+    @Override
+    public MapEntry<K, V> getDefaultInstanceForType() {
+      return metadata.defaultInstance;
+    }
+
+    @Override
+    public boolean isInitialized() {
+      if (dataBuilder != null) {
+        return dataBuilder.isInitialized();
+      } else {
+        return data.isInitialized();
+      }
+    }
+
+    @Override
+    public Map<FieldDescriptor, Object> getAllFields() {
+      final TreeMap<FieldDescriptor, Object> result =
+          new TreeMap<FieldDescriptor, Object>();
+      for (final FieldDescriptor field : metadata.descriptor.getFields()) {
+        if (hasField(field)) {
+          result.put(field, getField(field));
+        }
+      }
+      return Collections.unmodifiableMap(result);
+    }
+
+    @Override
+    public boolean hasField(FieldDescriptor field) {
+      checkFieldDescriptor(field);
+      return true;
+    }
+
+    @Override
+    public Object getField(FieldDescriptor field) {
+      checkFieldDescriptor(field);
+      Object result = field.getNumber() == 1 ? getKey() : getValue();
+      // Convert enums to EnumValueDescriptor.
+      if (field.getType() == FieldDescriptor.Type.ENUM) {
+        result = field.getEnumType().findValueByNumberCreatingIfUnknown(
+            (java.lang.Integer) result);
+      }
+      return result;
+    }
+
+    @Override
+    public int getRepeatedFieldCount(FieldDescriptor field) {
+      throw new RuntimeException(
+          "There is no repeated field in a map entry message.");
+    }
+    
+    @Override
+    public Object getRepeatedField(FieldDescriptor field, int index) {
+      throw new RuntimeException(
+          "There is no repeated field in a map entry message.");
+    }
+    
+    @Override
+    public UnknownFieldSet getUnknownFields() {
+      return UnknownFieldSet.getDefaultInstance();
+    }
+
+    @Override
+    public Builder<K, V> clone() {
+      if (dataBuilder == null) {
+        return new Builder<K, V>(metadata, data);
+      } else {
+        return new Builder<K, V>(metadata, dataBuilder.build());
+      }
+    }
+  }
+}

java/src/main/java/com/google/protobuf/MapEntryLite.java

@@ -0,0 +1,331 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package com.google.protobuf;
+
+import java.io.IOException;
+
+/**
+ * Implements the lite version of map entry messages.
+ * 
+ * This class serves as an utility class to help do serialization/parsing of
+ * map entries. It's used in generated code and also in the full version
+ * MapEntry message.
+ * 
+ * Protobuf internal. Users shouldn't use.
+ */
+public class MapEntryLite<K, V> extends AbstractMessageLite {
+  private static class Metadata<K, V> {
+    public final MapEntryLite<K, V> defaultInstance;
+    public final WireFormat.FieldType keyType;
+    public final WireFormat.FieldType valueType;
+    public final Parser<MapEntryLite<K, V>> parser;
+    public Metadata(
+        MapEntryLite<K, V> defaultInstance,
+        WireFormat.FieldType keyType,
+        WireFormat.FieldType valueType) {
+      this.defaultInstance = defaultInstance; 
+      this.keyType = keyType;
+      this.valueType = valueType;
+      final Metadata<K, V> finalThis = this;
+      this.parser = new AbstractParser<MapEntryLite<K, V>>() {
+        @Override
+        public MapEntryLite<K, V> parsePartialFrom(
+            CodedInputStream input, ExtensionRegistryLite extensionRegistry)
+            throws InvalidProtocolBufferException {
+          return new MapEntryLite<K, V>(finalThis, input, extensionRegistry);
+        }
+      };
+    }
+  }
+  
+  private static final int KEY_FIELD_NUMBER = 1;
+  private static final int VALUE_FIELD_NUMBER = 2;
+  
+  private final Metadata<K, V> metadata;
+  private final K key;
+  private final V value;
+  
+  /** Creates a default MapEntryLite message instance. */
+  private MapEntryLite(
+      WireFormat.FieldType keyType, K defaultKey,
+      WireFormat.FieldType valueType, V defaultValue) {
+    this.metadata = new Metadata<K, V>(this, keyType, valueType);
+    this.key = defaultKey;
+    this.value = defaultValue;
+  }
+  
+  /** Creates a new MapEntryLite message. */
+  private MapEntryLite(Metadata<K, V> metadata, K key, V value) {
+    this.metadata = metadata;
+    this.key = key;
+    this.value = value;
+  }
+  
+  public K getKey() {
+    return key;
+  }
+  
+  public V getValue() {
+    return value;
+  }
+
+  /**
+   * Creates a default MapEntryLite message instance.
+   * 
+   * This method is used by generated code to create the default instance for
+   * a map entry message. The created default instance should be used to create
+   * new map entry messages of the same type. For each map entry message, only
+   * one default instance should be created. 
+   */
+  public static <K, V> MapEntryLite<K, V> newDefaultInstance(
+      WireFormat.FieldType keyType, K defaultKey,
+      WireFormat.FieldType valueType, V defaultValue) {
+    return new MapEntryLite<K, V>(
+        keyType, defaultKey, valueType, defaultValue);
+  }
+  
+  @Override
+  public void writeTo(CodedOutputStream output) throws IOException {
+    writeField(KEY_FIELD_NUMBER, metadata.keyType, key, output);
+    writeField(VALUE_FIELD_NUMBER, metadata.valueType, value, output);
+  }
+
+  private void writeField(
+      int number, WireFormat.FieldType type, Object value,
+      CodedOutputStream output) throws IOException {
+    output.writeTag(number, type.getWireType());
+    FieldSet.writeElementNoTag(output, type, value);
+  }
+
+  private volatile int cachedSerializedSize = -1;
+  @Override
+  public int getSerializedSize() {
+    if (cachedSerializedSize != -1) {
+      return cachedSerializedSize;
+    }
+    int size = 0;
+    size += getFieldSize(KEY_FIELD_NUMBER, metadata.keyType, key);
+    size += getFieldSize(VALUE_FIELD_NUMBER, metadata.valueType, value);
+    cachedSerializedSize = size;
+    return size;
+  }
+
+  private int getFieldSize(
+      int number, WireFormat.FieldType type, Object value) {
+    return CodedOutputStream.computeTagSize(number)
+        + FieldSet.computeElementSizeNoTag(type, value);
+  }
+  
+  /** Parsing constructor. */
+  private MapEntryLite(
+      Metadata<K, V> metadata,
+      CodedInputStream input,
+      ExtensionRegistryLite extensionRegistry)
+      throws InvalidProtocolBufferException {
+    try {
+      K key = metadata.defaultInstance.key;
+      V value = metadata.defaultInstance.value;
+      while (true) {
+        int tag = input.readTag();
+        if (tag == 0) {
+          break;
+        }
+        if (tag == WireFormat.makeTag(
+                KEY_FIELD_NUMBER, metadata.keyType.getWireType())) {
+          key = mergeField(
+              input, extensionRegistry, metadata.keyType, key);
+        } else if (tag == WireFormat.makeTag(
+                       VALUE_FIELD_NUMBER, metadata.valueType.getWireType())) {
+          value = mergeField(
+              input, extensionRegistry, metadata.valueType, value);
+        } else {
+          if (!input.skipField(tag)) {
+            break;
+          }
+        }
+      }
+      this.metadata = metadata;
+      this.key = key;
+      this.value = value;
+    } catch (InvalidProtocolBufferException e) {
+      throw e.setUnfinishedMessage(this);
+    } catch (IOException e) {
+      throw new InvalidProtocolBufferException(e.getMessage())
+          .setUnfinishedMessage(this);
+    }
+  }
+  
+  @SuppressWarnings("unchecked")
+  private <T> T mergeField(
+      CodedInputStream input, ExtensionRegistryLite extensionRegistry,
+      WireFormat.FieldType type, T value) throws IOException {
+    switch (type) {
+      case MESSAGE:
+        MessageLite.Builder subBuilder = ((MessageLite) value).toBuilder();
+        input.readMessage(subBuilder, extensionRegistry);
+        return (T) subBuilder.buildPartial();
+      case ENUM:
+        return (T) (java.lang.Integer) input.readEnum();
+      case GROUP:
+        throw new RuntimeException("Groups are not allowed in maps.");
+      default:
+        return (T) FieldSet.readPrimitiveField(input, type, true);
+    }
+  }
+
+  @Override
+  public Parser<MapEntryLite<K, V>> getParserForType() {
+    return metadata.parser;
+  }
+
+  @Override
+  public Builder<K, V> newBuilderForType() {
+    return new Builder<K, V>(metadata);
+  }
+
+  @Override
+  public Builder<K, V> toBuilder() {
+    return new Builder<K, V>(metadata, key, value);
+  }
+
+  @Override
+  public MapEntryLite<K, V> getDefaultInstanceForType() {
+    return metadata.defaultInstance;
+  }
+
+  @Override
+  public boolean isInitialized() {
+    if (metadata.valueType.getJavaType() == WireFormat.JavaType.MESSAGE) {
+      return ((MessageLite) value).isInitialized();
+    }
+    return true;
+  }
+
+  /**
+   * Builder used to create {@link MapEntryLite} messages.
+   */
+  public static class Builder<K, V>
+      extends AbstractMessageLite.Builder<Builder<K, V>> {
+    private final Metadata<K, V> metadata;
+    private K key;
+    private V value;
+    
+    private Builder(Metadata<K, V> metadata) {
+      this.metadata = metadata;
+      this.key = metadata.defaultInstance.key;
+      this.value = metadata.defaultInstance.value;
+    }
+    
+    public K getKey() {
+      return key;
+    }
+    
+    public V getValue() {
+      return value;
+    }
+    
+    public Builder<K, V> setKey(K key) {
+      this.key = key;
+      return this;
+    }
+    
+    public Builder<K, V> setValue(V value) {
+      this.value = value;
+      return this;
+    }
+    
+    public Builder<K, V> clearKey() {
+      this.key = metadata.defaultInstance.key;
+      return this;
+    }
+    
+    public Builder<K, V> clearValue() {
+      this.value = metadata.defaultInstance.value;
+      return this;
+    }
+
+    @Override
+    public Builder<K, V> clear() {
+      this.key = metadata.defaultInstance.key;
+      this.value = metadata.defaultInstance.value;
+      return this;
+    }
+
+    @Override
+    public MapEntryLite<K, V> build() {
+      MapEntryLite<K, V> result = buildPartial();
+      if (!result.isInitialized()) {
+        throw newUninitializedMessageException(result);
+      }
+      return result;
+    }
+
+    @Override
+    public MapEntryLite<K, V> buildPartial() {
+      return new MapEntryLite<K, V>(metadata, key, value);
+    }
+
+    @Override
+    public MessageLite getDefaultInstanceForType() {
+      return metadata.defaultInstance;
+    }
+
+    @Override
+    public boolean isInitialized() {
+      if (metadata.valueType.getJavaType() == WireFormat.JavaType.MESSAGE) {
+        return ((MessageLite) value).isInitialized();
+      }
+      return true;
+    }
+
+    private Builder(Metadata<K, V> metadata, K key, V value) {
+      this.metadata = metadata;
+      this.key = key;
+      this.value = value;
+    }
+    
+    @Override
+    public Builder<K, V> clone() {
+      return new Builder<K, V>(metadata, key, value);
+    }
+
+    @Override
+    public Builder<K, V> mergeFrom(
+        CodedInputStream input, ExtensionRegistryLite extensionRegistry)
+        throws IOException {
+      MapEntryLite<K, V> entry =
+          new MapEntryLite<K, V>(metadata, input, extensionRegistry);
+      this.key = entry.key;
+      this.value = entry.value;
+      return this;
+    }
+  }
+}

java/src/main/java/com/google/protobuf/MapField.java

@@ -0,0 +1,259 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package com.google.protobuf;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Internal representation of map fields in generated messages.
+ * 
+ * This class supports accessing the map field as a {@link Map} to be used in
+ * generated API and also supports accessing the field as a {@link List} to be
+ * used in reflection API. It keeps track of where the data is currently stored
+ * and do necessary conversions between map and list.  
+ * 
+ * This class is a protobuf implementation detail. Users shouldn't use this
+ * class directly.
+ * 
+ * THREAD-SAFETY NOTE: Read-only access is thread-safe. Users can call getMap()
+ * and getList() concurrently in multiple threads. If write-access is needed,
+ * all access must be synchronized.
+ */
+public class MapField<K, V> {
+  /**
+   * Indicates where the data of this map field is currently stored.
+   * 
+   * MAP: Data is stored in mapData.
+   * LIST: Data is stored in listData.
+   * BOTH: mapData and listData have the same data.
+   *
+   * When the map field is accessed (through generated API or reflection API),
+   * it will shift between these 3 modes:
+   * 
+   *          getMap()   getList()   getMutableMap()   getMutableList()
+   *   MAP      MAP        BOTH          MAP               LIST
+   *   LIST     BOTH       LIST          MAP               LIST
+   *   BOTH     BOTH       BOTH          MAP               LIST
+   *   
+   * As the map field changes its mode, the list/map reference returned in a
+   * previous method call may be invalidated. 
+   */
+  private enum StorageMode {MAP, LIST, BOTH}
+
+  private volatile StorageMode mode;
+  private Map<K, V> mapData;
+  private List<Message> listData;
+  
+  // Convert between a map entry Message and a key-value pair.
+  private static interface Converter<K, V> {
+    Message convertKeyAndValueToMessage(K key, V value);
+    void convertMessageToKeyAndValue(Message message, Map<K, V> map);
+    
+    Message getMessageDefaultInstance();
+  }
+  
+  private static class ImmutableMessageConverter<K, V> implements Converter<K, V> {
+    private final MapEntry<K, V> defaultEntry;
+    public ImmutableMessageConverter(MapEntry<K, V> defaultEntry) {
+      this.defaultEntry = defaultEntry;
+    }
+    
+    public Message convertKeyAndValueToMessage(K key, V value) {
+      return defaultEntry.newBuilderForType().setKey(key).setValue(value).build();
+    }
+    
+    public void convertMessageToKeyAndValue(Message message, Map<K, V> map) {
+      MapEntry<K, V> entry = (MapEntry<K, V>) message;
+      map.put(entry.getKey(), entry.getValue());
+    }
+
+    public Message getMessageDefaultInstance() {
+      return defaultEntry;
+    }
+  }
+  
+
+  private final Converter<K, V> converter;
+  
+  private MapField(
+      Converter<K, V> converter,
+      StorageMode mode,
+      Map<K, V> mapData,
+      List<Message> listData) {
+    this.converter = converter;
+    this.mode = mode;
+    this.mapData = mapData;
+    this.listData = listData;
+  }
+    
+  private MapField(
+      MapEntry<K, V> defaultEntry,
+      StorageMode mode,
+      Map<K, V> mapData,
+      List<Message> listData) {
+    this(new ImmutableMessageConverter<K, V>(defaultEntry), mode, mapData, listData);
+  }
+  
+  
+  /** Returns an immutable empty MapField. */
+  public static <K, V> MapField<K, V> emptyMapField(
+      MapEntry<K, V> defaultEntry) {
+    return new MapField<K, V>(
+        defaultEntry, StorageMode.MAP, Collections.<K, V>emptyMap(), null);
+  }
+  
+  
+  /** Creates a new mutable empty MapField. */
+  public static <K, V> MapField<K, V> newMapField(MapEntry<K, V> defaultEntry) {
+    return new MapField<K, V>(
+        defaultEntry, StorageMode.MAP, new HashMap<K, V>(), null);
+  }
+  
+  
+  private Message convertKeyAndValueToMessage(K key, V value) {
+    return converter.convertKeyAndValueToMessage(key, value);
+  }
+  
+  @SuppressWarnings("unchecked")
+  private void convertMessageToKeyAndValue(Message message, Map<K, V> map) {
+    converter.convertMessageToKeyAndValue(message, map);
+  }
+
+  private List<Message> convertMapToList(Map<K, V> mapData) {
+    List<Message> listData = new ArrayList<Message>();
+    for (Map.Entry<K, V> entry : mapData.entrySet()) {
+      listData.add(
+          convertKeyAndValueToMessage(
+              entry.getKey(), entry.getValue()));
+    }
+    return listData;
+  }
+
+  private Map<K, V> convertListToMap(List<Message> listData) {
+    Map<K, V> mapData = new HashMap<K, V>();
+    for (Message item : listData) {
+      convertMessageToKeyAndValue(item, mapData);
+    }
+    return mapData;
+  }
+  
+  /** Returns the content of this MapField as a read-only Map. */
+  public Map<K, V> getMap() {
+    if (mode == StorageMode.LIST) {
+      synchronized (this) {
+        if (mode == StorageMode.LIST) {
+          mapData = convertListToMap(listData);
+          mode = StorageMode.BOTH;
+        }
+      }
+    }
+    return Collections.unmodifiableMap(mapData);
+  }
+  
+  /** Gets a mutable Map view of this MapField. */
+  public Map<K, V> getMutableMap() {
+    if (mode != StorageMode.MAP) {
+      if (mode == StorageMode.LIST) {
+        mapData = convertListToMap(listData);
+      }
+      listData = null;
+      mode = StorageMode.MAP; 
+    }
+    return mapData;
+  }
+  
+  public void mergeFrom(MapField<K, V> other) {
+    getMutableMap().putAll(MapFieldLite.copy(other.getMap()));
+  }
+  
+  public void clear() {
+    mapData = new HashMap<K, V>();
+    mode = StorageMode.MAP;
+  }
+  
+  @SuppressWarnings("unchecked")
+  @Override
+  public boolean equals(Object object) {
+    if (!(object instanceof MapField)) {
+      return false;
+    }
+    MapField<K, V> other = (MapField<K, V>) object;
+    return MapFieldLite.<K, V>equals(getMap(), other.getMap());
+  }
+  
+  @Override
+  public int hashCode() {
+    return MapFieldLite.<K, V>calculateHashCodeForMap(getMap());
+  }
+  
+  /** Returns a deep copy of this MapField. */
+  public MapField<K, V> copy() {
+    return new MapField<K, V>(
+        converter, StorageMode.MAP, MapFieldLite.copy(getMap()), null);
+  }
+  
+  /** Gets the content of this MapField as a read-only List. */
+  List<Message> getList() {
+    if (mode == StorageMode.MAP) {
+      synchronized (this) {
+        if (mode == StorageMode.MAP) {
+          listData = convertMapToList(mapData);
+          mode = StorageMode.BOTH;
+        }
+      }
+    }
+    return Collections.unmodifiableList(listData);
+  }
+  
+  /** Gets a mutable List view of this MapField. */
+  List<Message> getMutableList() {
+    if (mode != StorageMode.LIST) {
+      if (mode == StorageMode.MAP) {
+        listData = convertMapToList(mapData);
+      }
+      mapData = null;
+      mode = StorageMode.LIST;
+    }
+    return listData;
+  }
+  
+  /**
+   * Gets the default instance of the message stored in the list view of this
+   * map field.
+   */
+  Message getMapEntryMessageDefaultInstance() {
+    return converter.getMessageDefaultInstance();
+  }
+}

java/src/main/java/com/google/protobuf/MapFieldLite.java

@@ -0,0 +1,182 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package com.google.protobuf;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Internal representation of map fields in generated lite-runtime messages.
+ * 
+ * This class is a protobuf implementation detail. Users shouldn't use this
+ * class directly.
+ */
+public class MapFieldLite<K, V> {
+  private Map<K, V> mapData;
+  
+  private MapFieldLite(Map<K, V> mapData) {
+    this.mapData = mapData;
+  }
+  
+  @SuppressWarnings({"rawtypes", "unchecked"})
+  private static final MapFieldLite EMPTY_MAP_FIELD =
+      new MapFieldLite(Collections.emptyMap());
+  
+  /** Returns an singleton immutable empty MapFieldLite instance. */
+  @SuppressWarnings({"unchecked", "cast"})
+  public static <K, V> MapFieldLite<K, V> emptyMapField() {
+    return (MapFieldLite<K, V>) EMPTY_MAP_FIELD;
+  }
+  
+  /** Creates a new MapFieldLite instance. */
+  public static <K, V> MapFieldLite<K, V> newMapField() {
+    return new MapFieldLite<K, V>(new HashMap<K, V>());
+  }
+  
+  /** Gets the content of this MapField as a read-only Map. */
+  public Map<K, V> getMap() {
+    return Collections.unmodifiableMap(mapData);
+  }
+  
+  /** Gets a mutable Map view of this MapField. */
+  public Map<K, V> getMutableMap() {
+    return mapData;
+  }
+  
+  public void mergeFrom(MapFieldLite<K, V> other) {
+    mapData.putAll(copy(other.mapData));
+  }
+  
+  public void clear() {
+    mapData.clear();
+  }
+  
+  private static boolean equals(Object a, Object b) {
+    if (a instanceof byte[] && b instanceof byte[]) {
+      return Arrays.equals((byte[]) a, (byte[]) b);
+    }
+    return a.equals(b);
+  }
+  
+  /**
+   * Checks whether two {@link Map}s are equal. We don't use the default equals
+   * method of {@link Map} because it compares by identity not by content for
+   * byte arrays.
+   */
+  static <K, V> boolean equals(Map<K, V> a, Map<K, V> b) {
+    if (a == b) {
+      return true;
+    }
+    if (a.size() != a.size()) {
+      return false;
+    }
+    for (Map.Entry<K, V> entry : a.entrySet()) {
+      if (!b.containsKey(entry.getKey())) {
+        return false;
+      }
+      if (!equals(entry.getValue(), b.get(entry.getKey()))) {
+        return false;
+      }
+    }
+    return true;
+  }
+  
+  /**
+   * Checks whether two map fields are equal.
+   */
+  @SuppressWarnings("unchecked")
+  @Override
+  public boolean equals(Object object) {
+    if (!(object instanceof MapFieldLite)) {
+      return false;
+    }
+    MapFieldLite<K, V> other = (MapFieldLite<K, V>) object;
+    return equals(mapData, other.mapData);
+  }
+  
+  private static int calculateHashCodeForObject(Object a) {
+    if (a instanceof byte[]) {
+      return LiteralByteString.hashCode((byte[]) a);
+    }
+    if (a instanceof Internal.EnumLite) {
+      return Internal.hashEnum((Internal.EnumLite) a);
+    }
+    return a.hashCode();
+  }
+
+  /**
+   * Calculates the hash code for a {@link Map}. We don't use the default hash
+   * code method of {@link Map} because for byte arrays and protobuf enums it
+   * use {@link Object#hashCode()}.
+   */
+  static <K, V> int calculateHashCodeForMap(Map<K, V> a) {
+    int result = 0;
+    for (Map.Entry<K, V> entry : a.entrySet()) {
+      result += calculateHashCodeForObject(entry.getKey())
+          ^ calculateHashCodeForObject(entry.getValue());
+    }
+    return result;    
+  }
+  
+  @Override
+  public int hashCode() {
+    return calculateHashCodeForMap(mapData);
+  }
+  
+  private static Object copy(Object object) {
+    if (object instanceof byte[]) {
+      byte[] data = (byte[]) object;
+      return Arrays.copyOf(data, data.length);
+    }
+    return object;
+  }
+  
+  /**
+   * Makes a deep copy of a {@link Map}. Immutable objects in the map will be
+   * shared (e.g., integers, strings, immutable messages) and mutable ones will
+   * have a copy (e.g., byte arrays, mutable messages).
+   */
+  @SuppressWarnings("unchecked")
+  static <K, V> Map<K, V> copy(Map<K, V> map) {
+    Map<K, V> result = new HashMap<K, V>();
+    for (Map.Entry<K, V> entry : map.entrySet()) {
+      result.put(entry.getKey(), (V) copy(entry.getValue()));
+    }
+    return result;
+  }
+  
+  /** Returns a deep copy of this map field. */
+  public MapFieldLite<K, V> copy() {
+    return new MapFieldLite<K, V>(copy(mapData));
+  }
+}

java/src/main/java/com/google/protobuf/Message.java

@@ -167,6 +167,25 @@ public interface Message extends MessageLite, MessageOrBuilder {
      */
     Builder getFieldBuilder(Descriptors.FieldDescriptor field);
 
+    /**
+     * Get a nested builder instance for the given repeated field instance.
+     * <p>
+     * Normally, we hold a reference to the immutable message object for the
+     * message type field. Some implementations(the generated message builders),
+     * however, can also hold a reference to the builder object (a nested
+     * builder) for the field.
+     * <p>
+     * If the field is already backed up by a nested builder, the nested builder
+     * will be returned. Otherwise, a new field builder will be created and
+     * returned. The original message field (if exist) will be merged into the
+     * field builder, which will then be nested into its parent builder.
+     * <p>
+     * NOTE: implementations that do not support nested builders will throw
+     * <code>UnsupportedException</code>.
+     */
+    Builder getRepeatedFieldBuilder(Descriptors.FieldDescriptor field,
+                                    int index);
+
     /**
      * Sets a field to the given value.  The value must be of the correct type
      * for this field, i.e. the same type that

java/src/main/java/com/google/protobuf/MessageReflection.java

@@ -752,13 +752,18 @@ class MessageReflection {
       if (field.getLiteType() == WireFormat.FieldType.ENUM) {
         while (input.getBytesUntilLimit() > 0) {
           final int rawValue = input.readEnum();
-          final Object value = field.getEnumType().findValueByNumber(rawValue);
-          if (value == null) {
-            // If the number isn't recognized as a valid value for this
-            // enum, drop it (don't even add it to unknownFields).
-            return true;
+          if (field.getFile().supportsUnknownEnumValue()) {
+            target.addRepeatedField(field,
+                field.getEnumType().findValueByNumberCreatingIfUnknown(rawValue));
+          } else {
+            final Object value = field.getEnumType().findValueByNumber(rawValue);
+            if (value == null) {
+              // If the number isn't recognized as a valid value for this
+              // enum, drop it (don't even add it to unknownFields).
+              return true;
+            }
+            target.addRepeatedField(field, value);
           }
-          target.addRepeatedField(field, value);
         }
       } else {
         while (input.getBytesUntilLimit() > 0) {
@@ -783,12 +788,16 @@ class MessageReflection {
         }
         case ENUM:
           final int rawValue = input.readEnum();
-          value = field.getEnumType().findValueByNumber(rawValue);
-          // If the number isn't recognized as a valid value for this enum,
-          // drop it.
-          if (value == null) {
-            unknownFields.mergeVarintField(fieldNumber, rawValue);
-            return true;
+          if (field.getFile().supportsUnknownEnumValue()) {
+            value = field.getEnumType().findValueByNumberCreatingIfUnknown(rawValue);
+          } else {
+            value = field.getEnumType().findValueByNumber(rawValue);
+            // If the number isn't recognized as a valid value for this enum,
+            // drop it.
+            if (value == null) {
+              unknownFields.mergeVarintField(fieldNumber, rawValue);
+              return true;
+            }
           }
           break;
         default:

java/src/main/java/com/google/protobuf/RepeatedFieldBuilder.java

@@ -367,22 +367,28 @@ public class RepeatedFieldBuilder
         throw new NullPointerException();
       }
     }
+
+    // If we can inspect the size, we can more efficiently add messages.
+    int size = -1;
     if (values instanceof Collection) {
       @SuppressWarnings("unchecked") final
       Collection<MType> collection = (Collection<MType>) values;
       if (collection.size() == 0) {
         return this;
       }
-      ensureMutableMessageList();
-      for (MType value : values) {
-        addMessage(value);
-      }
-    } else {
-      ensureMutableMessageList();
-      for (MType value : values) {
-        addMessage(value);
-      }
+      size = collection.size();
+    }
+    ensureMutableMessageList();
+
+    if (size >= 0 && messages instanceof ArrayList) {
+      ((ArrayList<MType>) messages)
+          .ensureCapacity(messages.size() + size);
     }
+
+    for (MType value : values) {
+      addMessage(value);
+    }
+
     onChanged();
     incrementModCounts();
     return this;

java/src/main/java/com/google/protobuf/TextFormat.java

@@ -1211,6 +1211,7 @@ public final class TextFormat {
       private SingularOverwritePolicy singularOverwritePolicy =
           SingularOverwritePolicy.ALLOW_SINGULAR_OVERWRITES;
 
+
       /**
        * Sets parser behavior when a non-repeated field appears more than once.
        */
@@ -1418,6 +1419,12 @@ public final class TextFormat {
       } else {
         consumeFieldValue(tokenizer, extensionRegistry, target, field, extension);
       }
+
+      // For historical reasons, fields may optionally be separated by commas or
+      // semicolons.
+      if (!tokenizer.tryConsume(";")) {
+        tokenizer.tryConsume(",");
+      }
     }
 
     /**
@@ -1656,10 +1663,9 @@ public final class TextFormat {
         case '\'': builder.append("\\\'"); break;
         case '"' : builder.append("\\\""); break;
         default:
-          // Note:  Bytes with the high-order bit set should be escaped.  Since
-          //   bytes are signed, such bytes will compare less than 0x20, hence
-          //   the following line is correct.
-          if (b >= 0x20) {
+          // Only ASCII characters between 0x20 (space) and 0x7e (tilde) are
+          // printable.  Other byte values must be escaped.
+          if (b >= 0x20 && b <= 0x7e) {
             builder.append((char) b);
           } else {
             builder.append('\\');

java/src/main/java/com/google/protobuf/UnknownFieldSet.java

@@ -431,6 +431,21 @@ public final class UnknownFieldSet implements MessageLite {
       return this;
     }
 
+
+    /**
+     * Convenience method for merging a length-delimited field.
+     *
+     * <p>For use by generated code only.
+     */
+    public Builder mergeLengthDelimitedField(
+        final int number, final ByteString value) {  
+      if (number == 0) {
+        throw new IllegalArgumentException("Zero is not a valid field number.");
+      }
+      getFieldBuilder(number).addLengthDelimited(value);
+      return this;
+    }
+
     /** Check if the given field number is present in the set. */
     public boolean hasField(final int number) {
       if (number == 0) {

java/src/main/java/com/google/protobuf/UnknownFieldSetLite.java

@@ -0,0 +1,297 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package com.google.protobuf;
+
+import java.io.IOException;
+
+/**
+ * {@code UnknownFieldSetLite} is used to keep track of fields which were seen
+ * when parsing a protocol message but whose field numbers or types are
+ * unrecognized. This most frequently occurs when new fields are added to a
+ * message type and then messages containing those fields are read by old
+ * software that was compiled before the new types were added.
+ *
+ * <p>For use by generated code only.
+ *
+ * @author dweis@google.com (Daniel Weis)
+ */
+public final class UnknownFieldSetLite {
+
+  private static final UnknownFieldSetLite DEFAULT_INSTANCE =
+      new UnknownFieldSetLite(ByteString.EMPTY);
+
+  /**
+   * Get an empty {@code UnknownFieldSetLite}.
+   *
+   * <p>For use by generated code only.
+   */
+  public static UnknownFieldSetLite getDefaultInstance() {
+    return DEFAULT_INSTANCE;
+  }
+
+  /**
+   * Create a new {@link Builder}.
+   *
+   * <p>For use by generated code only.
+   */
+  public static Builder newBuilder() {
+    return new Builder();
+  }
+
+  /**
+   * Returns an {@code UnknownFieldSetLite} that is the composite of {@code first} and
+   * {@code second}.
+   */
+  static UnknownFieldSetLite concat(UnknownFieldSetLite first, UnknownFieldSetLite second) {
+    return new UnknownFieldSetLite(first.byteString.concat(second.byteString));
+  }
+
+  /**
+   * The internal representation of the unknown fields.
+   */
+  private final ByteString byteString;
+
+  /**
+   * Constructs the {@code UnknownFieldSetLite} as a thin wrapper around {@link ByteString}.
+   */
+  private UnknownFieldSetLite(ByteString byteString) {
+    this.byteString = byteString;
+  }
+
+  /**
+   * Serializes the set and writes it to {@code output}.
+   *
+   * <p>For use by generated code only.
+   */
+  public void writeTo(CodedOutputStream output) throws IOException {
+    output.writeRawBytes(byteString);
+  }
+
+
+  /**
+   * Get the number of bytes required to encode this set.
+   *
+   * <p>For use by generated code only.
+   */
+  public int getSerializedSize() {
+    return byteString.size();
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj) {
+      return true;
+    }
+
+    if (obj instanceof UnknownFieldSetLite) {
+      return byteString.equals(((UnknownFieldSetLite) obj).byteString);
+    }
+
+    return false;
+  }
+
+  @Override
+  public int hashCode() {
+    return byteString.hashCode();
+  }
+
+  /**
+   * Builder for {@link UnknownFieldSetLite}s.
+   *
+   * <p>Use {@link UnknownFieldSet#newBuilder()} to construct a {@code Builder}.
+   *
+   * <p>For use by generated code only.
+   */
+  public static final class Builder {
+
+    private ByteString.Output byteStringOutput;
+    private CodedOutputStream output;
+    private boolean built;
+
+    /**
+     * Constructs a {@code Builder}. Lazily initialized by
+     * {@link #ensureInitializedButNotBuilt()}.
+     */
+    private Builder() {}
+
+    /**
+     * Ensures internal state is initialized for use.
+     */
+    private void ensureInitializedButNotBuilt() {
+      if (built) {
+        throw new IllegalStateException("Do not reuse UnknownFieldSetLite Builders.");
+      }
+
+      if (output == null && byteStringOutput == null) {
+          byteStringOutput = ByteString.newOutput(100 /* initialCapacity */);
+          output = CodedOutputStream.newInstance(byteStringOutput);
+      }
+    }
+
+    /**
+     * Parse a single field from {@code input} and merge it into this set.
+     *
+     * <p>For use by generated code only.
+     *
+     * @param tag The field's tag number, which was already parsed.
+     * @return {@code false} if the tag is an end group tag.
+     */
+    public boolean mergeFieldFrom(final int tag, final CodedInputStream input)
+                                  throws IOException {
+      ensureInitializedButNotBuilt();
+
+      final int fieldNumber = WireFormat.getTagFieldNumber(tag);
+      switch (WireFormat.getTagWireType(tag)) {
+        case WireFormat.WIRETYPE_VARINT:
+          output.writeUInt64(fieldNumber, input.readInt64());
+          return true;
+        case WireFormat.WIRETYPE_FIXED32:
+          output.writeFixed32(fieldNumber, input.readFixed32());
+          return true;
+        case WireFormat.WIRETYPE_FIXED64:
+          output.writeFixed64(fieldNumber, input.readFixed64());
+          return true;
+        case WireFormat.WIRETYPE_LENGTH_DELIMITED:
+          output.writeBytes(fieldNumber, input.readBytes());
+          return true;
+        case WireFormat.WIRETYPE_START_GROUP:
+          final Builder subBuilder = newBuilder();
+          subBuilder.mergeFrom(input);
+          input.checkLastTagWas(
+              WireFormat.makeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP));
+
+          output.writeTag(fieldNumber, WireFormat.WIRETYPE_START_GROUP);
+          subBuilder.build().writeTo(output);
+          output.writeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP);
+          return true;
+        case WireFormat.WIRETYPE_END_GROUP:
+          return false;
+        default:
+          throw InvalidProtocolBufferException.invalidWireType();
+      }
+    }
+
+    /**
+     * Convenience method for merging a new field containing a single varint
+     * value. This is used in particular when an unknown enum value is
+     * encountered.
+     *
+     * <p>For use by generated code only.
+     */
+    public Builder mergeVarintField(int fieldNumber, int value) {
+      if (fieldNumber == 0) {
+        throw new IllegalArgumentException("Zero is not a valid field number.");
+      }
+      ensureInitializedButNotBuilt();
+      try {
+        output.writeUInt64(fieldNumber, value);
+      } catch (IOException e) {
+        // Should never happen.
+      }
+      return this;
+    }
+
+    /**
+     * Convenience method for merging a length-delimited field.
+     *
+     * <p>For use by generated code only.
+     */
+    public Builder mergeLengthDelimitedField(
+        final int fieldNumber, final ByteString value) {  
+      if (fieldNumber == 0) {
+        throw new IllegalArgumentException("Zero is not a valid field number.");
+      }
+      ensureInitializedButNotBuilt();
+      try {
+        output.writeBytes(fieldNumber, value);
+      } catch (IOException e) {
+        // Should never happen.
+      }
+      return this;
+    }
+
+    /**
+     * Build the {@link UnknownFieldSetLite} and return it.
+     *
+     * <p>Once {@code build()} has been called, the {@code Builder} will no
+     * longer be usable.  Calling any method after {@code build()} will result
+     * in undefined behavior and can cause a {@code IllegalStateException} to be
+     * thrown.
+     *
+     * <p>For use by generated code only.
+     */
+    public UnknownFieldSetLite build() {
+      if (built) {
+        throw new IllegalStateException("Do not reuse UnknownFieldSetLite Builders.");
+      }
+
+      built = true;
+
+      final UnknownFieldSetLite result;
+      // If we were never initialized, no data was written.
+      if (output == null) {
+        result = getDefaultInstance();
+      } else {
+        try {
+          output.flush();
+        } catch (IOException e) {
+          // Should never happen.
+        }
+        ByteString byteString = byteStringOutput.toByteString();
+        if (byteString.isEmpty()) {
+          result = getDefaultInstance();
+        } else {
+          result = new UnknownFieldSetLite(byteString);
+        }
+      }
+
+      // Allow for garbage collection.
+      output = null;
+      byteStringOutput = null;
+      return result;
+    }
+
+    /**
+     * Parse an entire message from {@code input} and merge its fields into
+     * this set.
+     */
+    private Builder mergeFrom(final CodedInputStream input) throws IOException {
+      // Ensures initialization in mergeFieldFrom.
+      while (true) {
+        final int tag = input.readTag();
+        if (tag == 0 || !mergeFieldFrom(tag, input)) {
+          break;
+        }
+      }
+      return this;
+    }
+  }
+}

java/src/test/java/com/google/protobuf/CodedInputStreamTest.java

@@ -455,19 +455,19 @@ public class CodedInputStreamTest extends TestCase {
   }
 
   public void testMaliciousRecursion() throws Exception {
-    ByteString data64 = makeRecursiveMessage(64).toByteString();
-    ByteString data65 = makeRecursiveMessage(65).toByteString();
+    ByteString data100 = makeRecursiveMessage(100).toByteString();
+    ByteString data101 = makeRecursiveMessage(101).toByteString();
 
-    assertMessageDepth(TestRecursiveMessage.parseFrom(data64), 64);
+    assertMessageDepth(TestRecursiveMessage.parseFrom(data100), 100);
 
     try {
-      TestRecursiveMessage.parseFrom(data65);
+      TestRecursiveMessage.parseFrom(data101);
       fail("Should have thrown an exception!");
     } catch (InvalidProtocolBufferException e) {
       // success.
     }
 
-    CodedInputStream input = data64.newCodedInput();
+    CodedInputStream input = data100.newCodedInput();
     input.setRecursionLimit(8);
     try {
       TestRecursiveMessage.parseFrom(input);

java/src/test/java/com/google/protobuf/DescriptorsTest.java

@@ -706,6 +706,12 @@ public class DescriptorsTest extends TestCase {
     assertTrue(TestMultipleExtensionRanges.getDescriptor().isExtensionNumber(4143));
   }
 
+  public void testToString() {
+    assertEquals("protobuf_unittest.TestAllTypes.optional_uint64",
+        UnittestProto.TestAllTypes.getDescriptor().findFieldByNumber(
+            UnittestProto.TestAllTypes.OPTIONAL_UINT64_FIELD_NUMBER).toString());
+  }
+
   public void testPackedEnumField() throws Exception {
     FileDescriptorProto fileDescriptorProto = FileDescriptorProto.newBuilder()
         .setName("foo.proto")

java/src/test/java/com/google/protobuf/FieldPresenceTest.java

@@ -0,0 +1,363 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package com.google.protobuf;
+
+import com.google.protobuf.Descriptors.Descriptor;
+import com.google.protobuf.Descriptors.FieldDescriptor;
+import com.google.protobuf.FieldPresenceTestProto.TestAllTypes;
+import com.google.protobuf.FieldPresenceTestProto.TestOptionalFieldsOnly;
+import com.google.protobuf.FieldPresenceTestProto.TestRepeatedFieldsOnly;
+import protobuf_unittest.UnittestProto;
+
+import junit.framework.TestCase;
+
+/**
+ * Unit tests for protos that doesn't support field presence test for optional
+ * non-message fields.
+ */
+public class FieldPresenceTest extends TestCase {
+  private static boolean hasMethod(Class clazz, String name) {
+    try {
+      if (clazz.getMethod(name, new Class[]{}) != null) {
+        return true;
+      } else {
+        return false;
+      }
+    } catch (NoSuchMethodException e) {
+      return false;
+    }
+  }
+
+  private static boolean isHasMethodRemoved(
+      Class classWithFieldPresence,
+      Class classWithoutFieldPresence,
+      String camelName) {
+    return hasMethod(classWithFieldPresence, "get" + camelName)
+        && hasMethod(classWithFieldPresence, "has" + camelName)
+        && hasMethod(classWithoutFieldPresence, "get" + camelName)
+        && !hasMethod(classWithoutFieldPresence, "has" + camelName);
+  }
+
+  public void testHasMethod() {
+    // Optional non-message fields don't have a hasFoo() method generated.
+    assertTrue(isHasMethodRemoved(
+        UnittestProto.TestAllTypes.class,
+        TestAllTypes.class,
+        "OptionalInt32"));
+    assertTrue(isHasMethodRemoved(
+        UnittestProto.TestAllTypes.class,
+        TestAllTypes.class,
+        "OptionalString"));
+    assertTrue(isHasMethodRemoved(
+        UnittestProto.TestAllTypes.class,
+        TestAllTypes.class,
+        "OptionalBytes"));
+    assertTrue(isHasMethodRemoved(
+        UnittestProto.TestAllTypes.class,
+        TestAllTypes.class,
+        "OptionalNestedEnum"));
+
+    assertTrue(isHasMethodRemoved(
+        UnittestProto.TestAllTypes.Builder.class,
+        TestAllTypes.Builder.class,
+        "OptionalInt32"));
+    assertTrue(isHasMethodRemoved(
+        UnittestProto.TestAllTypes.Builder.class,
+        TestAllTypes.Builder.class,
+        "OptionalString"));
+    assertTrue(isHasMethodRemoved(
+        UnittestProto.TestAllTypes.Builder.class,
+        TestAllTypes.Builder.class,
+        "OptionalBytes"));
+    assertTrue(isHasMethodRemoved(
+        UnittestProto.TestAllTypes.Builder.class,
+        TestAllTypes.Builder.class,
+        "OptionalNestedEnum"));
+
+    // message fields still have the hasFoo() method generated.
+    assertFalse(TestAllTypes.newBuilder().build().hasOptionalNestedMessage());
+    assertFalse(TestAllTypes.newBuilder().hasOptionalNestedMessage());
+
+    // oneof fields don't have hasFoo() methods (even for message types).
+    assertTrue(isHasMethodRemoved(
+        UnittestProto.TestAllTypes.class,
+        TestAllTypes.class,
+        "OneofUint32"));
+    assertTrue(isHasMethodRemoved(
+        UnittestProto.TestAllTypes.class,
+        TestAllTypes.class,
+        "OneofString"));
+    assertTrue(isHasMethodRemoved(
+        UnittestProto.TestAllTypes.class,
+        TestAllTypes.class,
+        "OneofBytes"));
+    assertTrue(isHasMethodRemoved(
+        UnittestProto.TestAllTypes.class,
+        TestAllTypes.class,
+        "OneofNestedMessage"));
+
+    assertTrue(isHasMethodRemoved(
+        UnittestProto.TestAllTypes.Builder.class,
+        TestAllTypes.Builder.class,
+        "OneofUint32"));
+    assertTrue(isHasMethodRemoved(
+        UnittestProto.TestAllTypes.Builder.class,
+        TestAllTypes.Builder.class,
+        "OneofString"));
+    assertTrue(isHasMethodRemoved(
+        UnittestProto.TestAllTypes.Builder.class,
+        TestAllTypes.Builder.class,
+        "OneofBytes"));
+    assertTrue(isHasMethodRemoved(
+        UnittestProto.TestAllTypes.Builder.class,
+        TestAllTypes.Builder.class,
+        "OneofNestedMessage"));
+  }
+
+  public void testFieldPresence() {
+    // Optional non-message fields set to their default value are treated the
+    // same way as not set.
+
+    // Serialization will ignore such fields.
+    TestAllTypes.Builder builder = TestAllTypes.newBuilder();
+    builder.setOptionalInt32(0);
+    builder.setOptionalString("");
+    builder.setOptionalBytes(ByteString.EMPTY);
+    builder.setOptionalNestedEnum(TestAllTypes.NestedEnum.FOO);
+    TestAllTypes message = builder.build();
+    assertEquals(0, message.getSerializedSize());
+
+    // mergeFrom() will ignore such fields.
+    TestAllTypes.Builder a = TestAllTypes.newBuilder();
+    a.setOptionalInt32(1);
+    a.setOptionalString("x");
+    a.setOptionalBytes(ByteString.copyFromUtf8("y"));
+    a.setOptionalNestedEnum(TestAllTypes.NestedEnum.BAR);
+    TestAllTypes.Builder b = TestAllTypes.newBuilder();
+    b.setOptionalInt32(0);
+    b.setOptionalString("");
+    b.setOptionalBytes(ByteString.EMPTY);
+    b.setOptionalNestedEnum(TestAllTypes.NestedEnum.FOO);
+    a.mergeFrom(b.build());
+    message = a.build();
+    assertEquals(1, message.getOptionalInt32());
+    assertEquals("x", message.getOptionalString());
+    assertEquals(ByteString.copyFromUtf8("y"), message.getOptionalBytes());
+    assertEquals(TestAllTypes.NestedEnum.BAR, message.getOptionalNestedEnum());
+
+    // equals()/hashCode() should produce the same results.
+    TestAllTypes empty = TestAllTypes.newBuilder().build();
+    message = builder.build();
+    assertTrue(empty.equals(message));
+    assertTrue(message.equals(empty));
+    assertEquals(empty.hashCode(), message.hashCode());
+  }
+
+  public void testFieldPresenceByReflection() {
+    Descriptor descriptor = TestAllTypes.getDescriptor();
+    FieldDescriptor optionalInt32Field = descriptor.findFieldByName("optional_int32");
+    FieldDescriptor optionalStringField = descriptor.findFieldByName("optional_string");
+    FieldDescriptor optionalBytesField = descriptor.findFieldByName("optional_bytes");
+    FieldDescriptor optionalNestedEnumField = descriptor.findFieldByName("optional_nested_enum");
+
+    // Field not present.
+    TestAllTypes message = TestAllTypes.newBuilder().build();
+    assertFalse(message.hasField(optionalInt32Field));
+    assertFalse(message.hasField(optionalStringField));
+    assertFalse(message.hasField(optionalBytesField));
+    assertFalse(message.hasField(optionalNestedEnumField));
+    assertEquals(0, message.getAllFields().size());
+
+    // Field set to default value is seen as not present.
+    message = TestAllTypes.newBuilder()
+        .setOptionalInt32(0)
+        .setOptionalString("")
+        .setOptionalBytes(ByteString.EMPTY)
+        .setOptionalNestedEnum(TestAllTypes.NestedEnum.FOO)
+        .build();
+    assertFalse(message.hasField(optionalInt32Field));
+    assertFalse(message.hasField(optionalStringField));
+    assertFalse(message.hasField(optionalBytesField));
+    assertFalse(message.hasField(optionalNestedEnumField));
+    assertEquals(0, message.getAllFields().size());
+
+    // Field set to non-default value is seen as present.
+    message = TestAllTypes.newBuilder()
+        .setOptionalInt32(1)
+        .setOptionalString("x")
+        .setOptionalBytes(ByteString.copyFromUtf8("y"))
+        .setOptionalNestedEnum(TestAllTypes.NestedEnum.BAR)
+        .build();
+    assertTrue(message.hasField(optionalInt32Field));
+    assertTrue(message.hasField(optionalStringField));
+    assertTrue(message.hasField(optionalBytesField));
+    assertTrue(message.hasField(optionalNestedEnumField));
+    assertEquals(4, message.getAllFields().size());
+  }
+  
+  public void testMessageField() {
+    TestAllTypes.Builder builder = TestAllTypes.newBuilder();
+    assertFalse(builder.hasOptionalNestedMessage());
+    assertFalse(builder.build().hasOptionalNestedMessage());
+    
+    TestAllTypes.NestedMessage.Builder nestedBuilder =
+        builder.getOptionalNestedMessageBuilder();
+    assertTrue(builder.hasOptionalNestedMessage());
+    assertTrue(builder.build().hasOptionalNestedMessage());
+    
+    nestedBuilder.setValue(1);
+    assertEquals(1, builder.build().getOptionalNestedMessage().getValue());
+    
+    builder.clearOptionalNestedMessage();
+    assertFalse(builder.hasOptionalNestedMessage());
+    assertFalse(builder.build().hasOptionalNestedMessage());
+    
+    // Unlike non-message fields, if we set a message field to its default value (i.e.,
+    // default instance), the field should be seen as present.
+    builder.setOptionalNestedMessage(TestAllTypes.NestedMessage.getDefaultInstance());
+    assertTrue(builder.hasOptionalNestedMessage());
+    assertTrue(builder.build().hasOptionalNestedMessage());
+  }
+
+  public void testSerializeAndParse() throws Exception {
+    TestAllTypes.Builder builder = TestAllTypes.newBuilder();
+    builder.setOptionalInt32(1234);
+    builder.setOptionalString("hello");
+    builder.setOptionalNestedMessage(TestAllTypes.NestedMessage.getDefaultInstance());
+    // Set an oneof field to its default value and expect it to be serialized (i.e.,
+    // an oneof field set to the default value should be treated as present).
+    builder.setOneofInt32(0);
+    ByteString data = builder.build().toByteString();
+
+    TestAllTypes message = TestAllTypes.parseFrom(data);
+    assertEquals(1234, message.getOptionalInt32());
+    assertEquals("hello", message.getOptionalString());
+    // Fields not set will have the default value.
+    assertEquals(ByteString.EMPTY, message.getOptionalBytes());
+    assertEquals(TestAllTypes.NestedEnum.FOO, message.getOptionalNestedEnum());
+    // The message field is set despite that it's set with a default instance.
+    assertTrue(message.hasOptionalNestedMessage());
+    assertEquals(0, message.getOptionalNestedMessage().getValue());
+    // The oneof field set to its default value is also present.
+    assertEquals(
+        TestAllTypes.OneofFieldCase.ONEOF_INT32, message.getOneofFieldCase());
+  }
+
+  // Regression test for b/16173397
+  // Make sure we haven't screwed up the code generation for repeated fields.
+  public void testRepeatedFields() throws Exception {
+    TestAllTypes.Builder builder = TestAllTypes.newBuilder();
+    builder.setOptionalInt32(1234);
+    builder.setOptionalString("hello");
+    builder.setOptionalNestedMessage(TestAllTypes.NestedMessage.getDefaultInstance());
+    builder.addRepeatedInt32(4321);
+    builder.addRepeatedString("world");
+    builder.addRepeatedNestedMessage(TestAllTypes.NestedMessage.getDefaultInstance());
+    ByteString data = builder.build().toByteString();
+
+    TestOptionalFieldsOnly optionalOnlyMessage = TestOptionalFieldsOnly.parseFrom(data);
+    assertEquals(1234, optionalOnlyMessage.getOptionalInt32());
+    assertEquals("hello", optionalOnlyMessage.getOptionalString());
+    assertTrue(optionalOnlyMessage.hasOptionalNestedMessage());
+    assertEquals(0, optionalOnlyMessage.getOptionalNestedMessage().getValue());
+
+    TestRepeatedFieldsOnly repeatedOnlyMessage = TestRepeatedFieldsOnly.parseFrom(data);
+    assertEquals(1, repeatedOnlyMessage.getRepeatedInt32Count());
+    assertEquals(4321, repeatedOnlyMessage.getRepeatedInt32(0));
+    assertEquals(1, repeatedOnlyMessage.getRepeatedStringCount());
+    assertEquals("world", repeatedOnlyMessage.getRepeatedString(0));
+    assertEquals(1, repeatedOnlyMessage.getRepeatedNestedMessageCount());
+    assertEquals(0, repeatedOnlyMessage.getRepeatedNestedMessage(0).getValue());
+  }
+
+  public void testIsInitialized() throws Exception {
+    TestAllTypes.Builder builder = TestAllTypes.newBuilder();
+
+    // Test optional proto2 message fields.
+    UnittestProto.TestRequired.Builder proto2Builder =
+        builder.getOptionalProto2MessageBuilder();
+    assertFalse(builder.isInitialized());
+    assertFalse(builder.buildPartial().isInitialized());
+
+    proto2Builder.setA(1).setB(2).setC(3);
+    assertTrue(builder.isInitialized());
+    assertTrue(builder.buildPartial().isInitialized());
+
+    // Test oneof proto2 message fields.
+    proto2Builder = builder.getOneofProto2MessageBuilder();
+    assertFalse(builder.isInitialized());
+    assertFalse(builder.buildPartial().isInitialized());
+
+    proto2Builder.setA(1).setB(2).setC(3);
+    assertTrue(builder.isInitialized());
+    assertTrue(builder.buildPartial().isInitialized());
+
+    // Test repeated proto2 message fields.
+    proto2Builder = builder.addRepeatedProto2MessageBuilder();
+    assertFalse(builder.isInitialized());
+    assertFalse(builder.buildPartial().isInitialized());
+
+    proto2Builder.setA(1).setB(2).setC(3);
+    assertTrue(builder.isInitialized());
+    assertTrue(builder.buildPartial().isInitialized());
+  }
+
+  
+  // Test that unknown fields are dropped.
+  public void testUnknownFields() throws Exception {
+    TestAllTypes.Builder builder = TestAllTypes.newBuilder();
+    builder.setOptionalInt32(1234);
+    builder.addRepeatedInt32(5678);
+    TestAllTypes message = builder.build();
+    ByteString data = message.toByteString();
+    
+    TestOptionalFieldsOnly optionalOnlyMessage =
+        TestOptionalFieldsOnly.parseFrom(data);
+    // UnknownFieldSet should be empty.
+    assertEquals(
+        0, optionalOnlyMessage.getUnknownFields().toByteString().size());
+    assertEquals(1234, optionalOnlyMessage.getOptionalInt32());
+    message = TestAllTypes.parseFrom(optionalOnlyMessage.toByteString());
+    assertEquals(1234, message.getOptionalInt32());
+    // The repeated field is discarded because it's unknown to the optional-only
+    // message.
+    assertEquals(0, message.getRepeatedInt32Count());
+    
+    DynamicMessage dynamicOptionalOnlyMessage =
+        DynamicMessage.getDefaultInstance(
+            TestOptionalFieldsOnly.getDescriptor())
+        .getParserForType().parseFrom(data);
+    assertEquals(
+        0, dynamicOptionalOnlyMessage.getUnknownFields().toByteString().size());
+    assertEquals(optionalOnlyMessage.toByteString(),
+        dynamicOptionalOnlyMessage.toByteString());
+  }
+}

java/src/test/java/com/google/protobuf/GeneratedMessageTest.java

@@ -157,15 +157,12 @@ public class GeneratedMessageTest extends TestCase {
   public void testProtosShareRepeatedArraysIfDidntChange() throws Exception {
     TestAllTypes.Builder builder = TestAllTypes.newBuilder();
     builder.addRepeatedInt32(100);
-    builder.addRepeatedImportEnum(UnittestImport.ImportEnum.IMPORT_BAR);
     builder.addRepeatedForeignMessage(ForeignMessage.getDefaultInstance());
 
     TestAllTypes value1 = builder.build();
     TestAllTypes value2 = value1.toBuilder().build();
 
     assertSame(value1.getRepeatedInt32List(), value2.getRepeatedInt32List());
-    assertSame(value1.getRepeatedImportEnumList(),
-        value2.getRepeatedImportEnumList());
     assertSame(value1.getRepeatedForeignMessageList(),
         value2.getRepeatedForeignMessageList());
   }
@@ -1512,4 +1509,142 @@ public class GeneratedMessageTest extends TestCase {
       assertEquals(message2.getFooMessage().getQuxInt(), 234);
     }
   }
+
+  public void testGetRepeatedFieldBuilder() {
+    Descriptor descriptor = TestAllTypes.getDescriptor();
+
+    FieldDescriptor fieldDescriptor =
+        descriptor.findFieldByName("repeated_nested_message");
+    FieldDescriptor foreignFieldDescriptor =
+        descriptor.findFieldByName("repeated_foreign_message");
+    FieldDescriptor importFieldDescriptor =
+        descriptor.findFieldByName("repeated_import_message");
+
+    // Mutate the message with new field builder
+    // Mutate nested message
+    TestAllTypes.Builder builder1 = TestAllTypes.newBuilder();
+    Message.Builder fieldBuilder1 = builder1.newBuilderForField(
+        fieldDescriptor);
+    FieldDescriptor subFieldDescriptor1 =
+        fieldBuilder1.getDescriptorForType().findFieldByName("bb");
+    fieldBuilder1.setField(subFieldDescriptor1, 1);
+    builder1.addRepeatedField(fieldDescriptor, fieldBuilder1.build());
+
+    // Mutate foreign message
+    Message.Builder foreignFieldBuilder1 = builder1.newBuilderForField(
+        foreignFieldDescriptor);
+    FieldDescriptor subForeignFieldDescriptor1 =
+        foreignFieldBuilder1.getDescriptorForType().findFieldByName("c");
+    foreignFieldBuilder1.setField(subForeignFieldDescriptor1, 2);
+    builder1.addRepeatedField(foreignFieldDescriptor,
+        foreignFieldBuilder1.build());
+
+    // Mutate import message
+    Message.Builder importFieldBuilder1 = builder1.newBuilderForField(
+        importFieldDescriptor);
+    FieldDescriptor subImportFieldDescriptor1 =
+        importFieldBuilder1.getDescriptorForType().findFieldByName("d");
+    importFieldBuilder1.setField(subImportFieldDescriptor1, 3);
+    builder1.addRepeatedField(importFieldDescriptor,
+        importFieldBuilder1.build());
+
+    Message newMessage1 = builder1.build();
+
+    // Mutate the message with existing field builder
+    // Mutate nested message
+    TestAllTypes.Builder builder2 = TestAllTypes.newBuilder();
+    builder2.addRepeatedNestedMessageBuilder();
+    Message.Builder fieldBuilder2 = builder2.getRepeatedFieldBuilder(
+        fieldDescriptor, 0);
+    FieldDescriptor subFieldDescriptor2 =
+        fieldBuilder2.getDescriptorForType().findFieldByName("bb");
+    fieldBuilder2.setField(subFieldDescriptor2, 1);
+
+    // Mutate foreign message
+    Message.Builder foreignFieldBuilder2 = builder2.newBuilderForField(
+        foreignFieldDescriptor);
+    FieldDescriptor subForeignFieldDescriptor2 =
+        foreignFieldBuilder2.getDescriptorForType().findFieldByName("c");
+    foreignFieldBuilder2.setField(subForeignFieldDescriptor2, 2);
+    builder2.addRepeatedField(foreignFieldDescriptor,
+        foreignFieldBuilder2.build());
+
+    // Mutate import message
+    Message.Builder importFieldBuilder2 = builder2.newBuilderForField(
+        importFieldDescriptor);
+    FieldDescriptor subImportFieldDescriptor2 =
+        importFieldBuilder2.getDescriptorForType().findFieldByName("d");
+    importFieldBuilder2.setField(subImportFieldDescriptor2, 3);
+    builder2.addRepeatedField(importFieldDescriptor,
+        importFieldBuilder2.build());
+
+    Message newMessage2 = builder2.build();
+
+    // These two messages should be equal.
+    assertEquals(newMessage1, newMessage2);
+  }
+
+  public void testGetRepeatedFieldBuilderWithInitializedValue() {
+    Descriptor descriptor = TestAllTypes.getDescriptor();
+    FieldDescriptor fieldDescriptor =
+        descriptor.findFieldByName("repeated_nested_message");
+
+    // Before setting field, builder is initialized by default value. 
+    TestAllTypes.Builder builder = TestAllTypes.newBuilder();
+    builder.addRepeatedNestedMessageBuilder();
+    NestedMessage.Builder fieldBuilder =
+        (NestedMessage.Builder) builder.getRepeatedFieldBuilder(fieldDescriptor, 0);
+    assertEquals(0, fieldBuilder.getBb());
+
+    // Setting field value with new field builder instance.
+    builder = TestAllTypes.newBuilder();
+    NestedMessage.Builder newFieldBuilder =
+        builder.addRepeatedNestedMessageBuilder();
+    newFieldBuilder.setBb(2);
+    // Then get the field builder instance by getRepeatedFieldBuilder().
+    fieldBuilder =
+        (NestedMessage.Builder) builder.getRepeatedFieldBuilder(fieldDescriptor, 0);
+    // It should contain new value.
+    assertEquals(2, fieldBuilder.getBb());
+    // These two builder should be equal.
+    assertSame(fieldBuilder, newFieldBuilder);
+  }
+
+  public void testGetRepeatedFieldBuilderNotSupportedException() {
+    Descriptor descriptor = TestAllTypes.getDescriptor();
+    TestAllTypes.Builder builder = TestAllTypes.newBuilder();
+    try {
+      builder.getRepeatedFieldBuilder(descriptor.findFieldByName("repeated_int32"), 0);
+      fail("Exception was not thrown");
+    } catch (UnsupportedOperationException e) {
+      // We expect this exception.
+    }
+    try {
+      builder.getRepeatedFieldBuilder(
+          descriptor.findFieldByName("repeated_nested_enum"), 0);
+      fail("Exception was not thrown");
+    } catch (UnsupportedOperationException e) {
+      // We expect this exception.
+    }
+    try {
+      builder.getRepeatedFieldBuilder(descriptor.findFieldByName("optional_int32"), 0);
+      fail("Exception was not thrown");
+    } catch (UnsupportedOperationException e) {
+      // We expect this exception.
+    }
+    try {
+      builder.getRepeatedFieldBuilder(
+          descriptor.findFieldByName("optional_nested_enum"), 0);
+      fail("Exception was not thrown");
+    } catch (UnsupportedOperationException e) {
+      // We expect this exception.
+    }
+    try {
+      builder.getRepeatedFieldBuilder(
+          descriptor.findFieldByName("optional_nested_message"), 0);
+      fail("Exception was not thrown");
+    } catch (UnsupportedOperationException e) {
+      // We expect this exception.
+    }
+  }
 }

java/src/test/java/com/google/protobuf/LazyMessageLiteTest.java

@@ -36,8 +36,6 @@ import protobuf_unittest.LazyFieldsLite.LazyNestedInnerMessageLite;
 
 import junit.framework.TestCase;
 
-import org.easymock.classextension.EasyMock;
-
 import java.util.ArrayList;
 
 /**
@@ -52,14 +50,10 @@ public class LazyMessageLiteTest extends TestCase {
   @Override
   protected void setUp() throws Exception {
     super.setUp();
-
-    originalLazyInnerMessageLiteParser = LazyInnerMessageLite.PARSER;
   }
 
   @Override
   protected void tearDown() throws Exception {
-    LazyInnerMessageLite.PARSER = originalLazyInnerMessageLiteParser;
-
     super.tearDown();
   }
 
@@ -291,29 +285,4 @@ public class LazyMessageLiteTest extends TestCase {
 
     assertEquals(bytes, deserialized.toByteString());
   }
-
-  public void testLaziness() throws InvalidProtocolBufferException {
-    LazyInnerMessageLite inner = LazyInnerMessageLite.newBuilder()
-        .setNum(2)
-        .build();
-    LazyMessageLite outer = LazyMessageLite.newBuilder()
-        .setNum(1)
-        .setInner(inner)
-        .setOneofInner(inner)
-        .build();
-    ByteString bytes = outer.toByteString();
-
-
-    // The parser for inner / oneofInner message shouldn't be used if
-    // getInner / getOneofInner is not called.
-    LazyInnerMessageLite.PARSER = EasyMock.createStrictMock(Parser.class);
-
-    EasyMock.replay(LazyInnerMessageLite.PARSER);
-
-    LazyMessageLite deserialized = LazyMessageLite.parseFrom(bytes);
-    assertEquals(1, deserialized.getNum());
-    assertEquals(421,  deserialized.getNumWithDefault());
-
-    EasyMock.verify(LazyInnerMessageLite.PARSER);
-  }
 }

java/src/test/java/com/google/protobuf/LiteEqualsAndHashTest.java

@@ -82,4 +82,27 @@ public class LiteEqualsAndHashTest extends TestCase {
     BarPrime barPrime = BarPrime.newBuilder().setName("bar").build();
     assertFalse(bar.equals(barPrime));
   }
+
+  public void testEqualsAndHashCodeWithUnknownFields() throws InvalidProtocolBufferException {
+    Foo fooWithOnlyValue = Foo.newBuilder()
+        .setValue(1)
+        .build();
+
+    Foo fooWithValueAndExtension = fooWithOnlyValue.toBuilder()
+        .setValue(1)
+        .setExtension(Bar.fooExt, Bar.newBuilder()
+            .setName("name")
+            .build())
+        .build();
+
+    Foo fooWithValueAndUnknownFields = Foo.parseFrom(fooWithValueAndExtension.toByteArray());
+
+    assertEqualsAndHashCodeAreFalse(fooWithOnlyValue, fooWithValueAndUnknownFields);
+    assertEqualsAndHashCodeAreFalse(fooWithValueAndExtension, fooWithValueAndUnknownFields);
+  }
+
+  private void assertEqualsAndHashCodeAreFalse(Object o1, Object o2) {
+    assertFalse(o1.equals(o2));
+    assertFalse(o1.hashCode() == o2.hashCode());
+  }
 }

java/src/test/java/com/google/protobuf/MapForProto2LiteTest.java

@@ -0,0 +1,277 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package com.google.protobuf;
+
+import map_lite_test.MapForProto2TestProto.TestMap;
+import map_lite_test.MapForProto2TestProto.TestMap.MessageValue;
+import map_lite_test.MapForProto2TestProto.TestUnknownEnumValue;
+
+import junit.framework.TestCase;
+
+/**
+ * Unit tests for map fields.
+ */
+public class MapForProto2LiteTest extends TestCase {
+  private void setMapValues(TestMap.Builder builder) {
+    builder.getMutableInt32ToInt32Field().put(1, 11);
+    builder.getMutableInt32ToInt32Field().put(2, 22);
+    builder.getMutableInt32ToInt32Field().put(3, 33);
+
+    builder.getMutableInt32ToStringField().put(1, "11");
+    builder.getMutableInt32ToStringField().put(2, "22");
+    builder.getMutableInt32ToStringField().put(3, "33");
+    
+    builder.getMutableInt32ToBytesField().put(1, TestUtil.toBytes("11"));
+    builder.getMutableInt32ToBytesField().put(2, TestUtil.toBytes("22"));
+    builder.getMutableInt32ToBytesField().put(3, TestUtil.toBytes("33"));
+    
+    builder.getMutableInt32ToEnumField().put(1, TestMap.EnumValue.FOO);
+    builder.getMutableInt32ToEnumField().put(2, TestMap.EnumValue.BAR);
+    builder.getMutableInt32ToEnumField().put(3, TestMap.EnumValue.BAZ);
+    
+    builder.getMutableInt32ToMessageField().put(
+        1, MessageValue.newBuilder().setValue(11).build());
+    builder.getMutableInt32ToMessageField().put(
+        2, MessageValue.newBuilder().setValue(22).build());
+    builder.getMutableInt32ToMessageField().put(
+        3, MessageValue.newBuilder().setValue(33).build());
+    
+    builder.getMutableStringToInt32Field().put("1", 11);
+    builder.getMutableStringToInt32Field().put("2", 22);
+    builder.getMutableStringToInt32Field().put("3", 33);
+  }
+
+  private void assertMapValuesSet(TestMap message) {
+    assertEquals(3, message.getInt32ToInt32Field().size());
+    assertEquals(11, message.getInt32ToInt32Field().get(1).intValue());
+    assertEquals(22, message.getInt32ToInt32Field().get(2).intValue());
+    assertEquals(33, message.getInt32ToInt32Field().get(3).intValue());
+
+    assertEquals(3, message.getInt32ToStringField().size());
+    assertEquals("11", message.getInt32ToStringField().get(1));
+    assertEquals("22", message.getInt32ToStringField().get(2));
+    assertEquals("33", message.getInt32ToStringField().get(3));
+    
+    assertEquals(3, message.getInt32ToBytesField().size());
+    assertEquals(TestUtil.toBytes("11"), message.getInt32ToBytesField().get(1));
+    assertEquals(TestUtil.toBytes("22"), message.getInt32ToBytesField().get(2));
+    assertEquals(TestUtil.toBytes("33"), message.getInt32ToBytesField().get(3));
+    
+    assertEquals(3, message.getInt32ToEnumField().size());
+    assertEquals(TestMap.EnumValue.FOO, message.getInt32ToEnumField().get(1));
+    assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumField().get(2));
+    assertEquals(TestMap.EnumValue.BAZ, message.getInt32ToEnumField().get(3));
+    
+    assertEquals(3, message.getInt32ToMessageField().size());
+    assertEquals(11, message.getInt32ToMessageField().get(1).getValue());
+    assertEquals(22, message.getInt32ToMessageField().get(2).getValue());
+    assertEquals(33, message.getInt32ToMessageField().get(3).getValue());
+    
+    assertEquals(3, message.getStringToInt32Field().size());
+    assertEquals(11, message.getStringToInt32Field().get("1").intValue());
+    assertEquals(22, message.getStringToInt32Field().get("2").intValue());
+    assertEquals(33, message.getStringToInt32Field().get("3").intValue());
+  }
+
+  private void updateMapValues(TestMap.Builder builder) {
+    builder.getMutableInt32ToInt32Field().put(1, 111);
+    builder.getMutableInt32ToInt32Field().remove(2);
+    builder.getMutableInt32ToInt32Field().put(4, 44);
+
+    builder.getMutableInt32ToStringField().put(1, "111");
+    builder.getMutableInt32ToStringField().remove(2);
+    builder.getMutableInt32ToStringField().put(4, "44");
+    
+    builder.getMutableInt32ToBytesField().put(1, TestUtil.toBytes("111"));
+    builder.getMutableInt32ToBytesField().remove(2);
+    builder.getMutableInt32ToBytesField().put(4, TestUtil.toBytes("44"));
+    
+    builder.getMutableInt32ToEnumField().put(1, TestMap.EnumValue.BAR);
+    builder.getMutableInt32ToEnumField().remove(2);
+    builder.getMutableInt32ToEnumField().put(4, TestMap.EnumValue.QUX);
+    
+    builder.getMutableInt32ToMessageField().put(
+        1, MessageValue.newBuilder().setValue(111).build());
+    builder.getMutableInt32ToMessageField().remove(2);
+    builder.getMutableInt32ToMessageField().put(
+        4, MessageValue.newBuilder().setValue(44).build());
+    
+    builder.getMutableStringToInt32Field().put("1", 111);
+    builder.getMutableStringToInt32Field().remove("2");
+    builder.getMutableStringToInt32Field().put("4", 44);
+  }
+
+  private void assertMapValuesUpdated(TestMap message) {
+    assertEquals(3, message.getInt32ToInt32Field().size());
+    assertEquals(111, message.getInt32ToInt32Field().get(1).intValue());
+    assertEquals(33, message.getInt32ToInt32Field().get(3).intValue());
+    assertEquals(44, message.getInt32ToInt32Field().get(4).intValue());
+
+    assertEquals(3, message.getInt32ToStringField().size());
+    assertEquals("111", message.getInt32ToStringField().get(1));
+    assertEquals("33", message.getInt32ToStringField().get(3));
+    assertEquals("44", message.getInt32ToStringField().get(4));
+    
+    assertEquals(3, message.getInt32ToBytesField().size());
+    assertEquals(TestUtil.toBytes("111"), message.getInt32ToBytesField().get(1));
+    assertEquals(TestUtil.toBytes("33"), message.getInt32ToBytesField().get(3));
+    assertEquals(TestUtil.toBytes("44"), message.getInt32ToBytesField().get(4));
+    
+    assertEquals(3, message.getInt32ToEnumField().size());
+    assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumField().get(1));
+    assertEquals(TestMap.EnumValue.BAZ, message.getInt32ToEnumField().get(3));
+    assertEquals(TestMap.EnumValue.QUX, message.getInt32ToEnumField().get(4));
+    
+    assertEquals(3, message.getInt32ToMessageField().size());
+    assertEquals(111, message.getInt32ToMessageField().get(1).getValue());
+    assertEquals(33, message.getInt32ToMessageField().get(3).getValue());
+    assertEquals(44, message.getInt32ToMessageField().get(4).getValue());
+    
+    assertEquals(3, message.getStringToInt32Field().size());
+    assertEquals(111, message.getStringToInt32Field().get("1").intValue());
+    assertEquals(33, message.getStringToInt32Field().get("3").intValue());
+    assertEquals(44, message.getStringToInt32Field().get("4").intValue());
+  }
+
+  private void assertMapValuesCleared(TestMap message) {
+    assertEquals(0, message.getInt32ToInt32Field().size());
+    assertEquals(0, message.getInt32ToStringField().size());
+    assertEquals(0, message.getInt32ToBytesField().size());
+    assertEquals(0, message.getInt32ToEnumField().size());
+    assertEquals(0, message.getInt32ToMessageField().size());
+    assertEquals(0, message.getStringToInt32Field().size());
+  }
+
+  public void testGettersAndSetters() throws Exception {
+    TestMap.Builder builder = TestMap.newBuilder();
+    TestMap message = builder.build();
+    assertMapValuesCleared(message);
+    
+    builder = message.toBuilder();
+    setMapValues(builder);
+    message = builder.build();
+    assertMapValuesSet(message);
+    
+    builder = message.toBuilder();
+    updateMapValues(builder);
+    message = builder.build();
+    assertMapValuesUpdated(message);
+    
+    builder = message.toBuilder();
+    builder.clear();
+    message = builder.build();
+    assertMapValuesCleared(message);
+  }
+
+  public void testSerializeAndParse() throws Exception {
+    TestMap.Builder builder = TestMap.newBuilder();
+    setMapValues(builder);
+    TestMap message = builder.build();
+    assertEquals(message.getSerializedSize(), message.toByteString().size());
+    message = TestMap.PARSER.parseFrom(message.toByteString());
+    assertMapValuesSet(message);
+    
+    builder = message.toBuilder();
+    updateMapValues(builder);
+    message = builder.build();
+    assertEquals(message.getSerializedSize(), message.toByteString().size());
+    message = TestMap.PARSER.parseFrom(message.toByteString());
+    assertMapValuesUpdated(message);
+    
+    builder = message.toBuilder();
+    builder.clear();
+    message = builder.build();
+    assertEquals(message.getSerializedSize(), message.toByteString().size());
+    message = TestMap.PARSER.parseFrom(message.toByteString());
+    assertMapValuesCleared(message);
+  }
+  
+  public void testMergeFrom() throws Exception {
+    TestMap.Builder builder = TestMap.newBuilder();
+    setMapValues(builder);
+    TestMap message = builder.build();
+    
+    TestMap.Builder other = TestMap.newBuilder();
+    other.mergeFrom(message);
+    assertMapValuesSet(other.build());
+  }
+
+  public void testEqualsAndHashCode() throws Exception {
+    // Test that generated equals() and hashCode() will disregard the order
+    // of map entries when comparing/hashing map fields.
+    
+    // We can't control the order of elements in a HashMap. The best we can do
+    // here is to add elements in different order.
+    TestMap.Builder b1 = TestMap.newBuilder();
+    b1.getMutableInt32ToInt32Field().put(1, 2);
+    b1.getMutableInt32ToInt32Field().put(3, 4);
+    b1.getMutableInt32ToInt32Field().put(5, 6);
+    TestMap m1 = b1.build();
+    
+    TestMap.Builder b2 = TestMap.newBuilder();
+    b2.getMutableInt32ToInt32Field().put(5, 6);
+    b2.getMutableInt32ToInt32Field().put(1, 2);
+    b2.getMutableInt32ToInt32Field().put(3, 4);
+    TestMap m2 = b2.build();
+    
+    assertEquals(m1, m2);
+    assertEquals(m1.hashCode(), m2.hashCode());
+    
+    // Make sure we did compare map fields.
+    b2.getMutableInt32ToInt32Field().put(1, 0);
+    m2 = b2.build();
+    assertFalse(m1.equals(m2));
+    // Don't check m1.hashCode() != m2.hashCode() because it's not guaranteed
+    // to be different.
+  }
+
+  public void testUnknownEnumValues() throws Exception {
+    TestUnknownEnumValue.Builder builder =
+        TestUnknownEnumValue.newBuilder();
+    builder.getMutableInt32ToInt32Field().put(1, 1);
+    builder.getMutableInt32ToInt32Field().put(2, 54321);
+    ByteString data = builder.build().toByteString();
+
+    TestMap message = TestMap.parseFrom(data);
+    // Entries with unknown enum values will be stored into UnknownFieldSet so
+    // there is only one entry in the map.
+    assertEquals(1, message.getInt32ToEnumField().size());
+    assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumField().get(1));
+    // Serializing and parsing should preserve the unknown entry.
+    data = message.toByteString();
+    TestUnknownEnumValue messageWithUnknownEnums =
+        TestUnknownEnumValue.parseFrom(data);
+    assertEquals(2, messageWithUnknownEnums.getInt32ToInt32Field().size());
+    assertEquals(1, messageWithUnknownEnums.getInt32ToInt32Field().get(1).intValue());
+    assertEquals(54321, messageWithUnknownEnums.getInt32ToInt32Field().get(2).intValue());
+  }
+  
+}

java/src/test/java/com/google/protobuf/MapForProto2Test.java

@@ -0,0 +1,488 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package com.google.protobuf;
+
+import com.google.protobuf.Descriptors.FieldDescriptor;
+import map_test.MapForProto2TestProto.TestMap;
+import map_test.MapForProto2TestProto.TestMap.MessageValue;
+import map_test.MapForProto2TestProto.TestUnknownEnumValue;
+
+import junit.framework.TestCase;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Unit tests for map fields in proto2 protos.
+ */
+public class MapForProto2Test extends TestCase {
+  private void setMapValues(TestMap.Builder builder) {
+    builder.getMutableInt32ToInt32Field().put(1, 11);
+    builder.getMutableInt32ToInt32Field().put(2, 22);
+    builder.getMutableInt32ToInt32Field().put(3, 33);
+
+    builder.getMutableInt32ToStringField().put(1, "11");
+    builder.getMutableInt32ToStringField().put(2, "22");
+    builder.getMutableInt32ToStringField().put(3, "33");
+    
+    builder.getMutableInt32ToBytesField().put(1, TestUtil.toBytes("11"));
+    builder.getMutableInt32ToBytesField().put(2, TestUtil.toBytes("22"));
+    builder.getMutableInt32ToBytesField().put(3, TestUtil.toBytes("33"));
+    
+    builder.getMutableInt32ToEnumField().put(1, TestMap.EnumValue.FOO);
+    builder.getMutableInt32ToEnumField().put(2, TestMap.EnumValue.BAR);
+    builder.getMutableInt32ToEnumField().put(3, TestMap.EnumValue.BAZ);
+    
+    builder.getMutableInt32ToMessageField().put(
+        1, MessageValue.newBuilder().setValue(11).build());
+    builder.getMutableInt32ToMessageField().put(
+        2, MessageValue.newBuilder().setValue(22).build());
+    builder.getMutableInt32ToMessageField().put(
+        3, MessageValue.newBuilder().setValue(33).build());
+    
+    builder.getMutableStringToInt32Field().put("1", 11);
+    builder.getMutableStringToInt32Field().put("2", 22);
+    builder.getMutableStringToInt32Field().put("3", 33);
+  }
+
+  private void assertMapValuesSet(TestMap message) {
+    assertEquals(3, message.getInt32ToInt32Field().size());
+    assertEquals(11, message.getInt32ToInt32Field().get(1).intValue());
+    assertEquals(22, message.getInt32ToInt32Field().get(2).intValue());
+    assertEquals(33, message.getInt32ToInt32Field().get(3).intValue());
+
+    assertEquals(3, message.getInt32ToStringField().size());
+    assertEquals("11", message.getInt32ToStringField().get(1));
+    assertEquals("22", message.getInt32ToStringField().get(2));
+    assertEquals("33", message.getInt32ToStringField().get(3));
+    
+    assertEquals(3, message.getInt32ToBytesField().size());
+    assertEquals(TestUtil.toBytes("11"), message.getInt32ToBytesField().get(1));
+    assertEquals(TestUtil.toBytes("22"), message.getInt32ToBytesField().get(2));
+    assertEquals(TestUtil.toBytes("33"), message.getInt32ToBytesField().get(3));
+    
+    assertEquals(3, message.getInt32ToEnumField().size());
+    assertEquals(TestMap.EnumValue.FOO, message.getInt32ToEnumField().get(1));
+    assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumField().get(2));
+    assertEquals(TestMap.EnumValue.BAZ, message.getInt32ToEnumField().get(3));
+    
+    assertEquals(3, message.getInt32ToMessageField().size());
+    assertEquals(11, message.getInt32ToMessageField().get(1).getValue());
+    assertEquals(22, message.getInt32ToMessageField().get(2).getValue());
+    assertEquals(33, message.getInt32ToMessageField().get(3).getValue());
+    
+    assertEquals(3, message.getStringToInt32Field().size());
+    assertEquals(11, message.getStringToInt32Field().get("1").intValue());
+    assertEquals(22, message.getStringToInt32Field().get("2").intValue());
+    assertEquals(33, message.getStringToInt32Field().get("3").intValue());
+  }
+
+  private void updateMapValues(TestMap.Builder builder) {
+    builder.getMutableInt32ToInt32Field().put(1, 111);
+    builder.getMutableInt32ToInt32Field().remove(2);
+    builder.getMutableInt32ToInt32Field().put(4, 44);
+
+    builder.getMutableInt32ToStringField().put(1, "111");
+    builder.getMutableInt32ToStringField().remove(2);
+    builder.getMutableInt32ToStringField().put(4, "44");
+    
+    builder.getMutableInt32ToBytesField().put(1, TestUtil.toBytes("111"));
+    builder.getMutableInt32ToBytesField().remove(2);
+    builder.getMutableInt32ToBytesField().put(4, TestUtil.toBytes("44"));
+    
+    builder.getMutableInt32ToEnumField().put(1, TestMap.EnumValue.BAR);
+    builder.getMutableInt32ToEnumField().remove(2);
+    builder.getMutableInt32ToEnumField().put(4, TestMap.EnumValue.QUX);
+    
+    builder.getMutableInt32ToMessageField().put(
+        1, MessageValue.newBuilder().setValue(111).build());
+    builder.getMutableInt32ToMessageField().remove(2);
+    builder.getMutableInt32ToMessageField().put(
+        4, MessageValue.newBuilder().setValue(44).build());
+    
+    builder.getMutableStringToInt32Field().put("1", 111);
+    builder.getMutableStringToInt32Field().remove("2");
+    builder.getMutableStringToInt32Field().put("4", 44);
+  }
+
+  private void assertMapValuesUpdated(TestMap message) {
+    assertEquals(3, message.getInt32ToInt32Field().size());
+    assertEquals(111, message.getInt32ToInt32Field().get(1).intValue());
+    assertEquals(33, message.getInt32ToInt32Field().get(3).intValue());
+    assertEquals(44, message.getInt32ToInt32Field().get(4).intValue());
+
+    assertEquals(3, message.getInt32ToStringField().size());
+    assertEquals("111", message.getInt32ToStringField().get(1));
+    assertEquals("33", message.getInt32ToStringField().get(3));
+    assertEquals("44", message.getInt32ToStringField().get(4));
+    
+    assertEquals(3, message.getInt32ToBytesField().size());
+    assertEquals(TestUtil.toBytes("111"), message.getInt32ToBytesField().get(1));
+    assertEquals(TestUtil.toBytes("33"), message.getInt32ToBytesField().get(3));
+    assertEquals(TestUtil.toBytes("44"), message.getInt32ToBytesField().get(4));
+    
+    assertEquals(3, message.getInt32ToEnumField().size());
+    assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumField().get(1));
+    assertEquals(TestMap.EnumValue.BAZ, message.getInt32ToEnumField().get(3));
+    assertEquals(TestMap.EnumValue.QUX, message.getInt32ToEnumField().get(4));
+    
+    assertEquals(3, message.getInt32ToMessageField().size());
+    assertEquals(111, message.getInt32ToMessageField().get(1).getValue());
+    assertEquals(33, message.getInt32ToMessageField().get(3).getValue());
+    assertEquals(44, message.getInt32ToMessageField().get(4).getValue());
+    
+    assertEquals(3, message.getStringToInt32Field().size());
+    assertEquals(111, message.getStringToInt32Field().get("1").intValue());
+    assertEquals(33, message.getStringToInt32Field().get("3").intValue());
+    assertEquals(44, message.getStringToInt32Field().get("4").intValue());
+  }
+
+  private void assertMapValuesCleared(TestMap message) {
+    assertEquals(0, message.getInt32ToInt32Field().size());
+    assertEquals(0, message.getInt32ToStringField().size());
+    assertEquals(0, message.getInt32ToBytesField().size());
+    assertEquals(0, message.getInt32ToEnumField().size());
+    assertEquals(0, message.getInt32ToMessageField().size());
+    assertEquals(0, message.getStringToInt32Field().size());
+  }
+
+  public void testGettersAndSetters() throws Exception {
+    TestMap.Builder builder = TestMap.newBuilder();
+    TestMap message = builder.build();
+    assertMapValuesCleared(message);
+    
+    builder = message.toBuilder();
+    setMapValues(builder);
+    message = builder.build();
+    assertMapValuesSet(message);
+    
+    builder = message.toBuilder();
+    updateMapValues(builder);
+    message = builder.build();
+    assertMapValuesUpdated(message);
+    
+    builder = message.toBuilder();
+    builder.clear();
+    message = builder.build();
+    assertMapValuesCleared(message);
+  }
+
+  public void testSerializeAndParse() throws Exception {
+    TestMap.Builder builder = TestMap.newBuilder();
+    setMapValues(builder);
+    TestMap message = builder.build();
+    assertEquals(message.getSerializedSize(), message.toByteString().size());
+    message = TestMap.PARSER.parseFrom(message.toByteString());
+    assertMapValuesSet(message);
+    
+    builder = message.toBuilder();
+    updateMapValues(builder);
+    message = builder.build();
+    assertEquals(message.getSerializedSize(), message.toByteString().size());
+    message = TestMap.PARSER.parseFrom(message.toByteString());
+    assertMapValuesUpdated(message);
+    
+    builder = message.toBuilder();
+    builder.clear();
+    message = builder.build();
+    assertEquals(message.getSerializedSize(), message.toByteString().size());
+    message = TestMap.PARSER.parseFrom(message.toByteString());
+    assertMapValuesCleared(message);
+  }
+  
+  public void testMergeFrom() throws Exception {
+    TestMap.Builder builder = TestMap.newBuilder();
+    setMapValues(builder);
+    TestMap message = builder.build();
+    
+    TestMap.Builder other = TestMap.newBuilder();
+    other.mergeFrom(message);
+    assertMapValuesSet(other.build());
+  }
+
+  public void testEqualsAndHashCode() throws Exception {
+    // Test that generated equals() and hashCode() will disregard the order
+    // of map entries when comparing/hashing map fields.
+    
+    // We can't control the order of elements in a HashMap. The best we can do
+    // here is to add elements in different order.
+    TestMap.Builder b1 = TestMap.newBuilder();
+    b1.getMutableInt32ToInt32Field().put(1, 2);
+    b1.getMutableInt32ToInt32Field().put(3, 4);
+    b1.getMutableInt32ToInt32Field().put(5, 6);
+    TestMap m1 = b1.build();
+    
+    TestMap.Builder b2 = TestMap.newBuilder();
+    b2.getMutableInt32ToInt32Field().put(5, 6);
+    b2.getMutableInt32ToInt32Field().put(1, 2);
+    b2.getMutableInt32ToInt32Field().put(3, 4);
+    TestMap m2 = b2.build();
+    
+    assertEquals(m1, m2);
+    assertEquals(m1.hashCode(), m2.hashCode());
+    
+    // Make sure we did compare map fields.
+    b2.getMutableInt32ToInt32Field().put(1, 0);
+    m2 = b2.build();
+    assertFalse(m1.equals(m2));
+    // Don't check m1.hashCode() != m2.hashCode() because it's not guaranteed
+    // to be different.
+  }
+  
+  
+  // The following methods are used to test reflection API.
+  
+  private static FieldDescriptor f(String name) {
+    return TestMap.getDescriptor().findFieldByName(name);
+  }
+  
+  private static Object getFieldValue(Message mapEntry, String name) {
+    FieldDescriptor field = mapEntry.getDescriptorForType().findFieldByName(name);
+    return mapEntry.getField(field);
+  }
+  
+  private static Message.Builder setFieldValue(
+      Message.Builder mapEntry, String name, Object value) {
+    FieldDescriptor field = mapEntry.getDescriptorForType().findFieldByName(name);
+    mapEntry.setField(field, value);
+    return mapEntry;
+  }
+  
+  private static void assertHasMapValues(Message message, String name, Map<?, ?> values) {
+    FieldDescriptor field = f(name);
+    for (Object entry : (List<?>) message.getField(field)) {
+      Message mapEntry = (Message) entry;
+      Object key = getFieldValue(mapEntry, "key");
+      Object value = getFieldValue(mapEntry, "value");
+      assertTrue(values.containsKey(key));
+      assertEquals(value, values.get(key));
+    }
+    assertEquals(values.size(), message.getRepeatedFieldCount(field));
+    for (int i = 0; i < message.getRepeatedFieldCount(field); i++) {
+      Message mapEntry = (Message) message.getRepeatedField(field, i);
+      Object key = getFieldValue(mapEntry, "key");
+      Object value = getFieldValue(mapEntry, "value");
+      assertTrue(values.containsKey(key));
+      assertEquals(value, values.get(key));
+    }
+  }
+  
+  private static <KeyType, ValueType>
+  Message newMapEntry(Message.Builder builder, String name, KeyType key, ValueType value) {
+    FieldDescriptor field = builder.getDescriptorForType().findFieldByName(name);
+    Message.Builder entryBuilder = builder.newBuilderForField(field);
+    FieldDescriptor keyField = entryBuilder.getDescriptorForType().findFieldByName("key");
+    FieldDescriptor valueField = entryBuilder.getDescriptorForType().findFieldByName("value");
+    entryBuilder.setField(keyField, key);
+    entryBuilder.setField(valueField, value);
+    return entryBuilder.build();
+  }
+  
+  private static void setMapValues(Message.Builder builder, String name, Map<?, ?> values) {
+    List<Message> entryList = new ArrayList<Message>();
+    for (Map.Entry<?, ?> entry : values.entrySet()) {
+      entryList.add(newMapEntry(builder, name, entry.getKey(), entry.getValue()));
+    }
+    FieldDescriptor field = builder.getDescriptorForType().findFieldByName(name);
+    builder.setField(field, entryList);
+  }
+  
+  private static <KeyType, ValueType>
+  Map<KeyType, ValueType> mapForValues(
+      KeyType key1, ValueType value1, KeyType key2, ValueType value2) {
+    Map<KeyType, ValueType> map = new HashMap<KeyType, ValueType>();
+    map.put(key1, value1);
+    map.put(key2, value2);
+    return map;
+  }
+
+  public void testReflectionApi() throws Exception {
+    // In reflection API, map fields are just repeated message fields.
+    TestMap.Builder builder = TestMap.newBuilder();
+    builder.getMutableInt32ToInt32Field().put(1, 2);
+    builder.getMutableInt32ToInt32Field().put(3, 4);
+    builder.getMutableInt32ToMessageField().put(
+        11, MessageValue.newBuilder().setValue(22).build());
+    builder.getMutableInt32ToMessageField().put(
+        33, MessageValue.newBuilder().setValue(44).build());
+    TestMap message = builder.build();
+
+    // Test getField(), getRepeatedFieldCount(), getRepeatedField().
+    assertHasMapValues(message, "int32_to_int32_field",
+        mapForValues(1, 2, 3, 4));
+    assertHasMapValues(message, "int32_to_message_field",
+        mapForValues(
+            11, MessageValue.newBuilder().setValue(22).build(),
+            33, MessageValue.newBuilder().setValue(44).build()));
+    
+    // Test clearField()
+    builder.clearField(f("int32_to_int32_field"));
+    builder.clearField(f("int32_to_message_field"));
+    message = builder.build();
+    assertEquals(0, message.getInt32ToInt32Field().size());
+    assertEquals(0, message.getInt32ToMessageField().size());
+    
+    // Test setField()
+    setMapValues(builder, "int32_to_int32_field",
+        mapForValues(11, 22, 33, 44));
+    setMapValues(builder, "int32_to_message_field",
+        mapForValues(
+            111, MessageValue.newBuilder().setValue(222).build(),
+            333, MessageValue.newBuilder().setValue(444).build()));
+    message = builder.build();
+    assertEquals(22, message.getInt32ToInt32Field().get(11).intValue());
+    assertEquals(44, message.getInt32ToInt32Field().get(33).intValue());
+    assertEquals(222, message.getInt32ToMessageField().get(111).getValue());
+    assertEquals(444, message.getInt32ToMessageField().get(333).getValue());
+    
+    // Test addRepeatedField
+    builder.addRepeatedField(f("int32_to_int32_field"),
+        newMapEntry(builder, "int32_to_int32_field", 55, 66));
+    builder.addRepeatedField(f("int32_to_message_field"),
+        newMapEntry(builder, "int32_to_message_field", 555,
+            MessageValue.newBuilder().setValue(666).build()));
+    message = builder.build();
+    assertEquals(66, message.getInt32ToInt32Field().get(55).intValue());
+    assertEquals(666, message.getInt32ToMessageField().get(555).getValue());
+
+    // Test addRepeatedField (overriding existing values)
+    builder.addRepeatedField(f("int32_to_int32_field"),
+        newMapEntry(builder, "int32_to_int32_field", 55, 55));
+    builder.addRepeatedField(f("int32_to_message_field"),
+        newMapEntry(builder, "int32_to_message_field", 555,
+            MessageValue.newBuilder().setValue(555).build()));
+    message = builder.build();
+    assertEquals(55, message.getInt32ToInt32Field().get(55).intValue());
+    assertEquals(555, message.getInt32ToMessageField().get(555).getValue());
+    
+    // Test setRepeatedField
+    for (int i = 0; i < builder.getRepeatedFieldCount(f("int32_to_int32_field")); i++) {
+      Message mapEntry = (Message) builder.getRepeatedField(f("int32_to_int32_field"), i);
+      int oldKey = ((Integer) getFieldValue(mapEntry, "key")).intValue();
+      int oldValue = ((Integer) getFieldValue(mapEntry, "value")).intValue();
+      // Swap key with value for each entry.
+      Message.Builder mapEntryBuilder = mapEntry.toBuilder();
+      setFieldValue(mapEntryBuilder, "key", oldValue);
+      setFieldValue(mapEntryBuilder, "value", oldKey);
+      builder.setRepeatedField(f("int32_to_int32_field"), i, mapEntryBuilder.build());
+    }
+    message = builder.build();
+    assertEquals(11, message.getInt32ToInt32Field().get(22).intValue());
+    assertEquals(33, message.getInt32ToInt32Field().get(44).intValue());
+    assertEquals(55, message.getInt32ToInt32Field().get(55).intValue());
+  }
+  
+  public void testTextFormat() throws Exception {
+    TestMap.Builder builder = TestMap.newBuilder();
+    setMapValues(builder);
+    TestMap message = builder.build();
+    
+    String textData = TextFormat.printToString(message);
+    
+    builder = TestMap.newBuilder();
+    TextFormat.merge(textData, builder);
+    message = builder.build();
+    
+    assertMapValuesSet(message);
+  }
+  
+  public void testDynamicMessage() throws Exception {
+    TestMap.Builder builder = TestMap.newBuilder();
+    setMapValues(builder);
+    TestMap message = builder.build();
+    
+    Message dynamicDefaultInstance =
+        DynamicMessage.getDefaultInstance(TestMap.getDescriptor());
+    Message dynamicMessage = dynamicDefaultInstance
+        .newBuilderForType().mergeFrom(message.toByteString()).build();
+    
+    assertEquals(message, dynamicMessage);
+    assertEquals(message.hashCode(), dynamicMessage.hashCode());
+  }
+  
+  public void testReflectionEqualsAndHashCode() throws Exception {
+    // Test that generated equals() and hashCode() will disregard the order
+    // of map entries when comparing/hashing map fields.
+
+    // We use DynamicMessage to test reflection based equals()/hashCode().
+    Message dynamicDefaultInstance =
+        DynamicMessage.getDefaultInstance(TestMap.getDescriptor());
+    FieldDescriptor field = f("int32_to_int32_field");
+    
+    Message.Builder b1 = dynamicDefaultInstance.newBuilderForType();
+    b1.addRepeatedField(field, newMapEntry(b1, "int32_to_int32_field", 1, 2));
+    b1.addRepeatedField(field, newMapEntry(b1, "int32_to_int32_field", 3, 4));
+    b1.addRepeatedField(field, newMapEntry(b1, "int32_to_int32_field", 5, 6));
+    Message m1 = b1.build();
+    
+    Message.Builder b2 = dynamicDefaultInstance.newBuilderForType();
+    b2.addRepeatedField(field, newMapEntry(b2, "int32_to_int32_field", 5, 6));
+    b2.addRepeatedField(field, newMapEntry(b2, "int32_to_int32_field", 1, 2));
+    b2.addRepeatedField(field, newMapEntry(b2, "int32_to_int32_field", 3, 4));
+    Message m2 = b2.build();
+    
+    assertEquals(m1, m2);
+    assertEquals(m1.hashCode(), m2.hashCode());
+    
+    // Make sure we did compare map fields.
+    b2.setRepeatedField(field, 0, newMapEntry(b1, "int32_to_int32_field", 0, 0));
+    m2 = b2.build();
+    assertFalse(m1.equals(m2));
+    // Don't check m1.hashCode() != m2.hashCode() because it's not guaranteed
+    // to be different.
+  }
+  
+  public void testUnknownEnumValues() throws Exception {
+    TestUnknownEnumValue.Builder builder =
+        TestUnknownEnumValue.newBuilder();
+    builder.getMutableInt32ToInt32Field().put(1, 1);
+    builder.getMutableInt32ToInt32Field().put(2, 54321);
+    ByteString data = builder.build().toByteString();
+
+    TestMap message = TestMap.parseFrom(data);
+    // Entries with unknown enum values will be stored into UnknownFieldSet so
+    // there is only one entry in the map.
+    assertEquals(1, message.getInt32ToEnumField().size());
+    assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumField().get(1));
+    // UnknownFieldSet should not be empty.
+    assertFalse(message.getUnknownFields().asMap().isEmpty());
+    // Serializing and parsing should preserve the unknown entry.
+    data = message.toByteString();
+    TestUnknownEnumValue messageWithUnknownEnums =
+        TestUnknownEnumValue.parseFrom(data);
+    assertEquals(2, messageWithUnknownEnums.getInt32ToInt32Field().size());
+    assertEquals(1, messageWithUnknownEnums.getInt32ToInt32Field().get(1).intValue());
+    assertEquals(54321, messageWithUnknownEnums.getInt32ToInt32Field().get(2).intValue());
+  }
+
+}

java/src/test/java/com/google/protobuf/MapTest.java

@@ -0,0 +1,569 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package com.google.protobuf;
+
+import com.google.protobuf.Descriptors.Descriptor;
+import com.google.protobuf.Descriptors.EnumDescriptor;
+import com.google.protobuf.Descriptors.EnumValueDescriptor;
+import com.google.protobuf.Descriptors.FieldDescriptor;
+import map_test.MapTestProto.TestMap;
+import map_test.MapTestProto.TestMap.MessageValue;
+import map_test.MapTestProto.TestOnChangeEventPropagation;
+
+import junit.framework.TestCase;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Unit tests for map fields.
+ */
+public class MapTest extends TestCase {
+  private void setMapValues(TestMap.Builder builder) {
+    builder.getMutableInt32ToInt32Field().put(1, 11);
+    builder.getMutableInt32ToInt32Field().put(2, 22);
+    builder.getMutableInt32ToInt32Field().put(3, 33);
+
+    builder.getMutableInt32ToStringField().put(1, "11");
+    builder.getMutableInt32ToStringField().put(2, "22");
+    builder.getMutableInt32ToStringField().put(3, "33");
+    
+    builder.getMutableInt32ToBytesField().put(1, TestUtil.toBytes("11"));
+    builder.getMutableInt32ToBytesField().put(2, TestUtil.toBytes("22"));
+    builder.getMutableInt32ToBytesField().put(3, TestUtil.toBytes("33"));
+    
+    builder.getMutableInt32ToEnumField().put(1, TestMap.EnumValue.FOO);
+    builder.getMutableInt32ToEnumField().put(2, TestMap.EnumValue.BAR);
+    builder.getMutableInt32ToEnumField().put(3, TestMap.EnumValue.BAZ);
+    
+    builder.getMutableInt32ToMessageField().put(
+        1, MessageValue.newBuilder().setValue(11).build());
+    builder.getMutableInt32ToMessageField().put(
+        2, MessageValue.newBuilder().setValue(22).build());
+    builder.getMutableInt32ToMessageField().put(
+        3, MessageValue.newBuilder().setValue(33).build());
+    
+    builder.getMutableStringToInt32Field().put("1", 11);
+    builder.getMutableStringToInt32Field().put("2", 22);
+    builder.getMutableStringToInt32Field().put("3", 33);
+  }
+
+  private void assertMapValuesSet(TestMap message) {
+    assertEquals(3, message.getInt32ToInt32Field().size());
+    assertEquals(11, message.getInt32ToInt32Field().get(1).intValue());
+    assertEquals(22, message.getInt32ToInt32Field().get(2).intValue());
+    assertEquals(33, message.getInt32ToInt32Field().get(3).intValue());
+
+    assertEquals(3, message.getInt32ToStringField().size());
+    assertEquals("11", message.getInt32ToStringField().get(1));
+    assertEquals("22", message.getInt32ToStringField().get(2));
+    assertEquals("33", message.getInt32ToStringField().get(3));
+    
+    assertEquals(3, message.getInt32ToBytesField().size());
+    assertEquals(TestUtil.toBytes("11"), message.getInt32ToBytesField().get(1));
+    assertEquals(TestUtil.toBytes("22"), message.getInt32ToBytesField().get(2));
+    assertEquals(TestUtil.toBytes("33"), message.getInt32ToBytesField().get(3));
+    
+    assertEquals(3, message.getInt32ToEnumField().size());
+    assertEquals(TestMap.EnumValue.FOO, message.getInt32ToEnumField().get(1));
+    assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumField().get(2));
+    assertEquals(TestMap.EnumValue.BAZ, message.getInt32ToEnumField().get(3));
+    
+    assertEquals(3, message.getInt32ToMessageField().size());
+    assertEquals(11, message.getInt32ToMessageField().get(1).getValue());
+    assertEquals(22, message.getInt32ToMessageField().get(2).getValue());
+    assertEquals(33, message.getInt32ToMessageField().get(3).getValue());
+    
+    assertEquals(3, message.getStringToInt32Field().size());
+    assertEquals(11, message.getStringToInt32Field().get("1").intValue());
+    assertEquals(22, message.getStringToInt32Field().get("2").intValue());
+    assertEquals(33, message.getStringToInt32Field().get("3").intValue());
+  }
+
+  private void updateMapValues(TestMap.Builder builder) {
+    builder.getMutableInt32ToInt32Field().put(1, 111);
+    builder.getMutableInt32ToInt32Field().remove(2);
+    builder.getMutableInt32ToInt32Field().put(4, 44);
+
+    builder.getMutableInt32ToStringField().put(1, "111");
+    builder.getMutableInt32ToStringField().remove(2);
+    builder.getMutableInt32ToStringField().put(4, "44");
+    
+    builder.getMutableInt32ToBytesField().put(1, TestUtil.toBytes("111"));
+    builder.getMutableInt32ToBytesField().remove(2);
+    builder.getMutableInt32ToBytesField().put(4, TestUtil.toBytes("44"));
+    
+    builder.getMutableInt32ToEnumField().put(1, TestMap.EnumValue.BAR);
+    builder.getMutableInt32ToEnumField().remove(2);
+    builder.getMutableInt32ToEnumField().put(4, TestMap.EnumValue.QUX);
+    
+    builder.getMutableInt32ToMessageField().put(
+        1, MessageValue.newBuilder().setValue(111).build());
+    builder.getMutableInt32ToMessageField().remove(2);
+    builder.getMutableInt32ToMessageField().put(
+        4, MessageValue.newBuilder().setValue(44).build());
+    
+    builder.getMutableStringToInt32Field().put("1", 111);
+    builder.getMutableStringToInt32Field().remove("2");
+    builder.getMutableStringToInt32Field().put("4", 44);
+  }
+
+  private void assertMapValuesUpdated(TestMap message) {
+    assertEquals(3, message.getInt32ToInt32Field().size());
+    assertEquals(111, message.getInt32ToInt32Field().get(1).intValue());
+    assertEquals(33, message.getInt32ToInt32Field().get(3).intValue());
+    assertEquals(44, message.getInt32ToInt32Field().get(4).intValue());
+
+    assertEquals(3, message.getInt32ToStringField().size());
+    assertEquals("111", message.getInt32ToStringField().get(1));
+    assertEquals("33", message.getInt32ToStringField().get(3));
+    assertEquals("44", message.getInt32ToStringField().get(4));
+    
+    assertEquals(3, message.getInt32ToBytesField().size());
+    assertEquals(TestUtil.toBytes("111"), message.getInt32ToBytesField().get(1));
+    assertEquals(TestUtil.toBytes("33"), message.getInt32ToBytesField().get(3));
+    assertEquals(TestUtil.toBytes("44"), message.getInt32ToBytesField().get(4));
+    
+    assertEquals(3, message.getInt32ToEnumField().size());
+    assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumField().get(1));
+    assertEquals(TestMap.EnumValue.BAZ, message.getInt32ToEnumField().get(3));
+    assertEquals(TestMap.EnumValue.QUX, message.getInt32ToEnumField().get(4));
+    
+    assertEquals(3, message.getInt32ToMessageField().size());
+    assertEquals(111, message.getInt32ToMessageField().get(1).getValue());
+    assertEquals(33, message.getInt32ToMessageField().get(3).getValue());
+    assertEquals(44, message.getInt32ToMessageField().get(4).getValue());
+    
+    assertEquals(3, message.getStringToInt32Field().size());
+    assertEquals(111, message.getStringToInt32Field().get("1").intValue());
+    assertEquals(33, message.getStringToInt32Field().get("3").intValue());
+    assertEquals(44, message.getStringToInt32Field().get("4").intValue());
+  }
+
+  private void assertMapValuesCleared(TestMap message) {
+    assertEquals(0, message.getInt32ToInt32Field().size());
+    assertEquals(0, message.getInt32ToStringField().size());
+    assertEquals(0, message.getInt32ToBytesField().size());
+    assertEquals(0, message.getInt32ToEnumField().size());
+    assertEquals(0, message.getInt32ToMessageField().size());
+    assertEquals(0, message.getStringToInt32Field().size());
+  }
+
+  public void testGettersAndSetters() throws Exception {
+    TestMap.Builder builder = TestMap.newBuilder();
+    TestMap message = builder.build();
+    assertMapValuesCleared(message);
+    
+    builder = message.toBuilder();
+    setMapValues(builder);
+    message = builder.build();
+    assertMapValuesSet(message);
+    
+    builder = message.toBuilder();
+    updateMapValues(builder);
+    message = builder.build();
+    assertMapValuesUpdated(message);
+    
+    builder = message.toBuilder();
+    builder.clear();
+    message = builder.build();
+    assertMapValuesCleared(message);
+  }
+
+  public void testSerializeAndParse() throws Exception {
+    TestMap.Builder builder = TestMap.newBuilder();
+    setMapValues(builder);
+    TestMap message = builder.build();
+    assertEquals(message.getSerializedSize(), message.toByteString().size());
+    message = TestMap.PARSER.parseFrom(message.toByteString());
+    assertMapValuesSet(message);
+    
+    builder = message.toBuilder();
+    updateMapValues(builder);
+    message = builder.build();
+    assertEquals(message.getSerializedSize(), message.toByteString().size());
+    message = TestMap.PARSER.parseFrom(message.toByteString());
+    assertMapValuesUpdated(message);
+    
+    builder = message.toBuilder();
+    builder.clear();
+    message = builder.build();
+    assertEquals(message.getSerializedSize(), message.toByteString().size());
+    message = TestMap.PARSER.parseFrom(message.toByteString());
+    assertMapValuesCleared(message);
+  }
+  
+  public void testMergeFrom() throws Exception {
+    TestMap.Builder builder = TestMap.newBuilder();
+    setMapValues(builder);
+    TestMap message = builder.build();
+    
+    TestMap.Builder other = TestMap.newBuilder();
+    other.mergeFrom(message);
+    assertMapValuesSet(other.build());
+  }
+
+  public void testEqualsAndHashCode() throws Exception {
+    // Test that generated equals() and hashCode() will disregard the order
+    // of map entries when comparing/hashing map fields.
+    
+    // We can't control the order of elements in a HashMap. The best we can do
+    // here is to add elements in different order.
+    TestMap.Builder b1 = TestMap.newBuilder();
+    b1.getMutableInt32ToInt32Field().put(1, 2);
+    b1.getMutableInt32ToInt32Field().put(3, 4);
+    b1.getMutableInt32ToInt32Field().put(5, 6);
+    TestMap m1 = b1.build();
+    
+    TestMap.Builder b2 = TestMap.newBuilder();
+    b2.getMutableInt32ToInt32Field().put(5, 6);
+    b2.getMutableInt32ToInt32Field().put(1, 2);
+    b2.getMutableInt32ToInt32Field().put(3, 4);
+    TestMap m2 = b2.build();
+    
+    assertEquals(m1, m2);
+    assertEquals(m1.hashCode(), m2.hashCode());
+    
+    // Make sure we did compare map fields.
+    b2.getMutableInt32ToInt32Field().put(1, 0);
+    m2 = b2.build();
+    assertFalse(m1.equals(m2));
+    // Don't check m1.hashCode() != m2.hashCode() because it's not guaranteed
+    // to be different.
+  }
+  
+  
+  public void testNestedBuilderOnChangeEventPropagation() {
+    TestOnChangeEventPropagation.Builder parent =
+        TestOnChangeEventPropagation.newBuilder();
+    parent.getOptionalMessageBuilder().getMutableInt32ToInt32Field().put(1, 2);
+    TestOnChangeEventPropagation message = parent.build();
+    assertEquals(2, message.getOptionalMessage().getInt32ToInt32Field().get(1).intValue());
+    
+    // Make a change using nested builder.
+    parent.getOptionalMessageBuilder().getMutableInt32ToInt32Field().put(1, 3);
+    
+    // Should be able to observe the change.
+    message = parent.build();
+    assertEquals(3, message.getOptionalMessage().getInt32ToInt32Field().get(1).intValue());
+    
+    // Make another change using mergeFrom()
+    TestMap.Builder other = TestMap.newBuilder();
+    other.getMutableInt32ToInt32Field().put(1, 4);
+    parent.getOptionalMessageBuilder().mergeFrom(other.build());
+    
+    // Should be able to observe the change.
+    message = parent.build();
+    assertEquals(4, message.getOptionalMessage().getInt32ToInt32Field().get(1).intValue());
+    
+    // Make yet another change by clearing the nested builder.
+    parent.getOptionalMessageBuilder().clear();
+    
+    // Should be able to observe the change.
+    message = parent.build();
+    assertEquals(0, message.getOptionalMessage().getInt32ToInt32Field().size());
+  }
+  
+  // The following methods are used to test reflection API.
+  
+  private static FieldDescriptor f(String name) {
+    return TestMap.getDescriptor().findFieldByName(name);
+  }
+  
+  private static Object getFieldValue(Message mapEntry, String name) {
+    FieldDescriptor field = mapEntry.getDescriptorForType().findFieldByName(name);
+    return mapEntry.getField(field);
+  }
+  
+  private static Message.Builder setFieldValue(
+      Message.Builder mapEntry, String name, Object value) {
+    FieldDescriptor field = mapEntry.getDescriptorForType().findFieldByName(name);
+    mapEntry.setField(field, value);
+    return mapEntry;
+  }
+  
+  private static void assertHasMapValues(Message message, String name, Map<?, ?> values) {
+    FieldDescriptor field = f(name);
+    for (Object entry : (List<?>) message.getField(field)) {
+      Message mapEntry = (Message) entry;
+      Object key = getFieldValue(mapEntry, "key");
+      Object value = getFieldValue(mapEntry, "value");
+      assertTrue(values.containsKey(key));
+      assertEquals(value, values.get(key));
+    }
+    assertEquals(values.size(), message.getRepeatedFieldCount(field));
+    for (int i = 0; i < message.getRepeatedFieldCount(field); i++) {
+      Message mapEntry = (Message) message.getRepeatedField(field, i);
+      Object key = getFieldValue(mapEntry, "key");
+      Object value = getFieldValue(mapEntry, "value");
+      assertTrue(values.containsKey(key));
+      assertEquals(value, values.get(key));
+    }
+  }
+  
+  private static <KeyType, ValueType>
+  Message newMapEntry(Message.Builder builder, String name, KeyType key, ValueType value) {
+    FieldDescriptor field = builder.getDescriptorForType().findFieldByName(name);
+    Message.Builder entryBuilder = builder.newBuilderForField(field);
+    FieldDescriptor keyField = entryBuilder.getDescriptorForType().findFieldByName("key");
+    FieldDescriptor valueField = entryBuilder.getDescriptorForType().findFieldByName("value");
+    entryBuilder.setField(keyField, key);
+    entryBuilder.setField(valueField, value);
+    return entryBuilder.build();
+  }
+  
+  private static void setMapValues(Message.Builder builder, String name, Map<?, ?> values) {
+    List<Message> entryList = new ArrayList<Message>();
+    for (Map.Entry<?, ?> entry : values.entrySet()) {
+      entryList.add(newMapEntry(builder, name, entry.getKey(), entry.getValue()));
+    }
+    FieldDescriptor field = builder.getDescriptorForType().findFieldByName(name);
+    builder.setField(field, entryList);
+  }
+  
+  private static <KeyType, ValueType>
+  Map<KeyType, ValueType> mapForValues(
+      KeyType key1, ValueType value1, KeyType key2, ValueType value2) {
+    Map<KeyType, ValueType> map = new HashMap<KeyType, ValueType>();
+    map.put(key1, value1);
+    map.put(key2, value2);
+    return map;
+  }
+
+  public void testReflectionApi() throws Exception {
+    // In reflection API, map fields are just repeated message fields.
+    TestMap.Builder builder = TestMap.newBuilder();
+    builder.getMutableInt32ToInt32Field().put(1, 2);
+    builder.getMutableInt32ToInt32Field().put(3, 4);
+    builder.getMutableInt32ToMessageField().put(
+        11, MessageValue.newBuilder().setValue(22).build());
+    builder.getMutableInt32ToMessageField().put(
+        33, MessageValue.newBuilder().setValue(44).build());
+    TestMap message = builder.build();
+
+    // Test getField(), getRepeatedFieldCount(), getRepeatedField().
+    assertHasMapValues(message, "int32_to_int32_field",
+        mapForValues(1, 2, 3, 4));
+    assertHasMapValues(message, "int32_to_message_field",
+        mapForValues(
+            11, MessageValue.newBuilder().setValue(22).build(),
+            33, MessageValue.newBuilder().setValue(44).build()));
+    
+    // Test clearField()
+    builder.clearField(f("int32_to_int32_field"));
+    builder.clearField(f("int32_to_message_field"));
+    message = builder.build();
+    assertEquals(0, message.getInt32ToInt32Field().size());
+    assertEquals(0, message.getInt32ToMessageField().size());
+    
+    // Test setField()
+    setMapValues(builder, "int32_to_int32_field",
+        mapForValues(11, 22, 33, 44));
+    setMapValues(builder, "int32_to_message_field",
+        mapForValues(
+            111, MessageValue.newBuilder().setValue(222).build(),
+            333, MessageValue.newBuilder().setValue(444).build()));
+    message = builder.build();
+    assertEquals(22, message.getInt32ToInt32Field().get(11).intValue());
+    assertEquals(44, message.getInt32ToInt32Field().get(33).intValue());
+    assertEquals(222, message.getInt32ToMessageField().get(111).getValue());
+    assertEquals(444, message.getInt32ToMessageField().get(333).getValue());
+    
+    // Test addRepeatedField
+    builder.addRepeatedField(f("int32_to_int32_field"),
+        newMapEntry(builder, "int32_to_int32_field", 55, 66));
+    builder.addRepeatedField(f("int32_to_message_field"),
+        newMapEntry(builder, "int32_to_message_field", 555,
+            MessageValue.newBuilder().setValue(666).build()));
+    message = builder.build();
+    assertEquals(66, message.getInt32ToInt32Field().get(55).intValue());
+    assertEquals(666, message.getInt32ToMessageField().get(555).getValue());
+
+    // Test addRepeatedField (overriding existing values)
+    builder.addRepeatedField(f("int32_to_int32_field"),
+        newMapEntry(builder, "int32_to_int32_field", 55, 55));
+    builder.addRepeatedField(f("int32_to_message_field"),
+        newMapEntry(builder, "int32_to_message_field", 555,
+            MessageValue.newBuilder().setValue(555).build()));
+    message = builder.build();
+    assertEquals(55, message.getInt32ToInt32Field().get(55).intValue());
+    assertEquals(555, message.getInt32ToMessageField().get(555).getValue());
+    
+    // Test setRepeatedField
+    for (int i = 0; i < builder.getRepeatedFieldCount(f("int32_to_int32_field")); i++) {
+      Message mapEntry = (Message) builder.getRepeatedField(f("int32_to_int32_field"), i);
+      int oldKey = ((Integer) getFieldValue(mapEntry, "key")).intValue();
+      int oldValue = ((Integer) getFieldValue(mapEntry, "value")).intValue();
+      // Swap key with value for each entry.
+      Message.Builder mapEntryBuilder = mapEntry.toBuilder();
+      setFieldValue(mapEntryBuilder, "key", oldValue);
+      setFieldValue(mapEntryBuilder, "value", oldKey);
+      builder.setRepeatedField(f("int32_to_int32_field"), i, mapEntryBuilder.build());
+    }
+    message = builder.build();
+    assertEquals(11, message.getInt32ToInt32Field().get(22).intValue());
+    assertEquals(33, message.getInt32ToInt32Field().get(44).intValue());
+    assertEquals(55, message.getInt32ToInt32Field().get(55).intValue());
+  }
+  
+  public void testTextFormat() throws Exception {
+    TestMap.Builder builder = TestMap.newBuilder();
+    setMapValues(builder);
+    TestMap message = builder.build();
+    
+    String textData = TextFormat.printToString(message);
+    
+    builder = TestMap.newBuilder();
+    TextFormat.merge(textData, builder);
+    message = builder.build();
+    
+    assertMapValuesSet(message);
+  }
+  
+  public void testDynamicMessage() throws Exception {
+    TestMap.Builder builder = TestMap.newBuilder();
+    setMapValues(builder);
+    TestMap message = builder.build();
+    
+    Message dynamicDefaultInstance =
+        DynamicMessage.getDefaultInstance(TestMap.getDescriptor());
+    Message dynamicMessage = dynamicDefaultInstance
+        .newBuilderForType().mergeFrom(message.toByteString()).build();
+    
+    assertEquals(message, dynamicMessage);
+    assertEquals(message.hashCode(), dynamicMessage.hashCode());
+  }
+  
+  public void testReflectionEqualsAndHashCode() throws Exception {
+    // Test that generated equals() and hashCode() will disregard the order
+    // of map entries when comparing/hashing map fields.
+
+    // We use DynamicMessage to test reflection based equals()/hashCode().
+    Message dynamicDefaultInstance =
+        DynamicMessage.getDefaultInstance(TestMap.getDescriptor());
+    FieldDescriptor field = f("int32_to_int32_field");
+    
+    Message.Builder b1 = dynamicDefaultInstance.newBuilderForType();
+    b1.addRepeatedField(field, newMapEntry(b1, "int32_to_int32_field", 1, 2));
+    b1.addRepeatedField(field, newMapEntry(b1, "int32_to_int32_field", 3, 4));
+    b1.addRepeatedField(field, newMapEntry(b1, "int32_to_int32_field", 5, 6));
+    Message m1 = b1.build();
+    
+    Message.Builder b2 = dynamicDefaultInstance.newBuilderForType();
+    b2.addRepeatedField(field, newMapEntry(b2, "int32_to_int32_field", 5, 6));
+    b2.addRepeatedField(field, newMapEntry(b2, "int32_to_int32_field", 1, 2));
+    b2.addRepeatedField(field, newMapEntry(b2, "int32_to_int32_field", 3, 4));
+    Message m2 = b2.build();
+    
+    assertEquals(m1, m2);
+    assertEquals(m1.hashCode(), m2.hashCode());
+    
+    // Make sure we did compare map fields.
+    b2.setRepeatedField(field, 0, newMapEntry(b1, "int32_to_int32_field", 0, 0));
+    m2 = b2.build();
+    assertFalse(m1.equals(m2));
+    // Don't check m1.hashCode() != m2.hashCode() because it's not guaranteed
+    // to be different.
+  }
+  
+  public void testUnknownEnumValues() throws Exception {
+    TestMap.Builder builder = TestMap.newBuilder();
+    builder.getMutableInt32ToEnumFieldValue().put(0, 0);
+    builder.getMutableInt32ToEnumFieldValue().put(1, 1);
+    builder.getMutableInt32ToEnumFieldValue().put(2, 1000);  // unknown value.
+    TestMap message = builder.build();
+
+    assertEquals(TestMap.EnumValue.FOO,
+        message.getInt32ToEnumField().get(0));
+    assertEquals(TestMap.EnumValue.BAR,
+        message.getInt32ToEnumField().get(1));
+    assertEquals(TestMap.EnumValue.UNRECOGNIZED,
+        message.getInt32ToEnumField().get(2));
+    assertEquals(1000, message.getInt32ToEnumFieldValue().get(2).intValue());
+    
+    // Unknown enum values should be preserved after:
+    //   1. Serialization and parsing.
+    //   2. toBuild().
+    //   3. mergeFrom().
+    message = TestMap.parseFrom(message.toByteString());
+    assertEquals(1000, message.getInt32ToEnumFieldValue().get(2).intValue());
+    builder = message.toBuilder();
+    assertEquals(1000, builder.getInt32ToEnumFieldValue().get(2).intValue());
+    builder = TestMap.newBuilder().mergeFrom(message);
+    assertEquals(1000, builder.getInt32ToEnumFieldValue().get(2).intValue());
+    
+    // hashCode()/equals() should take unknown enum values into account.
+    builder.getMutableInt32ToEnumFieldValue().put(2, 1001);
+    TestMap message2 = builder.build();
+    assertFalse(message.hashCode() == message2.hashCode());
+    assertFalse(message.equals(message2));
+    // Unknown values will be converted to UNRECOGNIZED so the resulted enum map
+    // should be the same.
+    assertTrue(message.getInt32ToEnumField().equals(message2.getInt32ToEnumField()));
+  }
+  
+  public void testUnknownEnumValuesInReflectionApi() throws Exception {
+    Descriptor descriptor = TestMap.getDescriptor();
+    EnumDescriptor enumDescriptor = TestMap.EnumValue.getDescriptor();
+    FieldDescriptor field = descriptor.findFieldByName("int32_to_enum_field");
+    
+    Map<Integer, Integer> data = new HashMap<Integer, Integer>();
+    data.put(0, 0);
+    data.put(1, 1);
+    data.put(2, 1000);  // unknown value.
+    
+    TestMap.Builder builder = TestMap.newBuilder();
+    for (Map.Entry<Integer, Integer> entry : data.entrySet()) {
+      builder.getMutableInt32ToEnumFieldValue().put(entry.getKey(), entry.getValue());
+    }
+
+    // Try to read unknown enum values using reflection API.
+    for (int i = 0; i < builder.getRepeatedFieldCount(field); i++) {
+      Message mapEntry = (Message) builder.getRepeatedField(field, i);
+      int key = ((Integer) getFieldValue(mapEntry, "key")).intValue();
+      int value = ((EnumValueDescriptor) getFieldValue(mapEntry, "value")).getNumber();
+      assertEquals(data.get(key).intValue(), value);
+      Message.Builder mapEntryBuilder = mapEntry.toBuilder();
+      // Increase the value by 1.
+      setFieldValue(mapEntryBuilder, "value",
+          enumDescriptor.findValueByNumberCreatingIfUnknown(value + 1));
+      builder.setRepeatedField(field, i, mapEntryBuilder.build());
+    }
+
+    // Verify that enum values have been successfully updated.
+    TestMap message = builder.build();
+    for (Map.Entry<Integer, Integer> entry : message.getInt32ToEnumFieldValue().entrySet()) {
+      assertEquals(data.get(entry.getKey()) + 1, entry.getValue().intValue());
+    }
+  }
+}

java/src/test/java/com/google/protobuf/TestBadIdentifiers.java

@@ -72,7 +72,7 @@ public class TestBadIdentifiers extends TestCase {
     assertEquals(0, message.getMessageField5Count());
 
     assertEquals(0, message.getInt32FieldCount11());
-    assertEquals(1, message.getEnumFieldCount12().getNumber());
+    assertEquals(0, message.getEnumFieldCount12().getNumber());
     assertEquals("", message.getStringFieldCount13());
     assertEquals(ByteString.EMPTY, message.getBytesFieldCount14());
     assertEquals(0, message.getMessageFieldCount15().getSerializedSize());

java/src/test/java/com/google/protobuf/TextFormatTest.java

@@ -73,7 +73,7 @@ public class TextFormatTest extends TestCase {
   private static String exoticText =
     "repeated_int32: -1\n" +
     "repeated_int32: -2147483648\n" +
-    "repeated_int64: -1\n" +
+    "repeated_int64: -1,\n" +
     "repeated_int64: -9223372036854775808\n" +
     "repeated_uint32: 4294967295\n" +
     "repeated_uint32: 2147483648\n" +
@@ -101,7 +101,7 @@ public class TextFormatTest extends TestCase {
 
   private static String canonicalExoticText =
       exoticText.replace(": .", ": 0.").replace(": -.", ": -0.")   // short-form double
-      .replace("23e", "23E").replace("E+", "E").replace("0.23E17", "2.3E16");
+      .replace("23e", "23E").replace("E+", "E").replace("0.23E17", "2.3E16").replace(",", "");
 
   private String messageSetText =
     "[protobuf_unittest.TestMessageSetExtension1] {\n" +
@@ -119,6 +119,7 @@ public class TextFormatTest extends TestCase {
       "  i: 456\n" +
       "}\n";
 
+
   private final TextFormat.Parser parserWithOverwriteForbidden =
       TextFormat.Parser.newBuilder()
           .setSingularOverwritePolicy(
@@ -460,6 +461,7 @@ public class TextFormatTest extends TestCase {
     }
   }
 
+
   private void assertParseErrorWithOverwriteForbidden(String error,
       String text) {
     TestAllTypes.Builder builder = TestAllTypes.newBuilder();
@@ -553,10 +555,10 @@ public class TextFormatTest extends TestCase {
 
   public void testEscape() throws Exception {
     // Escape sequences.
-    assertEquals("\\000\\001\\a\\b\\f\\n\\r\\t\\v\\\\\\'\\\"",
-      TextFormat.escapeBytes(bytes("\0\001\007\b\f\n\r\t\013\\\'\"")));
-    assertEquals("\\000\\001\\a\\b\\f\\n\\r\\t\\v\\\\\\'\\\"",
-      TextFormat.escapeText("\0\001\007\b\f\n\r\t\013\\\'\""));
+    assertEquals("\\000\\001\\a\\b\\f\\n\\r\\t\\v\\\\\\'\\\"\\177",
+      TextFormat.escapeBytes(bytes("\0\001\007\b\f\n\r\t\013\\\'\"\177")));
+    assertEquals("\\000\\001\\a\\b\\f\\n\\r\\t\\v\\\\\\'\\\"\\177",
+      TextFormat.escapeText("\0\001\007\b\f\n\r\t\013\\\'\"\177"));
     assertEquals(bytes("\0\001\007\b\f\n\r\t\013\\\'\""),
       TextFormat.unescapeBytes("\\000\\001\\a\\b\\f\\n\\r\\t\\v\\\\\\'\\\""));
     assertEquals("\0\001\007\b\f\n\r\t\013\\\'\"",
@@ -900,6 +902,7 @@ public class TextFormatTest extends TestCase {
             .build()));
   }
 
+
   public void testParseNonRepeatedFields() throws Exception {
     assertParseSuccessWithOverwriteForbidden(
         "repeated_int32: 1\n" +

java/src/test/java/com/google/protobuf/UnknownEnumValueTest.java

@@ -0,0 +1,255 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package com.google.protobuf;
+
+import com.google.protobuf.Descriptors.Descriptor;
+import com.google.protobuf.Descriptors.EnumDescriptor;
+import com.google.protobuf.Descriptors.EnumValueDescriptor;
+import com.google.protobuf.Descriptors.FieldDescriptor;
+import com.google.protobuf.FieldPresenceTestProto.TestAllTypes;
+import com.google.protobuf.TextFormat.ParseException;
+
+import junit.framework.TestCase;
+
+/**
+ * Unit tests for protos that keep unknown enum values rather than discard
+ * them as unknown fields.
+ */
+public class UnknownEnumValueTest extends TestCase {
+  public void testUnknownEnumValues() throws Exception {
+    TestAllTypes.Builder builder = TestAllTypes.newBuilder();
+    builder.setOptionalNestedEnumValue(4321);
+    builder.addRepeatedNestedEnumValue(5432);
+    builder.addPackedNestedEnumValue(6543);
+    TestAllTypes message = builder.build();
+    assertEquals(4321, message.getOptionalNestedEnumValue());
+    assertEquals(5432, message.getRepeatedNestedEnumValue(0));
+    assertEquals(5432, message.getRepeatedNestedEnumValueList().get(0).intValue());
+    assertEquals(6543, message.getPackedNestedEnumValue(0));
+    // Returns UNRECOGNIZED if an enum type is requested.
+    assertEquals(TestAllTypes.NestedEnum.UNRECOGNIZED, message.getOptionalNestedEnum());
+    assertEquals(TestAllTypes.NestedEnum.UNRECOGNIZED, message.getRepeatedNestedEnum(0));
+    assertEquals(TestAllTypes.NestedEnum.UNRECOGNIZED, message.getRepeatedNestedEnumList().get(0));
+    assertEquals(TestAllTypes.NestedEnum.UNRECOGNIZED, message.getPackedNestedEnum(0));
+    
+    // Test serialization and parsing.
+    ByteString data = message.toByteString();
+    message = TestAllTypes.parseFrom(data);
+    assertEquals(4321, message.getOptionalNestedEnumValue());
+    assertEquals(5432, message.getRepeatedNestedEnumValue(0));
+    assertEquals(5432, message.getRepeatedNestedEnumValueList().get(0).intValue());
+    assertEquals(6543, message.getPackedNestedEnumValue(0));
+    // Returns UNRECOGNIZED if an enum type is requested.
+    assertEquals(TestAllTypes.NestedEnum.UNRECOGNIZED, message.getOptionalNestedEnum());
+    assertEquals(TestAllTypes.NestedEnum.UNRECOGNIZED, message.getRepeatedNestedEnum(0));
+    assertEquals(TestAllTypes.NestedEnum.UNRECOGNIZED, message.getRepeatedNestedEnumList().get(0));
+    assertEquals(TestAllTypes.NestedEnum.UNRECOGNIZED, message.getPackedNestedEnum(0));
+    
+    // Test toBuilder().
+    builder = message.toBuilder();
+    assertEquals(4321, builder.getOptionalNestedEnumValue());
+    assertEquals(5432, builder.getRepeatedNestedEnumValue(0));
+    assertEquals(5432, builder.getRepeatedNestedEnumValueList().get(0).intValue());
+    assertEquals(6543, builder.getPackedNestedEnumValue(0));    
+    // Returns UNRECOGNIZED if an enum type is requested.
+    assertEquals(TestAllTypes.NestedEnum.UNRECOGNIZED, builder.getOptionalNestedEnum());
+    assertEquals(TestAllTypes.NestedEnum.UNRECOGNIZED, builder.getRepeatedNestedEnum(0));
+    assertEquals(TestAllTypes.NestedEnum.UNRECOGNIZED, builder.getRepeatedNestedEnumList().get(0));
+    assertEquals(TestAllTypes.NestedEnum.UNRECOGNIZED, builder.getPackedNestedEnum(0));
+    
+    // Test mergeFrom().
+    builder = TestAllTypes.newBuilder().mergeFrom(message);
+    assertEquals(4321, builder.getOptionalNestedEnumValue());
+    assertEquals(5432, builder.getRepeatedNestedEnumValue(0));
+    assertEquals(5432, builder.getRepeatedNestedEnumValueList().get(0).intValue());
+    assertEquals(6543, builder.getPackedNestedEnumValue(0));
+    // Returns UNRECOGNIZED if an enum type is requested.
+    assertEquals(TestAllTypes.NestedEnum.UNRECOGNIZED, builder.getOptionalNestedEnum());
+    assertEquals(TestAllTypes.NestedEnum.UNRECOGNIZED, builder.getRepeatedNestedEnum(0));
+    assertEquals(TestAllTypes.NestedEnum.UNRECOGNIZED, builder.getRepeatedNestedEnumList().get(0));
+    assertEquals(TestAllTypes.NestedEnum.UNRECOGNIZED, builder.getPackedNestedEnum(0));
+    
+    // Test equals() and hashCode()
+    TestAllTypes sameMessage = builder.build();
+    assertEquals(message, sameMessage);
+    assertEquals(message.hashCode(), sameMessage.hashCode());
+
+    // Getting the numeric value of UNRECOGNIZED will throw an exception.
+    try {
+      TestAllTypes.NestedEnum.UNRECOGNIZED.getNumber();
+      fail("Exception is expected.");
+    } catch (IllegalArgumentException e) {
+      // Expected.
+    }
+
+    // Setting an enum field to an UNRECOGNIZED value will throw an exception.
+    try {
+      builder.setOptionalNestedEnum(builder.getOptionalNestedEnum());
+      fail("Exception is expected.");
+    } catch (IllegalArgumentException e) {
+      // Expected.
+    }
+    try {
+      builder.addRepeatedNestedEnum(builder.getOptionalNestedEnum());
+      fail("Exception is expected.");
+    } catch (IllegalArgumentException e) {
+      // Expected.
+    }
+  }
+  
+  public void testUnknownEnumValueInReflectionApi() throws Exception {
+    Descriptor descriptor = TestAllTypes.getDescriptor();
+    FieldDescriptor optionalNestedEnumField = descriptor.findFieldByName("optional_nested_enum");
+    FieldDescriptor repeatedNestedEnumField = descriptor.findFieldByName("repeated_nested_enum");
+    FieldDescriptor packedNestedEnumField = descriptor.findFieldByName("packed_nested_enum");
+    EnumDescriptor enumType = TestAllTypes.NestedEnum.getDescriptor();
+
+    TestAllTypes.Builder builder = TestAllTypes.newBuilder();
+    builder.setField(optionalNestedEnumField,
+        enumType.findValueByNumberCreatingIfUnknown(4321));
+    builder.addRepeatedField(repeatedNestedEnumField,
+        enumType.findValueByNumberCreatingIfUnknown(5432));
+    builder.addRepeatedField(packedNestedEnumField,
+        enumType.findValueByNumberCreatingIfUnknown(6543));
+    TestAllTypes message = builder.build();
+    
+    // Getters will return unknown enum values as EnumValueDescriptor.
+    EnumValueDescriptor unknown4321 =
+        (EnumValueDescriptor) message.getField(optionalNestedEnumField);
+    EnumValueDescriptor unknown5432 =
+        (EnumValueDescriptor) message.getRepeatedField(repeatedNestedEnumField, 0);
+    EnumValueDescriptor unknown6543 =
+        (EnumValueDescriptor) message.getRepeatedField(packedNestedEnumField, 0);
+    assertEquals(4321, unknown4321.getNumber());
+    assertEquals(5432, unknown5432.getNumber());
+    assertEquals(6543, unknown6543.getNumber());
+    
+    // Unknown EnumValueDescriptor will map to UNRECOGNIZED.
+    assertEquals(
+        TestAllTypes.NestedEnum.valueOf(unknown4321),
+        TestAllTypes.NestedEnum.UNRECOGNIZED);
+    assertEquals(
+        TestAllTypes.NestedEnum.valueOf(unknown5432),
+        TestAllTypes.NestedEnum.UNRECOGNIZED);
+    assertEquals(
+        TestAllTypes.NestedEnum.valueOf(unknown6543),
+        TestAllTypes.NestedEnum.UNRECOGNIZED);
+    
+    // Setters also accept unknown EnumValueDescriptor.
+    builder.setField(optionalNestedEnumField, unknown6543);
+    builder.setRepeatedField(repeatedNestedEnumField, 0, unknown4321);
+    builder.setRepeatedField(packedNestedEnumField, 0, unknown5432);
+    message = builder.build();
+    // Like other descriptors, unknown EnumValueDescriptor can be compared by
+    // object identity.
+    assertTrue(unknown6543 == message.getField(optionalNestedEnumField));
+    assertTrue(unknown4321 == message.getRepeatedField(repeatedNestedEnumField, 0));
+    assertTrue(unknown5432 == message.getRepeatedField(packedNestedEnumField, 0));
+  }
+  
+  public void testUnknownEnumValueWithDynamicMessage() throws Exception {
+    Descriptor descriptor = TestAllTypes.getDescriptor();
+    FieldDescriptor optionalNestedEnumField = descriptor.findFieldByName("optional_nested_enum");
+    FieldDescriptor repeatedNestedEnumField = descriptor.findFieldByName("repeated_nested_enum");
+    FieldDescriptor packedNestedEnumField = descriptor.findFieldByName("packed_nested_enum");
+    EnumDescriptor enumType = TestAllTypes.NestedEnum.getDescriptor();
+    
+    Message dynamicMessageDefaultInstance = DynamicMessage.getDefaultInstance(descriptor);
+
+    Message.Builder builder = dynamicMessageDefaultInstance.newBuilderForType();
+    builder.setField(optionalNestedEnumField,
+        enumType.findValueByNumberCreatingIfUnknown(4321));
+    builder.addRepeatedField(repeatedNestedEnumField,
+        enumType.findValueByNumberCreatingIfUnknown(5432));
+    builder.addRepeatedField(packedNestedEnumField,
+        enumType.findValueByNumberCreatingIfUnknown(6543));
+    Message message = builder.build();
+    assertEquals(4321,
+        ((EnumValueDescriptor) message.getField(optionalNestedEnumField)).getNumber());
+    assertEquals(5432,
+        ((EnumValueDescriptor) message.getRepeatedField(repeatedNestedEnumField, 0)).getNumber());
+    assertEquals(6543,
+        ((EnumValueDescriptor) message.getRepeatedField(packedNestedEnumField, 0)).getNumber());
+    
+    // Test reflection based serialization/parsing implementation.
+    ByteString data = message.toByteString();
+    message = dynamicMessageDefaultInstance
+        .newBuilderForType()
+        .mergeFrom(data)
+        .build();
+    assertEquals(4321,
+        ((EnumValueDescriptor) message.getField(optionalNestedEnumField)).getNumber());
+    assertEquals(5432,
+        ((EnumValueDescriptor) message.getRepeatedField(repeatedNestedEnumField, 0)).getNumber());
+    assertEquals(6543,
+        ((EnumValueDescriptor) message.getRepeatedField(packedNestedEnumField, 0)).getNumber());
+    
+    // Test reflection based equals()/hashCode().
+    builder = dynamicMessageDefaultInstance.newBuilderForType();
+    builder.setField(optionalNestedEnumField,
+        enumType.findValueByNumberCreatingIfUnknown(4321));
+    builder.addRepeatedField(repeatedNestedEnumField,
+        enumType.findValueByNumberCreatingIfUnknown(5432));
+    builder.addRepeatedField(packedNestedEnumField,
+        enumType.findValueByNumberCreatingIfUnknown(6543));
+    Message sameMessage = builder.build();
+    assertEquals(message, sameMessage);
+    assertEquals(message.hashCode(), sameMessage.hashCode());
+    builder.setField(optionalNestedEnumField,
+        enumType.findValueByNumberCreatingIfUnknown(0));
+    Message differentMessage = builder.build();
+    assertFalse(message.equals(differentMessage));
+  }
+  
+  public void testUnknownEnumValuesInTextFormat() {
+    TestAllTypes.Builder builder = TestAllTypes.newBuilder();
+    builder.setOptionalNestedEnumValue(4321);
+    builder.addRepeatedNestedEnumValue(5432);
+    builder.addPackedNestedEnumValue(6543);
+    TestAllTypes message = builder.build();
+    
+    // We can print a message with unknown enum values.
+    String textData = TextFormat.printToString(message);
+    assertEquals(
+        "optional_nested_enum: UNKNOWN_ENUM_VALUE_NestedEnum_4321\n"
+        + "repeated_nested_enum: UNKNOWN_ENUM_VALUE_NestedEnum_5432\n"
+        + "packed_nested_enum: UNKNOWN_ENUM_VALUE_NestedEnum_6543\n", textData);
+    
+    // Parsing unknown enum values will fail just like parsing other kinds of
+    // unknown fields.
+    try {
+      TextFormat.merge(textData, builder);
+      fail();
+    } catch (ParseException e) {
+      // expected.
+    }
+  }
+}

java/src/test/java/com/google/protobuf/UnknownFieldSetLiteTest.java

@@ -0,0 +1,317 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package com.google.protobuf;
+
+import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash;
+import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.Bar;
+import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.Foo;
+
+import junit.framework.TestCase;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+/**
+ * Tests for {@link UnknownFieldSetLite}.
+ *
+ * @author dweis@google.com (Daniel Weis)
+ */
+public class UnknownFieldSetLiteTest extends TestCase {
+
+  public void testNoDataIsDefaultInstance() {
+    assertSame(
+        UnknownFieldSetLite.getDefaultInstance(),
+        UnknownFieldSetLite.newBuilder()
+            .build());
+  }
+
+  public void testDefaultInstance() {
+    UnknownFieldSetLite unknownFields = UnknownFieldSetLite.getDefaultInstance();
+
+    assertEquals(0, unknownFields.getSerializedSize());
+    assertEquals(ByteString.EMPTY, toByteString(unknownFields));
+  }
+
+  public void testMergeFieldFrom() throws IOException {
+    Foo foo = Foo.newBuilder()
+      .setValue(2)
+      .build();
+
+    CodedInputStream input = CodedInputStream.newInstance(foo.toByteArray());
+
+    UnknownFieldSetLite.Builder builder = UnknownFieldSetLite.newBuilder();
+    builder.mergeFieldFrom(input.readTag(), input);
+
+    assertEquals(foo.toByteString(), toByteString(builder.build()));
+  }
+
+  public void testSerializedSize() throws IOException {
+    Foo foo = Foo.newBuilder()
+      .setValue(2)
+      .build();
+
+    CodedInputStream input = CodedInputStream.newInstance(foo.toByteArray());
+
+    UnknownFieldSetLite.Builder builder = UnknownFieldSetLite.newBuilder();
+    builder.mergeFieldFrom(input.readTag(), input);
+
+    assertEquals(foo.toByteString().size(), builder.build().getSerializedSize());
+  }
+
+  public void testMergeVarintField() throws IOException {
+    UnknownFieldSetLite.Builder builder = UnknownFieldSetLite.newBuilder();
+    builder.mergeVarintField(10, 2);
+
+    CodedInputStream input =
+        CodedInputStream.newInstance(toByteString(builder.build()).toByteArray());
+
+    int tag = input.readTag();
+    assertEquals(10, WireFormat.getTagFieldNumber(tag));
+    assertEquals(WireFormat.WIRETYPE_VARINT, WireFormat.getTagWireType(tag));
+    assertEquals(2, input.readUInt64());
+    assertTrue(input.isAtEnd());
+  }
+
+  public void testMergeVarintField_negative() throws IOException {
+    UnknownFieldSetLite.Builder builder = UnknownFieldSetLite.newBuilder();
+    builder.mergeVarintField(10, -6);
+
+    CodedInputStream input =
+        CodedInputStream.newInstance(toByteString(builder.build()).toByteArray());
+
+    int tag = input.readTag();
+    assertEquals(10, WireFormat.getTagFieldNumber(tag));
+    assertEquals(WireFormat.WIRETYPE_VARINT, WireFormat.getTagWireType(tag));
+    assertEquals(-6, input.readUInt64());
+    assertTrue(input.isAtEnd());
+  }
+
+  public void testEqualsAndHashCode() {
+    UnknownFieldSetLite.Builder builder1 = UnknownFieldSetLite.newBuilder();
+    builder1.mergeVarintField(10, 2);
+    UnknownFieldSetLite unknownFields1 = builder1.build();
+
+    UnknownFieldSetLite.Builder builder2 = UnknownFieldSetLite.newBuilder();
+    builder2.mergeVarintField(10, 2);
+    UnknownFieldSetLite unknownFields2 = builder2.build();
+
+    assertEquals(unknownFields1, unknownFields2);
+    assertEquals(unknownFields1.hashCode(), unknownFields2.hashCode());
+    assertFalse(unknownFields1.equals(UnknownFieldSetLite.getDefaultInstance()));
+    assertFalse(unknownFields1.hashCode() == UnknownFieldSetLite.getDefaultInstance().hashCode());
+  }
+
+  public void testConcat() throws IOException {
+    UnknownFieldSetLite.Builder builder = UnknownFieldSetLite.newBuilder();
+    builder.mergeVarintField(10, 2);
+    UnknownFieldSetLite unknownFields = builder.build();
+
+    unknownFields = UnknownFieldSetLite.concat(unknownFields, unknownFields);
+
+    CodedInputStream input =
+        CodedInputStream.newInstance(toByteString(unknownFields).toByteArray());
+
+    int tag = input.readTag();
+    assertEquals(10, WireFormat.getTagFieldNumber(tag));
+    assertEquals(WireFormat.WIRETYPE_VARINT, WireFormat.getTagWireType(tag));
+    assertEquals(2, input.readUInt64());
+    assertFalse(input.isAtEnd());
+    input.readTag();
+    assertEquals(10, WireFormat.getTagFieldNumber(tag));
+    assertEquals(WireFormat.WIRETYPE_VARINT, WireFormat.getTagWireType(tag));
+    assertEquals(2, input.readUInt64());
+    assertTrue(input.isAtEnd());
+  }
+
+  public void testConcat_empty() {
+    UnknownFieldSetLite unknownFields = UnknownFieldSetLite.concat(
+        UnknownFieldSetLite.getDefaultInstance(), UnknownFieldSetLite.getDefaultInstance());
+
+    assertEquals(0, unknownFields.getSerializedSize());
+    assertEquals(ByteString.EMPTY, toByteString(unknownFields));
+  }
+
+  public void testBuilderReuse() throws IOException {
+    UnknownFieldSetLite.Builder builder = UnknownFieldSetLite.newBuilder();
+    builder.mergeVarintField(10, 2);
+    builder.build();
+
+    try {
+      builder.build();
+      fail();
+    } catch (IllegalStateException e) {
+      // Expected.
+    }
+
+    try {
+      builder.mergeFieldFrom(0, CodedInputStream.newInstance(new byte[0]));
+      fail();
+    } catch (IllegalStateException e) {
+      // Expected.
+    }
+
+    try {
+      builder.mergeVarintField(5, 1);
+      fail();
+    } catch (IllegalStateException e) {
+      // Expected.
+    }
+  }
+
+  public void testBuilderReuse_empty() {
+    UnknownFieldSetLite.Builder builder = UnknownFieldSetLite.newBuilder();
+    builder.build();
+
+    try {
+      builder.build();
+      fail();
+    } catch (IllegalStateException e) {
+      // Expected.
+    }
+  }
+
+  public void testRoundTrips() throws InvalidProtocolBufferException {
+    Foo foo = Foo.newBuilder()
+        .setValue(1)
+        .setExtension(Bar.fooExt, Bar.newBuilder()
+            .setName("name")
+            .build())
+        .setExtension(LiteEqualsAndHash.varint, 22)
+        .setExtension(LiteEqualsAndHash.fixed32, 44)
+        .setExtension(LiteEqualsAndHash.fixed64, 66L)
+        .setExtension(LiteEqualsAndHash.myGroup, LiteEqualsAndHash.MyGroup.newBuilder()
+            .setGroupValue("value")
+            .build())
+        .build();
+
+    Foo copy = Foo.parseFrom(foo.toByteArray());
+
+    assertEquals(foo.getSerializedSize(), copy.getSerializedSize());
+    assertFalse(foo.equals(copy));
+
+    Foo secondCopy = Foo.parseFrom(foo.toByteArray());
+    assertEquals(copy, secondCopy);
+
+    ExtensionRegistryLite extensionRegistry = ExtensionRegistryLite.newInstance();
+    LiteEqualsAndHash.registerAllExtensions(extensionRegistry);
+    Foo copyOfCopy = Foo.parseFrom(copy.toByteArray(), extensionRegistry);
+
+    assertEquals(foo, copyOfCopy);
+  }
+
+  public void testMalformedBytes() {
+    try {
+      Foo.parseFrom("this is a malformed protocol buffer".getBytes(StandardCharsets.UTF_8));
+      fail();
+    } catch (InvalidProtocolBufferException e) {
+      // Expected.
+    }
+  }
+
+  public void testMissingStartGroupTag() throws IOException {
+    ByteString.Output byteStringOutput = ByteString.newOutput();
+    CodedOutputStream output = CodedOutputStream.newInstance(byteStringOutput);
+    output.writeGroupNoTag(Foo.newBuilder().setValue(11).build());
+    output.writeTag(100, WireFormat.WIRETYPE_END_GROUP);
+    output.flush();
+
+    try {
+      Foo.parseFrom(byteStringOutput.toByteString());
+      fail();
+    } catch (InvalidProtocolBufferException e) {
+      // Expected.
+    }
+  }
+
+  public void testMissingEndGroupTag() throws IOException {
+    ByteString.Output byteStringOutput = ByteString.newOutput();
+    CodedOutputStream output = CodedOutputStream.newInstance(byteStringOutput);
+    output.writeTag(100, WireFormat.WIRETYPE_START_GROUP);
+    output.writeGroupNoTag(Foo.newBuilder().setValue(11).build());
+    output.flush();
+
+    try {
+      Foo.parseFrom(byteStringOutput.toByteString());
+      fail();
+    } catch (InvalidProtocolBufferException e) {
+      // Expected.
+    }
+  }
+
+  public void testMismatchingGroupTags() throws IOException {
+    ByteString.Output byteStringOutput = ByteString.newOutput();
+    CodedOutputStream output = CodedOutputStream.newInstance(byteStringOutput);
+    output.writeTag(100, WireFormat.WIRETYPE_START_GROUP);
+    output.writeGroupNoTag(Foo.newBuilder().setValue(11).build());
+    output.writeTag(101, WireFormat.WIRETYPE_END_GROUP);
+    output.flush();
+
+    try {
+      Foo.parseFrom(byteStringOutput.toByteString());
+      fail();
+    } catch (InvalidProtocolBufferException e) {
+      // Expected.
+    }
+  }
+
+  public void testTruncatedInput() {
+    Foo foo = Foo.newBuilder()
+        .setValue(1)
+        .setExtension(Bar.fooExt, Bar.newBuilder()
+            .setName("name")
+            .build())
+        .setExtension(LiteEqualsAndHash.varint, 22)
+        .setExtension(LiteEqualsAndHash.myGroup, LiteEqualsAndHash.MyGroup.newBuilder()
+            .setGroupValue("value")
+            .build())
+        .build();
+
+    try {
+      Foo.parseFrom(foo.toByteString().substring(0, foo.toByteString().size() - 10));
+      fail();
+    } catch (InvalidProtocolBufferException e) {
+      // Expected.
+    }
+  }
+
+  private ByteString toByteString(UnknownFieldSetLite unknownFields) {
+    ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+    CodedOutputStream output = CodedOutputStream.newInstance(byteArrayOutputStream);
+    try {
+      unknownFields.writeTo(output);
+      output.flush();
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+    return ByteString.copyFrom(byteArrayOutputStream.toByteArray());
+  }
+}

java/src/test/java/com/google/protobuf/field_presence_test.proto

@@ -0,0 +1,93 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+syntax = "proto3";
+
+package field_presence_test;
+
+import "google/protobuf/unittest.proto";
+
+option java_package = "com.google.protobuf";
+option java_outer_classname = "FieldPresenceTestProto";
+option java_generate_equals_and_hash = true;
+
+message TestAllTypes {
+  enum NestedEnum {
+    FOO = 0;
+    BAR = 1;
+    BAZ = 2;
+  }
+  message NestedMessage {
+    optional int32 value = 1;
+  }
+
+  optional int32 optional_int32 = 1;
+  optional string optional_string = 2;
+  optional bytes optional_bytes = 3;
+  optional NestedEnum optional_nested_enum = 4;
+  optional NestedMessage optional_nested_message = 5;
+  optional protobuf_unittest.TestRequired optional_proto2_message = 6;
+
+  oneof oneof_field {
+    int32 oneof_int32 = 11;
+    uint32 oneof_uint32 = 12;
+    string oneof_string = 13;
+    bytes oneof_bytes = 14;
+    NestedEnum oneof_nested_enum = 15;
+    NestedMessage oneof_nested_message = 16;
+    protobuf_unittest.TestRequired oneof_proto2_message = 17;
+  }
+
+  repeated int32 repeated_int32 = 21;
+  repeated string repeated_string = 22;
+  repeated bytes repeated_bytes = 23;
+  repeated NestedEnum repeated_nested_enum = 24;
+  repeated NestedMessage repeated_nested_message = 25;
+  repeated protobuf_unittest.TestRequired repeated_proto2_message = 26;
+  repeated NestedEnum packed_nested_enum = 27 [packed = true];
+}
+
+message TestOptionalFieldsOnly {
+  optional int32 optional_int32 = 1;
+  optional string optional_string = 2;
+  optional bytes optional_bytes = 3;
+  optional TestAllTypes.NestedEnum optional_nested_enum = 4;
+  optional TestAllTypes.NestedMessage optional_nested_message = 5;
+  optional protobuf_unittest.TestRequired optional_proto2_message = 6;
+}
+
+message TestRepeatedFieldsOnly {
+  repeated int32 repeated_int32 = 21;
+  repeated string repeated_string = 22;
+  repeated bytes repeated_bytes = 23;
+  repeated TestAllTypes.NestedEnum repeated_nested_enum = 24;
+  repeated TestAllTypes.NestedMessage repeated_nested_message = 25;
+  repeated protobuf_unittest.TestRequired repeated_proto2_message = 26;
+}

java/src/test/java/com/google/protobuf/lazy_fields_lite.proto

@@ -32,6 +32,7 @@
 //
 // A proto file with lazy fields
 
+syntax = "proto2";
 
 package protobuf_unittest;
 

java/src/test/java/com/google/protobuf/lite_equals_and_hash.proto

@@ -30,6 +30,7 @@
 
 // Author: pbogle@google.com (Phil Bogle)
 
+syntax = "proto2";
 
 package protobuf_unittest.lite_equals_and_hash;
 
@@ -41,9 +42,15 @@ option optimize_for = LITE_RUNTIME;
 message Foo {
   optional int32 value = 1;
   repeated Bar bar = 2;
+
+  extensions 100 to max;
 }
 
 message Bar {
+  extend Foo {
+    optional Bar foo_ext = 100;
+  }
+
   optional string name = 1;
 }
 
@@ -53,3 +60,13 @@ message BarPrime {
 
 message Empty {
 }
+
+extend Foo {
+  optional int32 varint = 101;
+  optional fixed32 fixed32 = 102;
+  optional fixed64 fixed64 = 103;
+  optional group MyGroup = 104 {
+    optional string group_value = 1;
+  }
+}
+

java/src/test/java/com/google/protobuf/map_for_proto2_lite_test.proto

@@ -0,0 +1,63 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+syntax = "proto2";
+
+
+option java_outer_classname = "MapForProto2TestProto";
+option java_generate_equals_and_hash = true;
+
+message TestMap {
+  message MessageValue {
+    optional int32 value = 1;
+  }
+  enum EnumValue {
+    FOO = 0;
+    BAR = 1;
+    BAZ = 2;
+    QUX = 3;
+  }
+
+  map<int32, int32>         int32_to_int32_field = 1;
+  map<int32, string>        int32_to_string_field = 2;
+  map<int32, bytes>         int32_to_bytes_field = 3;
+  map<int32, EnumValue>     int32_to_enum_field = 4;
+  map<int32, MessageValue>  int32_to_message_field = 5;
+  map<string, int32>        string_to_int32_field = 6;
+}
+
+message TestUnknownEnumValue {
+  // Wire-compatible with TestMap.int32_to_enum_field so we can test the
+  // parsing behavior of TestMap regarding unknown enum values.
+  map<int32, int32> int32_to_int32_field = 4;
+}
+package map_for_proto2_lite_test;
+option java_package = "map_lite_test";
+option optimize_for = LITE_RUNTIME;

java/src/test/java/com/google/protobuf/map_for_proto2_test.proto

@@ -0,0 +1,62 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+syntax = "proto2";
+
+package map_for_proto2_test;
+
+option java_package = "map_test";
+option java_outer_classname = "MapForProto2TestProto";
+option java_generate_equals_and_hash = true;
+
+message TestMap {
+  message MessageValue {
+    optional int32 value = 1;
+  }
+  enum EnumValue {
+    FOO = 0;
+    BAR = 1;
+    BAZ = 2;
+    QUX = 3;
+  }
+
+  map<int32, int32>         int32_to_int32_field = 1;
+  map<int32, string>        int32_to_string_field = 2;
+  map<int32, bytes>         int32_to_bytes_field = 3;
+  map<int32, EnumValue>     int32_to_enum_field = 4;
+  map<int32, MessageValue>  int32_to_message_field = 5;
+  map<string, int32>        string_to_int32_field = 6;
+}
+
+message TestUnknownEnumValue {
+  // Wire-compatible with TestMap.int32_to_enum_field so we can test the
+  // parsing behavior of TestMap regarding unknown enum values.
+  map<int32, int32> int32_to_int32_field = 4;
+}

java/src/test/java/com/google/protobuf/map_test.proto

@@ -0,0 +1,63 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+syntax = "proto3";
+
+package map_test;
+
+option java_package = "map_test";
+option java_outer_classname = "MapTestProto";
+option java_generate_equals_and_hash = true;
+
+message TestMap {
+  message MessageValue {
+    optional int32 value = 1;
+  }
+  enum EnumValue {
+    FOO = 0;
+    BAR = 1;
+    BAZ = 2;
+    QUX = 3;
+  }
+
+  map<int32, int32>        int32_to_int32_field = 1;
+  map<int32, string>       int32_to_string_field = 2;
+  map<int32, bytes>        int32_to_bytes_field = 3;
+  map<int32, EnumValue>    int32_to_enum_field = 4;
+  map<int32, MessageValue> int32_to_message_field = 5;
+  map<string, int32>       string_to_int32_field = 6;
+}
+
+// Used to test that a nested bulider containing map fields will properly
+// propagate the onChange event and mark its parent dirty when a change
+// is made to a map field.
+message TestOnChangeEventPropagation {
+  optional TestMap optional_message = 1;
+}

java/src/test/java/com/google/protobuf/multiple_files_test.proto

@@ -32,6 +32,7 @@
 //
 // A proto file which tests the java_multiple_files option.
 
+syntax = "proto2";
 
 // Some generic_services option(s) added automatically.
 // See:  http://go/proto2-generic-services-default

java/src/test/java/com/google/protobuf/nested_builders_test.proto

@@ -30,6 +30,7 @@
 
 // Author: jonp@google.com (Jon Perlow)
 //
+syntax = "proto2";
 
 package protobuf_unittest;
 

java/src/test/java/com/google/protobuf/nested_extension.proto

@@ -33,6 +33,7 @@
 // A proto file with nested extensions. Note that this must be defined in
 // a separate file to properly test the initialization of the outer class.
 
+syntax = "proto2";
 
 import "com/google/protobuf/non_nested_extension.proto";
 

java/src/test/java/com/google/protobuf/nested_extension_lite.proto

@@ -34,6 +34,7 @@
 // this must be defined in a separate file to properly test the initialization
 // of the outer class.
 
+syntax = "proto2";
 
 package protobuf_unittest;
 

java/src/test/java/com/google/protobuf/non_nested_extension.proto

@@ -32,6 +32,7 @@
 //
 // A proto file with extensions.
 
+syntax = "proto2";
 
 package protobuf_unittest;
 

java/src/test/java/com/google/protobuf/non_nested_extension_lite.proto

@@ -32,6 +32,7 @@
 //
 // A proto file with extensions for a MessageLite messages.
 
+syntax = "proto2";
 
 package protobuf_unittest;
 

java/src/test/java/com/google/protobuf/outer_class_name_test.proto

@@ -28,6 +28,8 @@
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+syntax = "proto2";
+
 package protobuf_unittest;
 
 

java/src/test/java/com/google/protobuf/outer_class_name_test2.proto

@@ -28,6 +28,8 @@
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+syntax = "proto2";
+
 package protobuf_unittest;
 
 

java/src/test/java/com/google/protobuf/outer_class_name_test3.proto

@@ -28,6 +28,8 @@
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+syntax = "proto2";
+
 package protobuf_unittest;
 
 

java/src/test/java/com/google/protobuf/test_bad_identifiers.proto

@@ -33,6 +33,7 @@
 // This file tests that various identifiers work as field and type names even
 // though the same identifiers are used internally by the java code generator.
 
+syntax = "proto2";
 
 // Some generic_services option(s) added automatically.
 // See:  http://go/proto2-generic-services-default
@@ -59,12 +60,14 @@ message Descriptor {
   }
   optional NestedDescriptor nested_descriptor = 2;
   enum NestedEnum {
+    UNKNOWN = 0;
     FOO = 1;
   }
 }
 
 message Parser {
   enum ParserEnum {
+    UNKNOWN = 0;
     PARSER = 1;
   }
   optional ParserEnum parser = 1;
@@ -72,6 +75,7 @@ message Parser {
 
 message Deprecated {
   enum TestEnum {
+    UNKNOWN = 0;
     FOO = 1;
 
     // Test if @Deprecated annotation conflicts with Deprecated message name.
@@ -118,6 +122,7 @@ service TestConflictingMethodNames {
 
 message TestConflictingFieldNames {
   enum TestEnum {
+    UNKNOWN = 0;
     FOO = 1;
   }
   message TestMessage {
@@ -142,16 +147,23 @@ message TestConflictingFieldNames {
 
   // This field conflicts with "int32_field" as they both generate
   // the method getInt32FieldList().
-  required int32 int32_field_list = 31;
+  required int32 int32_field_list = 31;  // NO_PROTO3
 
-  extensions 1000 to max;
+  extensions 1000 to max;  // NO_PROTO3
 
   repeated int64 int64_field = 41;
-  extend TestConflictingFieldNames {
+  extend TestConflictingFieldNames {  // NO_PROTO3
     // We don't generate accessors for extensions so the following extension
     // fields don't conflict with the repeated field "int64_field".
-    optional int64 int64_field_count = 1001;
-    optional int64 int64_field_list = 1002;
-  }
+    optional int64 int64_field_count = 1001;  // NO_PROTO3
+    optional int64 int64_field_list = 1002;  // NO_PROTO3
+  }  // NO_PROTO3
 }
 
+message TestMapField {
+  message MapField {}
+  message Pair {}
+  message Message {}
+
+  map<int32, int32> map_field = 1;
+}

java/src/test/java/com/google/protobuf/test_check_utf8.proto

@@ -31,6 +31,7 @@
 // Author: Jacob Butcher (jbaum@google.com)
 //
 // Test file option java_string_check_utf8.
+syntax = "proto2";
 
 package proto2_test_check_utf8;
 

java/src/test/java/com/google/protobuf/test_check_utf8_size.proto

@@ -31,6 +31,7 @@
 // Author: Jacob Butcher (jbaum@google.com)
 //
 // Test file option java_string_check_utf8.
+syntax = "proto2";
 
 package proto2_test_check_utf8_size;
 

java/src/test/java/com/google/protobuf/test_custom_options.proto

@@ -32,6 +32,7 @@
 //
 // Test that custom options defined in a proto file's dependencies are properly
 // initialized.
+syntax = "proto2";
 
 package protobuf_unittest;
 

java/src/test/java/com/google/protobuf/test_extra_interfaces.proto

@@ -29,6 +29,7 @@
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 // Author: Darick Tong (darick@google.com)
+syntax = "proto2";
 
 package protobuf_unittest;
 

python/google/protobuf/descriptor.py

@@ -841,9 +841,10 @@ def MakeDescriptor(desc_proto, package='', build_file_if_cpp=True):
         field_proto.number, field_proto.type,
         FieldDescriptor.ProtoTypeToCppProtoType(field_proto.type),
         field_proto.label, None, nested_desc, enum_desc, None, False, None,
-        has_default_value=False)
+        options=field_proto.options, has_default_value=False)
     fields.append(field)
 
   desc_name = '.'.join(full_message_name)
   return Descriptor(desc_proto.name, desc_name, None, None, fields,
-                    nested_types.values(), enum_types.values(), [])
+                    nested_types.values(), enum_types.values(), [],
+                    options=desc_proto.options)

python/google/protobuf/descriptor_database.py

@@ -133,5 +133,5 @@ def _ExtractSymbols(desc_proto, package):
   for nested_type in desc_proto.nested_type:
     for symbol in _ExtractSymbols(nested_type, message_name):
       yield symbol
-    for enum_type in desc_proto.enum_type:
-      yield '.'.join((message_name, enum_type.name))
+  for enum_type in desc_proto.enum_type:
+    yield '.'.join((message_name, enum_type.name))

python/google/protobuf/descriptor_pool.py

@@ -554,7 +554,7 @@ class DescriptorPool(object):
         field_desc.default_value = field_proto.default_value.lower() == 'true'
       elif field_proto.type == descriptor.FieldDescriptor.TYPE_ENUM:
         field_desc.default_value = field_desc.enum_type.values_by_name[
-            field_proto.default_value].index
+            field_proto.default_value].number
       elif field_proto.type == descriptor.FieldDescriptor.TYPE_BYTES:
         field_desc.default_value = text_encoding.CUnescape(
             field_proto.default_value)

python/google/protobuf/internal/descriptor_database_test.py

@@ -58,6 +58,8 @@ class DescriptorDatabaseTest(basetest.TestCase):
         'google.protobuf.python.internal.Factory2Enum'))
     self.assertEquals(file_desc_proto, db.FindFileContainingSymbol(
         'google.protobuf.python.internal.Factory2Message.NestedFactory2Enum'))
+    self.assertEquals(file_desc_proto, db.FindFileContainingSymbol(
+        'google.protobuf.python.internal.MessageWithNestedEnumOnly.NestedEnum'))
 
 if __name__ == '__main__':
   basetest.main()

python/google/protobuf/internal/descriptor_pool_test.py

@@ -48,6 +48,7 @@ from google.protobuf.internal import factory_test2_pb2
 from google.protobuf import descriptor
 from google.protobuf import descriptor_database
 from google.protobuf import descriptor_pool
+from google.protobuf import symbol_database
 
 
 class DescriptorPoolTest(basetest.TestCase):
@@ -237,6 +238,32 @@ class DescriptorPoolTest(basetest.TestCase):
     TEST2_FILE.CheckFile(self, self.pool)
 
 
+  def testEnumDefaultValue(self):
+    """Test the default value of enums which don't start at zero."""
+    def _CheckDefaultValue(file_descriptor):
+      default_value = (file_descriptor
+                       .message_types_by_name['DescriptorPoolTest1']
+                       .fields_by_name['nested_enum']
+                       .default_value)
+      self.assertEqual(default_value,
+                       descriptor_pool_test1_pb2.DescriptorPoolTest1.BETA)
+    # First check what the generated descriptor contains.
+    _CheckDefaultValue(descriptor_pool_test1_pb2.DESCRIPTOR)
+    # Then check the generated pool. Normally this is the same descriptor.
+    file_descriptor = symbol_database.Default().pool.FindFileByName(
+        'google/protobuf/internal/descriptor_pool_test1.proto')
+    self.assertIs(file_descriptor, descriptor_pool_test1_pb2.DESCRIPTOR)
+    _CheckDefaultValue(file_descriptor)
+
+    # Then check the dynamic pool and its internal DescriptorDatabase.
+    descriptor_proto = descriptor_pb2.FileDescriptorProto.FromString(
+        descriptor_pool_test1_pb2.DESCRIPTOR.serialized_pb)
+    self.pool.Add(descriptor_proto)
+    # And do the same check as above
+    file_descriptor = self.pool.FindFileByName(
+        'google/protobuf/internal/descriptor_pool_test1.proto')
+    _CheckDefaultValue(file_descriptor)
+
 
 class ProtoFile(object):
 
@@ -328,7 +355,7 @@ class EnumField(object):
     test.assertEqual(descriptor.FieldDescriptor.CPPTYPE_ENUM,
                      field_desc.cpp_type)
     test.assertTrue(field_desc.has_default_value)
-    test.assertEqual(enum_desc.values_by_name[self.default_value].index,
+    test.assertEqual(enum_desc.values_by_name[self.default_value].number,
                      field_desc.default_value)
     test.assertEqual(msg_desc, field_desc.containing_type)
     test.assertEqual(enum_desc, field_desc.enum_type)

python/google/protobuf/internal/descriptor_pool_test1.proto

@@ -28,6 +28,8 @@
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+syntax = "proto2";
+
 package google.protobuf.python.internal;
 
 

python/google/protobuf/internal/descriptor_pool_test2.proto

@@ -28,6 +28,8 @@
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+syntax = "proto2";
+
 package google.protobuf.python.internal;
 
 import "google/protobuf/internal/descriptor_pool_test1.proto";

python/google/protobuf/internal/descriptor_test.py

@@ -665,5 +665,15 @@ class MakeDescriptorTest(basetest.TestCase):
                      descriptor.FieldDescriptor.CPPTYPE_UINT64)
 
 
+  def testMakeDescriptorWithOptions(self):
+    descriptor_proto = descriptor_pb2.DescriptorProto()
+    aggregate_message = unittest_custom_options_pb2.AggregateMessage
+    aggregate_message.DESCRIPTOR.CopyToProto(descriptor_proto)
+    reformed_descriptor = descriptor.MakeDescriptor(descriptor_proto)
+
+    options = reformed_descriptor.GetOptions()
+    self.assertEquals(101,
+                      options.Extensions[unittest_custom_options_pb2.msgopt].i)
+
 if __name__ == '__main__':
   basetest.main()

python/google/protobuf/internal/factory_test1.proto

@@ -30,6 +30,7 @@
 
 // Author: matthewtoia@google.com (Matt Toia)
 
+syntax = "proto2";
 
 package google.protobuf.python.internal;
 

python/google/protobuf/internal/factory_test2.proto

@@ -30,6 +30,7 @@
 
 // Author: matthewtoia@google.com (Matt Toia)
 
+syntax = "proto2";
 
 package google.protobuf.python.internal;
 
@@ -87,6 +88,12 @@ message LoopMessage {
   optional Factory2Message loop = 1;
 }
 
+message MessageWithNestedEnumOnly {
+  enum NestedEnum {
+    NESTED_MESSAGE_ENUM_0 = 0;
+  }
+}
+
 extend Factory1Message {
   optional string another_field = 1002;
 }

python/google/protobuf/internal/generator_test.py

@@ -35,7 +35,7 @@
 # indirect testing of the protocol compiler output.
 
 """Unittest that directly tests the output of the pure-Python protocol
-compiler.  See //google/protobuf/reflection_test.py for a test which
+compiler.  See //google/protobuf/internal/reflection_test.py for a test which
 further ensures that we can use Python protocol message objects as we expect.
 """
 
@@ -281,6 +281,8 @@ class GeneratorTest(basetest.TestCase):
                      "baz")
     self.assertEqual(message.Extensions[test_bad_identifiers_pb2.service],
                      "qux")
+    self.assertEqual(message.Extensions[test_bad_identifiers_pb2.class_],
+                     "Foo")
 
   def testOneof(self):
     desc = unittest_pb2.TestAllTypes.DESCRIPTOR

python/google/protobuf/internal/import_test_package/BUILD

@@ -0,0 +1,27 @@
+# Description:
+#   An example package that contains nested protos that are imported from
+#   __init__.py. See testPackageInitializationImport in reflection_test.py for
+#   details.
+
+package(
+    default_visibility = ["//net/proto2/python/internal:__pkg__"],
+)
+
+proto_library(
+    name = "inner_proto",
+    srcs = ["inner.proto"],
+    py_api_version = 2,
+)
+
+proto_library(
+    name = "outer_proto",
+    srcs = ["outer.proto"],
+    py_api_version = 2,
+    deps = [":inner_proto"],
+)
+
+py_library(
+    name = "import_test_package",
+    srcs = ["__init__.py"],
+    deps = [":outer_proto"],
+)

python/google/protobuf/internal/import_test_package/__init__.py

@@ -0,0 +1,33 @@
+# Protocol Buffers - Google's data interchange format
+# Copyright 2008 Google Inc.  All rights reserved.
+# https://developers.google.com/protocol-buffers/
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Sample module importing a nested proto from itself."""
+
+from google.protobuf.internal.import_test_package import outer_pb2 as myproto

python/google/protobuf/internal/import_test_package/inner.proto

@@ -0,0 +1,37 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+syntax = "proto2";
+
+package google.protobuf.python.internal.import_test_package;
+
+message Inner {
+  optional int32 value = 1 [default = 57];
+}

python/google/protobuf/internal/import_test_package/outer.proto

@@ -0,0 +1,39 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+syntax = "proto2";
+
+package google.protobuf.python.internal.import_test_package;
+
+import "google/protobuf/internal/import_test_package/inner.proto";
+
+message Outer {
+  optional Inner inner = 1;
+}

python/google/protobuf/internal/message_test.py

@@ -337,6 +337,20 @@ class MessageTest(basetest.TestCase):
     empty.ParseFromString(populated.SerializeToString())
     self.assertEqual(str(empty), '')
 
+  def testRepeatedNestedFieldIteration(self):
+    msg = unittest_pb2.TestAllTypes()
+    msg.repeated_nested_message.add(bb=1)
+    msg.repeated_nested_message.add(bb=2)
+    msg.repeated_nested_message.add(bb=3)
+    msg.repeated_nested_message.add(bb=4)
+
+    self.assertEquals([1, 2, 3, 4],
+                      [m.bb for m in msg.repeated_nested_message])
+    self.assertEquals([4, 3, 2, 1],
+                      [m.bb for m in reversed(msg.repeated_nested_message)])
+    self.assertEquals([4, 3, 2, 1],
+                      [m.bb for m in msg.repeated_nested_message[::-1]])
+
   def testSortingRepeatedScalarFieldsDefaultComparator(self):
     """Check some different types with the default comparator."""
     message = unittest_pb2.TestAllTypes()
@@ -641,6 +655,32 @@ class MessageTest(basetest.TestCase):
     m2.ParseFromString(m.SerializeToString())
     self.assertEqual('oneof_uint32', m2.WhichOneof('oneof_field'))
 
+  def testOneofCopyFrom(self):
+    m = unittest_pb2.TestAllTypes()
+    m.oneof_uint32 = 11
+    m2 = unittest_pb2.TestAllTypes()
+    m2.CopyFrom(m)
+    self.assertEqual('oneof_uint32', m2.WhichOneof('oneof_field'))
+
+  def testOneofNestedMergeFrom(self):
+    m = unittest_pb2.NestedTestAllTypes()
+    m.payload.oneof_uint32 = 11
+    m2 = unittest_pb2.NestedTestAllTypes()
+    m2.payload.oneof_bytes = b'bb'
+    m2.child.payload.oneof_bytes = b'bb'
+    m2.MergeFrom(m)
+    self.assertEqual('oneof_uint32', m2.payload.WhichOneof('oneof_field'))
+    self.assertEqual('oneof_bytes', m2.child.payload.WhichOneof('oneof_field'))
+
+  def testOneofClear(self):
+    m = unittest_pb2.TestAllTypes()
+    m.oneof_uint32 = 11
+    m.Clear()
+    self.assertIsNone(m.WhichOneof('oneof_field'))
+    m.oneof_bytes = b'bb'
+    self.assertTrue(m.HasField('oneof_field'))
+
+
   def testSortEmptyRepeatedCompositeContainer(self):
     """Exercise a scenario that has led to segfaults in the past.
     """

python/google/protobuf/internal/missing_enum_values.proto

@@ -28,6 +28,8 @@
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+syntax = "proto2";
+
 package google.protobuf.python.internal;
 
 message TestEnumValues {

python/google/protobuf/internal/more_extensions.proto

@@ -30,6 +30,7 @@
 
 // Author: robinson@google.com (Will Robinson)
 
+syntax = "proto2";
 
 package google.protobuf.internal;
 

python/google/protobuf/internal/more_extensions_dynamic.proto

@@ -34,6 +34,7 @@
 // generated C++ type is available for the extendee, but the extension is
 // defined in a file whose C++ type is not in the binary.
 
+syntax = "proto2";
 
 import "google/protobuf/internal/more_extensions.proto";
 

python/google/protobuf/internal/more_messages.proto

@@ -30,6 +30,7 @@
 
 // Author: robinson@google.com (Will Robinson)
 
+syntax = "proto2";
 
 package google.protobuf.internal;
 

python/google/protobuf/internal/proto_builder_test.py

@@ -0,0 +1,77 @@
+#! /usr/bin/python
+#
+# Protocol Buffers - Google's data interchange format
+# Copyright 2008 Google Inc.  All rights reserved.
+# https://developers.google.com/protocol-buffers/
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Tests for google.protobuf.proto_builder."""
+
+from google.apputils import basetest
+
+from google.protobuf import descriptor_pb2
+from google.protobuf import descriptor_pool
+from google.protobuf import proto_builder
+from google.protobuf import text_format
+
+
+class ProtoBuilderTest(basetest.TestCase):
+
+  def setUp(self):
+    self._fields = {
+        'foo': descriptor_pb2.FieldDescriptorProto.TYPE_INT64,
+        'bar': descriptor_pb2.FieldDescriptorProto.TYPE_STRING,
+        }
+
+  def testMakeSimpleProtoClass(self):
+    """Test that we can create a proto class."""
+    proto_cls = proto_builder.MakeSimpleProtoClass(
+        self._fields,
+        full_name='net.proto2.python.public.proto_builder_test.Test')
+    proto = proto_cls()
+    proto.foo = 12345
+    proto.bar = 'asdf'
+    self.assertMultiLineEqual(
+        'bar: "asdf"\nfoo: 12345\n', text_format.MessageToString(proto))
+
+  def testMakeSameProtoClassTwice(self):
+    """Test that the DescriptorPool is used."""
+    pool = descriptor_pool.DescriptorPool()
+    proto_cls1 = proto_builder.MakeSimpleProtoClass(
+        self._fields,
+        full_name='net.proto2.python.public.proto_builder_test.Test',
+        pool=pool)
+    proto_cls2 = proto_builder.MakeSimpleProtoClass(
+        self._fields,
+        full_name='net.proto2.python.public.proto_builder_test.Test',
+        pool=pool)
+    self.assertIs(proto_cls1.DESCRIPTOR, proto_cls2.DESCRIPTOR)
+
+
+if __name__ == '__main__':
+  basetest.main()

python/google/protobuf/internal/python_message.py

@@ -306,6 +306,17 @@ def _DefaultValueConstructorForField(field):
   return MakeScalarDefault
 
 
+def _ReraiseTypeErrorWithFieldName(message_name, field_name):
+  """Re-raise the currently-handled TypeError with the field name added."""
+  exc = sys.exc_info()[1]
+  if len(exc.args) == 1 and type(exc) is TypeError:
+    # simple TypeError; add field name to exception message
+    exc = TypeError('%s for field %s.%s' % (str(exc), message_name, field_name))
+
+  # re-raise possibly-amended exception with original traceback:
+  raise type(exc), exc, sys.exc_info()[2]
+
+
 def _AddInitMethod(message_descriptor, cls):
   """Adds an __init__ method to cls."""
   fields = message_descriptor.fields
@@ -338,10 +349,16 @@ def _AddInitMethod(message_descriptor, cls):
         self._fields[field] = copy
       elif field.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE:
         copy = field._default_constructor(self)
-        copy.MergeFrom(field_value)
+        try:
+          copy.MergeFrom(field_value)
+        except TypeError:
+          _ReraiseTypeErrorWithFieldName(message_descriptor.name, field_name)
         self._fields[field] = copy
       else:
-        setattr(self, field_name, field_value)
+        try:
+          setattr(self, field_name, field_value)
+        except TypeError:
+          _ReraiseTypeErrorWithFieldName(message_descriptor.name, field_name)
 
   init.__module__ = None
   init.__doc__ = None
@@ -691,6 +708,7 @@ def _AddClearMethod(message_descriptor, cls):
     # Clear fields.
     self._fields = {}
     self._unknown_fields = ()
+    self._oneofs = {}
     self._Modified()
   cls.Clear = Clear
 
@@ -993,6 +1011,8 @@ def _AddMergeFromMethod(cls):
           field_value.MergeFrom(value)
       else:
         self._fields[field] = value
+        if field.containing_oneof:
+          self._UpdateOneofState(field)
 
     if msg._unknown_fields:
       if not self._unknown_fields:

python/google/protobuf/internal/reflection_test.py

@@ -35,8 +35,6 @@
 pure-Python protocol compiler.
 """
 
-__author__ = 'robinson@google.com (Will Robinson)'
-
 import copy
 import gc
 import operator
@@ -1252,15 +1250,18 @@ class ReflectionTest(basetest.TestCase):
 
     # Try something that *is* an extension handle, just not for
     # this message...
-    unknown_handle = more_extensions_pb2.optional_int_extension
-    self.assertRaises(KeyError, extendee_proto.HasExtension,
-                      unknown_handle)
-    self.assertRaises(KeyError, extendee_proto.ClearExtension,
-                      unknown_handle)
-    self.assertRaises(KeyError, extendee_proto.Extensions.__getitem__,
-                      unknown_handle)
-    self.assertRaises(KeyError, extendee_proto.Extensions.__setitem__,
-                      unknown_handle, 5)
+    for unknown_handle in (more_extensions_pb2.optional_int_extension,
+                           more_extensions_pb2.optional_message_extension,
+                           more_extensions_pb2.repeated_int_extension,
+                           more_extensions_pb2.repeated_message_extension):
+      self.assertRaises(KeyError, extendee_proto.HasExtension,
+                        unknown_handle)
+      self.assertRaises(KeyError, extendee_proto.ClearExtension,
+                        unknown_handle)
+      self.assertRaises(KeyError, extendee_proto.Extensions.__getitem__,
+                        unknown_handle)
+      self.assertRaises(KeyError, extendee_proto.Extensions.__setitem__,
+                        unknown_handle, 5)
 
     # Try call HasExtension() with a valid handle, but for a
     # *repeated* field.  (Just as with non-extension repeated
@@ -1669,16 +1670,15 @@ class ReflectionTest(basetest.TestCase):
     proto.optional_string = str('Testing')
     self.assertEqual(proto.optional_string, unicode('Testing'))
 
-    # Try to assign a 'str' value which contains bytes that aren't 7-bit ASCII.
+    # Try to assign a 'bytes' object which contains non-UTF-8.
     self.assertRaises(ValueError,
                       setattr, proto, 'optional_string', b'a\x80a')
-    if str is bytes:  # PY2
-      # Assign a 'str' object which contains a UTF-8 encoded string.
-      self.assertRaises(ValueError,
-                        setattr, proto, 'optional_string', 'Тест')
-    else:
-      proto.optional_string = 'Тест'
-    # No exception thrown.
+    # No exception: Assign already encoded UTF-8 bytes to a string field.
+    utf8_bytes = u'Тест'.encode('utf-8')
+    proto.optional_string = utf8_bytes
+    # No exception: Assign the a non-ascii unicode object.
+    proto.optional_string = u'Тест'
+    # No exception thrown (normal str assignment containing ASCII).
     proto.optional_string = 'abc'
 
   def testStringUTF8Serialization(self):
@@ -1774,6 +1774,24 @@ class ReflectionTest(basetest.TestCase):
     proto.optionalgroup.SetInParent()
     self.assertTrue(proto.HasField('optionalgroup'))
 
+  def testPackageInitializationImport(self):
+    """Test that we can import nested messages from their __init__.py.
+
+    Such setup is not trivial since at the time of processing of __init__.py one
+    can't refer to its submodules by name in code, so expressions like
+    google.protobuf.internal.import_test_package.inner_pb2
+    don't work. They do work in imports, so we have assign an alias at import
+    and then use that alias in generated code.
+    """
+    # We import here since it's the import that used to fail, and we want
+    # the failure to have the right context.
+    # pylint: disable=g-import-not-at-top
+    from google.protobuf.internal import import_test_package
+    # pylint: enable=g-import-not-at-top
+    msg = import_test_package.myproto.Outer()
+    # Just check the default value.
+    self.assertEqual(57, msg.inner.value)
+
 
 #  Since we had so many tests for protocol buffer equality, we broke these out
 #  into separate TestCase classes.
@@ -2802,6 +2820,9 @@ class OptionsTest(basetest.TestCase):
 
 class ClassAPITest(basetest.TestCase):
 
+  @basetest.unittest.skipIf(
+      api_implementation.Type() == 'cpp' and api_implementation.Version() == 2,
+      'C++ implementation requires a call to MakeDescriptor()')
   def testMakeClassWithNestedDescriptor(self):
     leaf_desc = descriptor.Descriptor('leaf', 'package.parent.child.leaf', '',
                                       containing_type=None, fields=[],

python/google/protobuf/internal/test_bad_identifiers.proto

@@ -30,6 +30,7 @@
 
 // Author: kenton@google.com (Kenton Varda)
 
+syntax = "proto2";
 
 package protobuf_unittest;
 
@@ -39,13 +40,15 @@ message TestBadIdentifiers {
   extensions 100 to max;
 }
 
-// Make sure these reasonable extension names don't conflict with internal
-// variables.
 extend TestBadIdentifiers {
+  // Make sure these reasonable extension names don't conflict with internal
+  // variables.
   optional string message = 100 [default="foo"];
   optional string descriptor = 101 [default="bar"];
   optional string reflection = 102 [default="baz"];
   optional string service = 103 [default="qux"];
+  // And Python keywords.
+  optional string class = 104 [default="Foo"];
 }
 
 message AnotherMessage {}

python/google/protobuf/internal/text_format_test.py

@@ -69,13 +69,18 @@ class TextFormatTest(basetest.TestCase):
     message.my_string = '115'
     message.my_int = 101
     message.my_float = 111
+    message.optional_nested_message.oo = 0
+    message.optional_nested_message.bb = 1
     self.CompareToGoldenText(
         self.RemoveRedundantZeros(text_format.MessageToString(
             message, use_index_order=True)),
-        'my_string: \"115\"\nmy_int: 101\nmy_float: 111\n')
+        'my_string: \"115\"\nmy_int: 101\nmy_float: 111\n'
+        'optional_nested_message {\n  oo: 0\n  bb: 1\n}\n')
     self.CompareToGoldenText(
         self.RemoveRedundantZeros(text_format.MessageToString(
-            message)), 'my_int: 101\nmy_string: \"115\"\nmy_float: 111\n')
+            message)),
+        'my_int: 101\nmy_string: \"115\"\nmy_float: 111\n'
+        'optional_nested_message {\n  bb: 1\n  oo: 0\n}\n')
 
   def testPrintAllExtensions(self):
     message = unittest_pb2.TestAllExtensions()
@@ -511,7 +516,7 @@ class TextFormatTest(basetest.TestCase):
                      message.repeated_string[4])
     self.assertEqual(SLASH + 'x20', message.repeated_string[5])
 
-  def testMergeRepeatedScalars(self):
+  def testMergeDuplicateScalars(self):
     message = unittest_pb2.TestAllTypes()
     text = ('optional_int32: 42 '
             'optional_int32: 67')
@@ -519,7 +524,7 @@ class TextFormatTest(basetest.TestCase):
     self.assertIs(r, message)
     self.assertEqual(67, message.optional_int32)
 
-  def testParseRepeatedScalars(self):
+  def testParseDuplicateScalars(self):
     message = unittest_pb2.TestAllTypes()
     text = ('optional_int32: 42 '
             'optional_int32: 67')
@@ -529,7 +534,7 @@ class TextFormatTest(basetest.TestCase):
          'have multiple "optional_int32" fields.'),
         text_format.Parse, text, message)
 
-  def testMergeRepeatedNestedMessageScalars(self):
+  def testMergeDuplicateNestedMessageScalars(self):
     message = unittest_pb2.TestAllTypes()
     text = ('optional_nested_message { bb: 1 } '
             'optional_nested_message { bb: 2 }')
@@ -537,7 +542,7 @@ class TextFormatTest(basetest.TestCase):
     self.assertTrue(r is message)
     self.assertEqual(2, message.optional_nested_message.bb)
 
-  def testParseRepeatedNestedMessageScalars(self):
+  def testParseDuplicateNestedMessageScalars(self):
     message = unittest_pb2.TestAllTypes()
     text = ('optional_nested_message { bb: 1 } '
             'optional_nested_message { bb: 2 }')
@@ -547,7 +552,7 @@ class TextFormatTest(basetest.TestCase):
          'should not have multiple "bb" fields.'),
         text_format.Parse, text, message)
 
-  def testMergeRepeatedExtensionScalars(self):
+  def testMergeDuplicateExtensionScalars(self):
     message = unittest_pb2.TestAllExtensions()
     text = ('[protobuf_unittest.optional_int32_extension]: 42 '
             '[protobuf_unittest.optional_int32_extension]: 67')
@@ -556,7 +561,7 @@ class TextFormatTest(basetest.TestCase):
         67,
         message.Extensions[unittest_pb2.optional_int32_extension])
 
-  def testParseRepeatedExtensionScalars(self):
+  def testParseDuplicateExtensionScalars(self):
     message = unittest_pb2.TestAllExtensions()
     text = ('[protobuf_unittest.optional_int32_extension]: 42 '
             '[protobuf_unittest.optional_int32_extension]: 67')

python/google/protobuf/internal/type_checkers.py

@@ -154,14 +154,13 @@ class UnicodeValueChecker(object):
                  (proposed_value, type(proposed_value), (bytes, unicode)))
       raise TypeError(message)
 
-    # If the value is of type 'bytes' make sure that it is in 7-bit ASCII
-    # encoding.
+    # If the value is of type 'bytes' make sure that it is valid UTF-8 data.
     if isinstance(proposed_value, bytes):
       try:
-        proposed_value = proposed_value.decode('ascii')
+        proposed_value = proposed_value.decode('utf-8')
       except UnicodeDecodeError:
-        raise ValueError('%.1024r has type bytes, but isn\'t in 7-bit ASCII '
-                         'encoding. Non-ASCII strings must be converted to '
+        raise ValueError('%.1024r has type bytes, but isn\'t valid UTF-8 '
+                         'encoding. Non-UTF-8 strings must be converted to '
                          'unicode objects before being added.' %
                          (proposed_value))
     return proposed_value

python/google/protobuf/internal/unknown_fields_test.py

@@ -38,12 +38,16 @@ __author__ = 'bohdank@google.com (Bohdan Koval)'
 from google.apputils import basetest
 from google.protobuf import unittest_mset_pb2
 from google.protobuf import unittest_pb2
+from google.protobuf.internal import api_implementation
 from google.protobuf.internal import encoder
 from google.protobuf.internal import missing_enum_values_pb2
 from google.protobuf.internal import test_util
 from google.protobuf.internal import type_checkers
 
 
+@basetest.unittest.skipIf(
+    api_implementation.Type() == 'cpp' and api_implementation.Version() == 2,
+    'C++ implementation does not expose unknown fields to Python')
 class UnknownFieldsTest(basetest.TestCase):
 
   def setUp(self):
@@ -175,7 +179,10 @@ class UnknownFieldsTest(basetest.TestCase):
     self.assertNotEqual(self.empty_message, message)
 
 
-class UnknownFieldsTest(basetest.TestCase):
+@basetest.unittest.skipIf(
+    api_implementation.Type() == 'cpp' and api_implementation.Version() == 2,
+    'C++ implementation does not expose unknown fields to Python')
+class UnknownEnumValuesTest(basetest.TestCase):
 
   def setUp(self):
     self.descriptor = missing_enum_values_pb2.TestEnumValues.DESCRIPTOR

python/google/protobuf/message.py

@@ -233,12 +233,21 @@ class Message(object):
     raise NotImplementedError
 
   def HasField(self, field_name):
-    """Checks if a certain field is set for the message. Note if the
-    field_name is not defined in the message descriptor, ValueError will be
-    raised."""
+    """Checks if a certain field is set for the message, or if any field inside
+    a oneof group is set.  Note that if the field_name is not defined in the
+    message descriptor, ValueError will be raised."""
     raise NotImplementedError
 
   def ClearField(self, field_name):
+    """Clears the contents of a given field, or the field set inside a oneof
+    group.  If the name neither refers to a defined field or oneof group,
+    ValueError is raised."""
+    raise NotImplementedError
+
+  def WhichOneof(self, oneof_group):
+    """Returns the name of the field that is set inside a oneof group, or
+    None if no field is set.  If no group with the given name exists, ValueError
+    will be raised."""
     raise NotImplementedError
 
   def HasExtension(self, extension_handle):

python/google/protobuf/proto_builder.py

@@ -0,0 +1,98 @@
+# Protocol Buffers - Google's data interchange format
+# Copyright 2008 Google Inc.  All rights reserved.
+# https://developers.google.com/protocol-buffers/
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Dynamic Protobuf class creator."""
+
+import hashlib
+import os
+
+from google.protobuf import descriptor_pb2
+from google.protobuf import message_factory
+
+
+def _GetMessageFromFactory(factory, full_name):
+  """Get a proto class from the MessageFactory by name.
+
+  Args:
+    factory: a MessageFactory instance.
+    full_name: str, the fully qualified name of the proto type.
+  Returns:
+    a class, for the type identified by full_name.
+  Raises:
+    KeyError, if the proto is not found in the factory's descriptor pool.
+  """
+  proto_descriptor = factory.pool.FindMessageTypeByName(full_name)
+  proto_cls = factory.GetPrototype(proto_descriptor)
+  return proto_cls
+
+
+def MakeSimpleProtoClass(fields, full_name, pool=None):
+  """Create a Protobuf class whose fields are basic types.
+
+  Note: this doesn't validate field names!
+
+  Args:
+    fields: dict of {name: field_type} mappings for each field in the proto.
+    full_name: str, the fully-qualified name of the proto type.
+    pool: optional DescriptorPool instance.
+  Returns:
+    a class, the new protobuf class with a FileDescriptor.
+  """
+  factory = message_factory.MessageFactory(pool=pool)
+  try:
+    proto_cls = _GetMessageFromFactory(factory, full_name)
+    return proto_cls
+  except KeyError:
+    # The factory's DescriptorPool doesn't know about this class yet.
+    pass
+
+  # Use a consistent file name that is unlikely to conflict with any imported
+  # proto files.
+  fields_hash = hashlib.sha1()
+  for f_name, f_type in sorted(fields.items()):
+    fields_hash.update(f_name.encode('utf8'))
+    fields_hash.update(str(f_type).encode('utf8'))
+  proto_file_name = fields_hash.hexdigest() + '.proto'
+
+  package, name = full_name.rsplit('.', 1)
+  file_proto = descriptor_pb2.FileDescriptorProto()
+  file_proto.name = os.path.join(package.replace('.', '/'), proto_file_name)
+  file_proto.package = package
+  desc_proto = file_proto.message_type.add()
+  desc_proto.name = name
+  for f_number, (f_name, f_type) in enumerate(sorted(fields.items()), 1):
+    field_proto = desc_proto.field.add()
+    field_proto.name = f_name
+    field_proto.number = f_number
+    field_proto.label = descriptor_pb2.FieldDescriptorProto.LABEL_OPTIONAL
+    field_proto.type = f_type
+
+  factory.pool.Add(file_proto)
+  return _GetMessageFromFactory(factory, full_name)

python/google/protobuf/pyext/cpp_message.py

@@ -53,9 +53,5 @@ def NewMessage(bases, message_descriptor, dictionary):
 
 
 def InitMessage(message_descriptor, cls):
-  """Constructs a new message instance (called before instance's __init__)."""
-
-  def SubInit(self, **kwargs):
-    super(cls, self).__init__(message_descriptor, **kwargs)
-  cls.__init__ = SubInit
+  """Finalizes the creation of a message class."""
   cls.AddDescriptors(message_descriptor)

python/google/protobuf/pyext/descriptor.cc

@@ -35,6 +35,7 @@
 
 #include <google/protobuf/descriptor.pb.h>
 #include <google/protobuf/pyext/descriptor.h>
+#include <google/protobuf/pyext/message.h>
 #include <google/protobuf/pyext/scoped_pyobject_ptr.h>
 
 #define C(str) const_cast<char*>(str)
@@ -46,7 +47,7 @@
     #error "Python 3.0 - 3.2 are not supported."
   #else
   #define PyString_AsString(ob) \
-    (PyUnicode_Check(ob)? PyUnicode_AsUTF8(ob): PyBytes_AS_STRING(ob))
+    (PyUnicode_Check(ob)? PyUnicode_AsUTF8(ob): PyBytes_AsString(ob))
   #endif
 #endif
 
@@ -65,10 +66,80 @@ namespace python {
 
 static google::protobuf::DescriptorPool* g_descriptor_pool = NULL;
 
+namespace cmessage_descriptor {
+
+static void Dealloc(CMessageDescriptor* self) {
+  Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self));
+}
+
+static PyObject* GetFullName(CMessageDescriptor* self, void *closure) {
+  return PyString_FromStringAndSize(
+      self->descriptor->full_name().c_str(),
+      self->descriptor->full_name().size());
+}
+
+static PyObject* GetName(CMessageDescriptor *self, void *closure) {
+  return PyString_FromStringAndSize(
+      self->descriptor->name().c_str(),
+      self->descriptor->name().size());
+}
+
+static PyGetSetDef Getters[] = {
+  { C("full_name"), (getter)GetFullName, NULL, "Full name", NULL},
+  { C("name"), (getter)GetName, NULL, "Unqualified name", NULL},
+  {NULL}
+};
+
+}  // namespace cmessage_descriptor
+
+PyTypeObject CMessageDescriptor_Type = {
+  PyVarObject_HEAD_INIT(&PyType_Type, 0)
+  C("google.protobuf.internal."
+    "_net_proto2___python."
+    "CMessageDescriptor"),              // tp_name
+  sizeof(CMessageDescriptor),           // tp_basicsize
+  0,                                    // tp_itemsize
+  (destructor)cmessage_descriptor::Dealloc,  // tp_dealloc
+  0,                                    // tp_print
+  0,                                    // tp_getattr
+  0,                                    // tp_setattr
+  0,                                    // tp_compare
+  0,                                    // tp_repr
+  0,                                    // tp_as_number
+  0,                                    // tp_as_sequence
+  0,                                    // tp_as_mapping
+  0,                                    // tp_hash
+  0,                                    // tp_call
+  0,                                    // tp_str
+  0,                                    // tp_getattro
+  0,                                    // tp_setattro
+  0,                                    // tp_as_buffer
+  Py_TPFLAGS_DEFAULT,                   // tp_flags
+  C("A Message Descriptor"),            // tp_doc
+  0,                                    // tp_traverse
+  0,                                    // tp_clear
+  0,                                    // tp_richcompare
+  0,                                    // tp_weaklistoffset
+  0,                                    // tp_iter
+  0,                                    // tp_iternext
+  0,                                    // tp_methods
+  0,                                    // tp_members
+  cmessage_descriptor::Getters,         // tp_getset
+  0,                                    // tp_base
+  0,                                    // tp_dict
+  0,                                    // tp_descr_get
+  0,                                    // tp_descr_set
+  0,                                    // tp_dictoffset
+  0,                                    // tp_init
+  PyType_GenericAlloc,                  // tp_alloc
+  PyType_GenericNew,                    // tp_new
+  PyObject_Del,                         // tp_free
+};
+
+
 namespace cfield_descriptor {
 
 static void Dealloc(CFieldDescriptor* self) {
-  Py_CLEAR(self->descriptor_field);
   Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self));
 }
 
@@ -98,7 +169,7 @@ static PyObject* GetID(CFieldDescriptor *self, void *closure) {
 
 static PyGetSetDef Getters[] = {
   { C("full_name"), (getter)GetFullName, NULL, "Full name", NULL},
-  { C("name"), (getter)GetName, NULL, "last name", NULL},
+  { C("name"), (getter)GetName, NULL, "Unqualified name", NULL},
   { C("cpp_type"), (getter)GetCppType, NULL, "C++ Type", NULL},
   { C("label"), (getter)GetLabel, NULL, "Label", NULL},
   { C("id"), (getter)GetID, NULL, "ID", NULL},
@@ -151,13 +222,56 @@ PyTypeObject CFieldDescriptor_Type = {
   PyObject_Del,                         // tp_free
 };
 
+
 namespace cdescriptor_pool {
 
-static void Dealloc(CDescriptorPool* self) {
+PyDescriptorPool* NewDescriptorPool() {
+  PyDescriptorPool* cdescriptor_pool = PyObject_New(
+      PyDescriptorPool, &PyDescriptorPool_Type);
+  if (cdescriptor_pool == NULL) {
+    return NULL;
+  }
+
+  // Build a DescriptorPool for messages only declared in Python libraries.
+  // generated_pool() contains all messages linked in C++ libraries, and is used
+  // as underlay.
+  cdescriptor_pool->pool = new google::protobuf::DescriptorPool(
+      google::protobuf::DescriptorPool::generated_pool());
+
+  // TODO(amauryfa): Rewrite the SymbolDatabase in C so that it uses the same
+  // storage.
+  cdescriptor_pool->classes_by_descriptor =
+      new PyDescriptorPool::ClassesByMessageMap();
+
+  return cdescriptor_pool;
+}
+
+static void Dealloc(PyDescriptorPool* self) {
+  for (auto it : (*self->classes_by_descriptor)) {
+    Py_DECREF(it.second);
+  }
+  delete self->classes_by_descriptor;
   Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self));
 }
 
-static PyObject* NewCDescriptor(
+const google::protobuf::Descriptor* FindMessageTypeByName(PyDescriptorPool* self,
+                                                const string& name) {
+  return self->pool->FindMessageTypeByName(name);
+}
+
+static PyObject* NewCMessageDescriptor(
+    const google::protobuf::Descriptor* message_descriptor) {
+  CMessageDescriptor* cmessage_descriptor = PyObject_New(
+      CMessageDescriptor, &CMessageDescriptor_Type);
+  if (cmessage_descriptor == NULL) {
+    return NULL;
+  }
+  cmessage_descriptor->descriptor = message_descriptor;
+
+  return reinterpret_cast<PyObject*>(cmessage_descriptor);
+}
+
+static PyObject* NewCFieldDescriptor(
     const google::protobuf::FieldDescriptor* field_descriptor) {
   CFieldDescriptor* cfield_descriptor = PyObject_New(
       CFieldDescriptor, &CFieldDescriptor_Type);
@@ -165,12 +279,61 @@ static PyObject* NewCDescriptor(
     return NULL;
   }
   cfield_descriptor->descriptor = field_descriptor;
-  cfield_descriptor->descriptor_field = NULL;
 
   return reinterpret_cast<PyObject*>(cfield_descriptor);
 }
 
-PyObject* FindFieldByName(CDescriptorPool* self, PyObject* name) {
+// Add a message class to our database.
+const google::protobuf::Descriptor* RegisterMessageClass(
+    PyDescriptorPool* self, PyObject *message_class, PyObject* descriptor) {
+  ScopedPyObjectPtr full_message_name(
+      PyObject_GetAttrString(descriptor, "full_name"));
+  const char* full_name = PyString_AsString(full_message_name);
+  if (full_name == NULL) {
+    return NULL;
+  }
+  const Descriptor *message_descriptor =
+      self->pool->FindMessageTypeByName(full_name);
+  if (!message_descriptor) {
+    PyErr_Format(PyExc_TypeError, "Could not find C++ descriptor for '%s'",
+                 full_name);
+    return NULL;
+  }
+  Py_INCREF(message_class);
+  auto ret = self->classes_by_descriptor->insert(
+      make_pair(message_descriptor, message_class));
+  if (!ret.second) {
+    // Update case: DECREF the previous value.
+    Py_DECREF(ret.first->second);
+    ret.first->second = message_class;
+  }
+
+  // Also add the C++ descriptor to the Python descriptor class.
+  ScopedPyObjectPtr cdescriptor(NewCMessageDescriptor(message_descriptor));
+  if (cdescriptor == NULL) {
+    return NULL;
+  }
+  if (PyObject_SetAttrString(
+          descriptor, "_cdescriptor", cdescriptor) < 0) {
+      return NULL;
+  }
+  return message_descriptor;
+}
+
+// Retrieve the message class added to our database.
+PyObject *GetMessageClass(PyDescriptorPool* self,
+                          const Descriptor *message_descriptor) {
+  auto ret = self->classes_by_descriptor->find(message_descriptor);
+  if (ret == self->classes_by_descriptor->end()) {
+    PyErr_Format(PyExc_TypeError, "No message class registered for '%s'",
+                 message_descriptor->full_name().c_str());
+    return NULL;
+  } else {
+    return ret->second;
+  }
+}
+
+PyObject* FindFieldByName(PyDescriptorPool* self, PyObject* name) {
   const char* full_field_name = PyString_AsString(name);
   if (full_field_name == NULL) {
     return NULL;
@@ -186,10 +349,10 @@ PyObject* FindFieldByName(CDescriptorPool* self, PyObject* name) {
     return NULL;
   }
 
-  return NewCDescriptor(field_descriptor);
+  return NewCFieldDescriptor(field_descriptor);
 }
 
-PyObject* FindExtensionByName(CDescriptorPool* self, PyObject* arg) {
+PyObject* FindExtensionByName(PyDescriptorPool* self, PyObject* arg) {
   const char* full_field_name = PyString_AsString(arg);
   if (full_field_name == NULL) {
     return NULL;
@@ -203,7 +366,7 @@ PyObject* FindExtensionByName(CDescriptorPool* self, PyObject* arg) {
     return NULL;
   }
 
-  return NewCDescriptor(field_descriptor);
+  return NewCFieldDescriptor(field_descriptor);
 }
 
 static PyMethodDef Methods[] = {
@@ -220,12 +383,12 @@ static PyMethodDef Methods[] = {
 
 }  // namespace cdescriptor_pool
 
-PyTypeObject CDescriptorPool_Type = {
+PyTypeObject PyDescriptorPool_Type = {
   PyVarObject_HEAD_INIT(&PyType_Type, 0)
   C("google.protobuf.internal."
     "_net_proto2___python."
     "CFieldDescriptor"),               // tp_name
-  sizeof(CDescriptorPool),             // tp_basicsize
+  sizeof(PyDescriptorPool),            // tp_basicsize
   0,                                   // tp_itemsize
   (destructor)cdescriptor_pool::Dealloc,  // tp_dealloc
   0,                                   // tp_print
@@ -259,29 +422,11 @@ PyTypeObject CDescriptorPool_Type = {
   0,                                   // tp_descr_set
   0,                                   // tp_dictoffset
   0,                                   // tp_init
-  PyType_GenericAlloc,                 // tp_alloc
-  PyType_GenericNew,                   // tp_new
+  0,                                   // tp_alloc
+  0,                                   // tp_new
   PyObject_Del,                        // tp_free
 };
 
-google::protobuf::DescriptorPool* GetDescriptorPool() {
-  if (g_descriptor_pool == NULL) {
-    g_descriptor_pool = new google::protobuf::DescriptorPool(
-        google::protobuf::DescriptorPool::generated_pool());
-  }
-  return g_descriptor_pool;
-}
-
-PyObject* Python_NewCDescriptorPool(PyObject* ignored, PyObject* args) {
-  CDescriptorPool* cdescriptor_pool = PyObject_New(
-      CDescriptorPool, &CDescriptorPool_Type);
-  if (cdescriptor_pool == NULL) {
-    return NULL;
-  }
-  cdescriptor_pool->pool = GetDescriptorPool();
-  return reinterpret_cast<PyObject*>(cdescriptor_pool);
-}
-
 
 // Collects errors that occur during proto file building to allow them to be
 // propagated in the python exception instead of only living in ERROR logs.
@@ -321,6 +466,8 @@ PyObject* Python_BuildFile(PyObject* ignored, PyObject* arg) {
     return NULL;
   }
 
+  // If the file was already part of a C++ library, all its descriptors are in
+  // the underlying pool.  No need to do anything else.
   if (google::protobuf::DescriptorPool::generated_pool()->FindFileByName(
           file_proto.name()) != NULL) {
     Py_RETURN_NONE;
@@ -328,8 +475,8 @@ PyObject* Python_BuildFile(PyObject* ignored, PyObject* arg) {
 
   BuildFileErrorCollector error_collector;
   const google::protobuf::FileDescriptor* descriptor =
-      GetDescriptorPool()->BuildFileCollectingErrors(file_proto,
-                                                     &error_collector);
+      GetDescriptorPool()->pool->BuildFileCollectingErrors(file_proto,
+                                                           &error_collector);
   if (descriptor == NULL) {
     PyErr_Format(PyExc_TypeError,
                  "Couldn't build proto file into descriptor pool!\n%s",
@@ -341,12 +488,13 @@ PyObject* Python_BuildFile(PyObject* ignored, PyObject* arg) {
 }
 
 bool InitDescriptor() {
-  CFieldDescriptor_Type.tp_new = PyType_GenericNew;
+  if (PyType_Ready(&CMessageDescriptor_Type) < 0)
+    return false;
   if (PyType_Ready(&CFieldDescriptor_Type) < 0)
     return false;
 
-  CDescriptorPool_Type.tp_new = PyType_GenericNew;
-  if (PyType_Ready(&CDescriptorPool_Type) < 0)
+  PyDescriptorPool_Type.tp_new = PyType_GenericNew;
+  if (PyType_Ready(&PyDescriptorPool_Type) < 0)
     return false;
 
   return true;

python/google/protobuf/pyext/descriptor.h

@@ -36,6 +36,8 @@
 #include <Python.h>
 #include <structmember.h>
 
+#include <google/protobuf/stubs/hash.h>
+
 #include <google/protobuf/descriptor.h>
 
 #if PY_VERSION_HEX < 0x02050000 && !defined(PY_SSIZE_T_MIN)
@@ -48,46 +50,90 @@ namespace google {
 namespace protobuf {
 namespace python {
 
+typedef struct CMessageDescriptor {
+  PyObject_HEAD
+
+  // The proto2 descriptor that this object represents.
+  const google::protobuf::Descriptor* descriptor;
+} CMessageDescriptor;
+
+
 typedef struct CFieldDescriptor {
   PyObject_HEAD
 
   // The proto2 descriptor that this object represents.
   const google::protobuf::FieldDescriptor* descriptor;
-
-  // Reference to the original field object in the Python DESCRIPTOR.
-  PyObject* descriptor_field;
 } CFieldDescriptor;
 
-typedef struct {
+
+// Wraps operations to the global DescriptorPool which contains information
+// about all messages and fields.
+//
+// There is normally one pool per process. We make it a Python object only
+// because it contains many Python references.
+// TODO(amauryfa): See whether such objects can appear in reference cycles, and
+// consider adding support for the cyclic GC.
+//
+// "Methods" that interacts with this DescriptorPool are in the cdescriptor_pool
+// namespace.
+typedef struct PyDescriptorPool {
   PyObject_HEAD
 
-  const google::protobuf::DescriptorPool* pool;
-} CDescriptorPool;
+  google::protobuf::DescriptorPool* pool;
 
+  // Make our own mapping to retrieve Python classes from C++ descriptors.
+  //
+  // Descriptor pointers stored here are owned by the DescriptorPool above.
+  // Python references to classes are owned by this PyDescriptorPool.
+  typedef hash_map<const Descriptor *, PyObject *> ClassesByMessageMap;
+  ClassesByMessageMap *classes_by_descriptor;
+} PyDescriptorPool;
+
+
+extern PyTypeObject CMessageDescriptor_Type;
 extern PyTypeObject CFieldDescriptor_Type;
 
-extern PyTypeObject CDescriptorPool_Type;
+extern PyTypeObject PyDescriptorPool_Type;
+
 
 namespace cdescriptor_pool {
 
+// Builds a new DescriptorPool. Normally called only once per process.
+PyDescriptorPool* NewDescriptorPool();
+
+// Looks up a message by name.
+// Returns a message Descriptor, or NULL if not found.
+const google::protobuf::Descriptor* FindMessageTypeByName(PyDescriptorPool* self,
+                                                const string& name);
+
+// Registers a new Python class for the given message descriptor.
+// Returns the message Descriptor.
+// On error, returns NULL with a Python exception set.
+const google::protobuf::Descriptor* RegisterMessageClass(
+    PyDescriptorPool* self, PyObject *message_class, PyObject *descriptor);
+
+// Retrieves the Python class registered with the given message descriptor.
+//
+// Returns a *borrowed* reference if found, otherwise returns NULL with an
+// exception set.
+PyObject *GetMessageClass(PyDescriptorPool* self,
+                          const Descriptor *message_descriptor);
+
 // Looks up a field by name. Returns a CDescriptor corresponding to
 // the field on success, or NULL on failure.
 //
 // Returns a new reference.
-PyObject* FindFieldByName(CDescriptorPool* self, PyObject* name);
+PyObject* FindFieldByName(PyDescriptorPool* self, PyObject* name);
 
 // Looks up an extension by name. Returns a CDescriptor corresponding
 // to the field on success, or NULL on failure.
 //
 // Returns a new reference.
-PyObject* FindExtensionByName(CDescriptorPool* self, PyObject* arg);
-
+PyObject* FindExtensionByName(PyDescriptorPool* self, PyObject* arg);
 }  // namespace cdescriptor_pool
 
-PyObject* Python_NewCDescriptorPool(PyObject* ignored, PyObject* args);
 PyObject* Python_BuildFile(PyObject* ignored, PyObject* args);
 bool InitDescriptor();
-google::protobuf::DescriptorPool* GetDescriptorPool();
 
 }  // namespace python
 }  // namespace protobuf

python/google/protobuf/pyext/extension_dict.cc

@@ -62,22 +62,6 @@ static google::protobuf::Message* GetMessage(ExtensionDict* self) {
   }
 }
 
-CFieldDescriptor* InternalGetCDescriptorFromExtension(PyObject* extension) {
-  PyObject* cdescriptor = PyObject_GetAttrString(extension, "_cdescriptor");
-  if (cdescriptor == NULL) {
-    PyErr_SetString(PyExc_KeyError, "Unregistered extension.");
-    return NULL;
-  }
-  if (!PyObject_TypeCheck(cdescriptor, &CFieldDescriptor_Type)) {
-    PyErr_SetString(PyExc_TypeError, "Not a CFieldDescriptor");
-    Py_DECREF(cdescriptor);
-    return NULL;
-  }
-  CFieldDescriptor* descriptor =
-      reinterpret_cast<CFieldDescriptor*>(cdescriptor);
-  return descriptor;
-}
-
 PyObject* len(ExtensionDict* self) {
 #if PY_MAJOR_VERSION >= 3
   return PyLong_FromLong(PyDict_Size(self->values));
@@ -118,16 +102,15 @@ int ReleaseExtension(ExtensionDict* self,
 }
 
 PyObject* subscript(ExtensionDict* self, PyObject* key) {
-  CFieldDescriptor* cdescriptor = InternalGetCDescriptorFromExtension(
-      key);
-  if (cdescriptor == NULL) {
+  const google::protobuf::FieldDescriptor* descriptor =
+      cmessage::GetExtensionDescriptor(key);
+  if (descriptor == NULL) {
     return NULL;
   }
-  ScopedPyObjectPtr py_cdescriptor(reinterpret_cast<PyObject*>(cdescriptor));
-  const google::protobuf::FieldDescriptor* descriptor = cdescriptor->descriptor;
-  if (descriptor == NULL) {
+  if (!CheckFieldBelongsToMessage(descriptor, self->parent->message)) {
     return NULL;
   }
+
   if (descriptor->label() != FieldDescriptor::LABEL_REPEATED &&
       descriptor->cpp_type() != FieldDescriptor::CPPTYPE_MESSAGE) {
     return cmessage::InternalGetScalar(self->parent, descriptor);
@@ -142,7 +125,7 @@ PyObject* subscript(ExtensionDict* self, PyObject* key) {
   if (descriptor->label() != FieldDescriptor::LABEL_REPEATED &&
       descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
     PyObject* sub_message = cmessage::InternalGetSubMessage(
-        self->parent, cdescriptor);
+        self->parent, descriptor);
     if (sub_message == NULL) {
       return NULL;
     }
@@ -152,33 +135,21 @@ PyObject* subscript(ExtensionDict* self, PyObject* key) {
 
   if (descriptor->label() == FieldDescriptor::LABEL_REPEATED) {
     if (descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
-      // COPIED
-      PyObject* py_container = PyObject_CallObject(
-          reinterpret_cast<PyObject*>(&RepeatedCompositeContainer_Type),
-          NULL);
+      PyObject *message_class = cdescriptor_pool::GetMessageClass(
+          GetDescriptorPool(), descriptor->message_type());
+      if (message_class == NULL) {
+        return NULL;
+      }
+      PyObject* py_container = repeated_composite_container::NewContainer(
+          self->parent, descriptor, message_class);
       if (py_container == NULL) {
         return NULL;
       }
-      RepeatedCompositeContainer* container =
-          reinterpret_cast<RepeatedCompositeContainer*>(py_container);
-      PyObject* field = cdescriptor->descriptor_field;
-      PyObject* message_type = PyObject_GetAttrString(field, "message_type");
-      PyObject* concrete_class = PyObject_GetAttrString(message_type,
-                                                        "_concrete_class");
-      container->owner = self->owner;
-      container->parent = self->parent;
-      container->message = self->parent->message;
-      container->parent_field = cdescriptor;
-      container->subclass_init = concrete_class;
-      Py_DECREF(message_type);
       PyDict_SetItem(self->values, key, py_container);
       return py_container;
     } else {
-      // COPIED
-      ScopedPyObjectPtr init_args(PyTuple_Pack(2, self->parent, cdescriptor));
-      PyObject* py_container = PyObject_CallObject(
-          reinterpret_cast<PyObject*>(&RepeatedScalarContainer_Type),
-          init_args);
+      PyObject* py_container = repeated_scalar_container::NewContainer(
+          self->parent, descriptor);
       if (py_container == NULL) {
         return NULL;
       }
@@ -191,13 +162,15 @@ PyObject* subscript(ExtensionDict* self, PyObject* key) {
 }
 
 int ass_subscript(ExtensionDict* self, PyObject* key, PyObject* value) {
-  CFieldDescriptor* cdescriptor = InternalGetCDescriptorFromExtension(
-      key);
-  if (cdescriptor == NULL) {
+  const google::protobuf::FieldDescriptor* descriptor =
+      cmessage::GetExtensionDescriptor(key);
+  if (descriptor == NULL) {
+    return -1;
+  }
+  if (!CheckFieldBelongsToMessage(descriptor, self->parent->message)) {
     return -1;
   }
-  ScopedPyObjectPtr py_cdescriptor(reinterpret_cast<PyObject*>(cdescriptor));
-  const google::protobuf::FieldDescriptor* descriptor = cdescriptor->descriptor;
+
   if (descriptor->label() != FieldDescriptor::LABEL_OPTIONAL ||
       descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
     PyErr_SetString(PyExc_TypeError, "Extension is repeated and/or composite "
@@ -214,20 +187,18 @@ int ass_subscript(ExtensionDict* self, PyObject* key, PyObject* value) {
 }
 
 PyObject* ClearExtension(ExtensionDict* self, PyObject* extension) {
-  CFieldDescriptor* cdescriptor = InternalGetCDescriptorFromExtension(
-      extension);
-  if (cdescriptor == NULL) {
+  const google::protobuf::FieldDescriptor* descriptor =
+      cmessage::GetExtensionDescriptor(extension);
+  if (descriptor == NULL) {
     return NULL;
   }
-  ScopedPyObjectPtr py_cdescriptor(reinterpret_cast<PyObject*>(cdescriptor));
   PyObject* value = PyDict_GetItem(self->values, extension);
   if (value != NULL) {
-    if (ReleaseExtension(self, value, cdescriptor->descriptor) < 0) {
+    if (ReleaseExtension(self, value, descriptor) < 0) {
       return NULL;
     }
   }
-  if (cmessage::ClearFieldByDescriptor(self->parent,
-                                       cdescriptor->descriptor) == NULL) {
+  if (cmessage::ClearFieldByDescriptor(self->parent, descriptor) == NULL) {
     return NULL;
   }
   if (PyDict_DelItem(self->values, extension) < 0) {
@@ -237,14 +208,12 @@ PyObject* ClearExtension(ExtensionDict* self, PyObject* extension) {
 }
 
 PyObject* HasExtension(ExtensionDict* self, PyObject* extension) {
-  CFieldDescriptor* cdescriptor = InternalGetCDescriptorFromExtension(
-      extension);
-  if (cdescriptor == NULL) {
+  const google::protobuf::FieldDescriptor* descriptor =
+      cmessage::GetExtensionDescriptor(extension);
+  if (descriptor == NULL) {
     return NULL;
   }
-  ScopedPyObjectPtr py_cdescriptor(reinterpret_cast<PyObject*>(cdescriptor));
-  PyObject* result = cmessage::HasFieldByDescriptor(
-      self->parent, cdescriptor->descriptor);
+  PyObject* result = cmessage::HasFieldByDescriptor(self->parent, descriptor);
   return result;
 }
 
@@ -263,11 +232,18 @@ PyObject* _FindExtensionByName(ExtensionDict* self, PyObject* name) {
   }
 }
 
-int init(ExtensionDict* self, PyObject* args, PyObject* kwargs) {
-  self->parent = NULL;
-  self->message = NULL;
+ExtensionDict* NewExtensionDict(CMessage *parent) {
+  ExtensionDict* self = reinterpret_cast<ExtensionDict*>(
+      PyType_GenericAlloc(&ExtensionDict_Type, 0));
+  if (self == NULL) {
+    return NULL;
+  }
+
+  self->parent = parent;  // Store a borrowed reference.
+  self->message = parent->message;
+  self->owner = parent->owner;
   self->values = PyDict_New();
-  return 0;
+  return self;
 }
 
 void dealloc(ExtensionDict* self) {
@@ -330,7 +306,7 @@ PyTypeObject ExtensionDict_Type = {
   0,                                   //  tp_descr_get
   0,                                   //  tp_descr_set
   0,                                   //  tp_dictoffset
-  (initproc)extension_dict::init,      //  tp_init
+  0,                                   //  tp_init
 };
 
 }  // namespace python

python/google/protobuf/pyext/extension_dict.h

@@ -53,13 +53,26 @@ using internal::shared_ptr;
 namespace python {
 
 struct CMessage;
-struct CFieldDescriptor;
 
 typedef struct ExtensionDict {
   PyObject_HEAD;
+
+  // This is the top-level C++ Message object that owns the whole
+  // proto tree.  Every Python container class holds a
+  // reference to it in order to keep it alive as long as there's a
+  // Python object that references any part of the tree.
   shared_ptr<Message> owner;
+
+  // Weak reference to parent message. Used to make sure
+  // the parent is writable when an extension field is modified.
   CMessage* parent;
+
+  // Pointer to the C++ Message that this ExtensionDict extends.
+  // Not owned by us.
   Message* message;
+
+  // A dict of child messages, indexed by Extension descriptors.
+  // Similar to CMessage::composite_fields.
   PyObject* values;
 } ExtensionDict;
 
@@ -67,11 +80,8 @@ extern PyTypeObject ExtensionDict_Type;
 
 namespace extension_dict {
 
-// Gets the _cdescriptor reference to a CFieldDescriptor object given a
-// python descriptor object.
-//
-// Returns a new reference.
-CFieldDescriptor* InternalGetCDescriptorFromExtension(PyObject* extension);
+// Builds an Extensions dict for a specific message.
+ExtensionDict* NewExtensionDict(CMessage *parent);
 
 // Gets the number of extension values in this ExtensionDict as a python object.
 //

python/google/protobuf/pyext/message.cc

@@ -71,7 +71,7 @@
     #error "Python 3.0 - 3.2 are not supported."
   #else
   #define PyString_AsString(ob) \
-    (PyUnicode_Check(ob)? PyUnicode_AsUTF8(ob): PyBytes_AS_STRING(ob))
+    (PyUnicode_Check(ob)? PyUnicode_AsUTF8(ob): PyBytes_AsString(ob))
   #endif
 #endif
 
@@ -81,7 +81,9 @@ namespace python {
 
 // Forward declarations
 namespace cmessage {
-static PyObject* GetDescriptor(CMessage* self, PyObject* name);
+static const google::protobuf::FieldDescriptor* GetFieldDescriptor(
+    CMessage* self, PyObject* name);
+static const google::protobuf::Descriptor* GetMessageDescriptor(PyTypeObject* cls);
 static string GetMessageName(CMessage* self);
 int InternalReleaseFieldByDescriptor(
     const google::protobuf::FieldDescriptor* field_descriptor,
@@ -147,12 +149,15 @@ int ForEachCompositeField(CMessage* self, Visitor visitor) {
   PyObject* key;
   PyObject* field;
 
+  // Never use self->message in this function, it may be already freed.
+  const google::protobuf::Descriptor* message_descriptor =
+      cmessage::GetMessageDescriptor(Py_TYPE(self));
+
   // Visit normal fields.
   while (PyDict_Next(self->composite_fields, &pos, &key, &field)) {
-    PyObject* cdescriptor = cmessage::GetDescriptor(self, key);
-    if (cdescriptor != NULL) {
-      const google::protobuf::FieldDescriptor* descriptor =
-          reinterpret_cast<CFieldDescriptor*>(cdescriptor)->descriptor;
+    const google::protobuf::FieldDescriptor* descriptor =
+      message_descriptor->FindFieldByName(PyString_AsString(key));
+    if (descriptor != NULL) {
       if (VisitCompositeField(descriptor, field, visitor) == -1)
         return -1;
     }
@@ -161,11 +166,11 @@ int ForEachCompositeField(CMessage* self, Visitor visitor) {
   // Visit extension fields.
   if (self->extensions != NULL) {
     while (PyDict_Next(self->extensions->values, &pos, &key, &field)) {
-      CFieldDescriptor* cdescriptor =
-          extension_dict::InternalGetCDescriptorFromExtension(key);
-      if (cdescriptor == NULL)
+      const google::protobuf::FieldDescriptor* descriptor =
+          cmessage::GetExtensionDescriptor(key);
+      if (descriptor == NULL)
         return -1;
-      if (VisitCompositeField(cdescriptor->descriptor, field, visitor) == -1)
+      if (VisitCompositeField(descriptor, field, visitor) == -1)
         return -1;
     }
   }
@@ -191,18 +196,19 @@ PyObject* PickleError_class;
 
 // Constant PyString values used for GetAttr/GetItem.
 static PyObject* kDESCRIPTOR;
-static PyObject* k__descriptors;
+static PyObject* k_cdescriptor;
 static PyObject* kfull_name;
 static PyObject* kname;
-static PyObject* kmessage_type;
-static PyObject* kis_extendable;
 static PyObject* kextensions_by_name;
 static PyObject* k_extensions_by_name;
 static PyObject* k_extensions_by_number;
-static PyObject* k_concrete_class;
 static PyObject* kfields_by_name;
 
-static CDescriptorPool* descriptor_pool;
+static PyDescriptorPool* descriptor_pool;
+
+PyDescriptorPool* GetDescriptorPool() {
+  return descriptor_pool;
+}
 
 /* Is 64bit */
 void FormatTypeError(PyObject* arg, char* expected_types) {
@@ -313,12 +319,12 @@ bool CheckAndSetString(
     }
 
     if (PyBytes_Check(arg)) {
-      PyObject* unicode = PyUnicode_FromEncodedObject(arg, "ascii", NULL);
+      PyObject* unicode = PyUnicode_FromEncodedObject(arg, "utf-8", NULL);
       if (unicode == NULL) {
         PyObject* repr = PyObject_Repr(arg);
         PyErr_Format(PyExc_ValueError,
-                     "%s has type str, but isn't in 7-bit ASCII "
-                     "encoding. Non-ASCII strings must be converted to "
+                     "%s has type str, but isn't valid UTF-8 "
+                     "encoding. Non-UTF-8 strings must be converted to "
                      "unicode objects before being added.",
                      PyString_AsString(repr));
         Py_DECREF(repr);
@@ -335,12 +341,9 @@ bool CheckAndSetString(
   PyObject* encoded_string = NULL;
   if (descriptor->type() == google::protobuf::FieldDescriptor::TYPE_STRING) {
     if (PyBytes_Check(arg)) {
-#if PY_MAJOR_VERSION < 3
-      encoded_string = PyString_AsEncodedObject(arg, "utf-8", NULL);
-#else
+      // The bytes were already validated as correctly encoded UTF-8 above.
       encoded_string = arg;  // Already encoded.
       Py_INCREF(encoded_string);
-#endif
     } else {
       encoded_string = PyUnicode_AsEncodedObject(arg, "utf-8", NULL);
     }
@@ -391,6 +394,17 @@ PyObject* ToStringObject(
   return result;
 }
 
+bool CheckFieldBelongsToMessage(const google::protobuf::FieldDescriptor* field_descriptor,
+                                const google::protobuf::Message* message) {
+  if (message->GetDescriptor() == field_descriptor->containing_type()) {
+    return true;
+  }
+  PyErr_Format(PyExc_KeyError, "Field '%s' does not belong to message '%s'",
+               field_descriptor->full_name().c_str(),
+               message->GetDescriptor()->full_name().c_str());
+  return false;
+}
+
 google::protobuf::DynamicMessageFactory* global_message_factory;
 
 namespace cmessage {
@@ -489,7 +503,7 @@ int AssureWritable(CMessage* self) {
     google::protobuf::Message* parent_message = self->parent->message;
     google::protobuf::Message* mutable_message = GetMutableMessage(
         self->parent,
-        self->parent_field->descriptor);
+        self->parent_field_descriptor);
     if (mutable_message == NULL) {
       return -1;
     }
@@ -512,26 +526,61 @@ int AssureWritable(CMessage* self) {
 
 // --- Globals:
 
-static PyObject* GetDescriptor(CMessage* self, PyObject* name) {
-  PyObject* descriptors =
-      PyDict_GetItem(Py_TYPE(self)->tp_dict, k__descriptors);
-  if (descriptors == NULL) {
-    PyErr_SetString(PyExc_TypeError, "No __descriptors");
+// Retrieve the C++ Descriptor of a message class.
+// On error, returns NULL with an exception set.
+static const google::protobuf::Descriptor* GetMessageDescriptor(PyTypeObject* cls) {
+  ScopedPyObjectPtr descriptor(PyObject_GetAttr(
+      reinterpret_cast<PyObject*>(cls), kDESCRIPTOR));
+  if (descriptor == NULL) {
+    PyErr_SetString(PyExc_TypeError, "Message class has no DESCRIPTOR");
+    return NULL;
+  }
+  ScopedPyObjectPtr cdescriptor(PyObject_GetAttr(descriptor, k_cdescriptor));
+  if (cdescriptor == NULL) {
+    PyErr_SetString(PyExc_TypeError, "Unregistered message.");
+    return NULL;
+  }
+  if (!PyObject_TypeCheck(cdescriptor, &CMessageDescriptor_Type)) {
+    PyErr_SetString(PyExc_TypeError, "Not a CMessageDescriptor");
     return NULL;
   }
+  return reinterpret_cast<CMessageDescriptor*>(cdescriptor.get())->descriptor;
+}
 
-  return PyDict_GetItem(descriptors, name);
+// Retrieve a C++ FieldDescriptor for a message attribute.
+// The C++ message must be valid.
+// TODO(amauryfa): This function should stay internal, because exception
+// handling is not consistent.
+static const google::protobuf::FieldDescriptor* GetFieldDescriptor(
+    CMessage* self, PyObject* name) {
+  const google::protobuf::Descriptor *message_descriptor = self->message->GetDescriptor();
+  const char* field_name = PyString_AsString(name);
+  if (field_name == NULL) {
+    return NULL;
+  }
+  const google::protobuf::FieldDescriptor *field_descriptor =
+      message_descriptor->FindFieldByName(field_name);
+  if (field_descriptor == NULL) {
+    // Note: No exception is set!
+    return NULL;
+  }
+  return field_descriptor;
 }
 
-static const google::protobuf::Message* CreateMessage(const char* message_type) {
-  string message_name(message_type);
-  const google::protobuf::Descriptor* descriptor =
-      GetDescriptorPool()->FindMessageTypeByName(message_name);
-  if (descriptor == NULL) {
-    PyErr_SetString(PyExc_TypeError, message_type);
+// Retrieve a C++ FieldDescriptor for an extension handle.
+const google::protobuf::FieldDescriptor* GetExtensionDescriptor(PyObject* extension) {
+  ScopedPyObjectPtr cdescriptor(
+      PyObject_GetAttrString(extension, "_cdescriptor"));
+  if (cdescriptor == NULL) {
+    PyErr_SetString(PyExc_KeyError, "Unregistered extension.");
+    return NULL;
+  }
+  if (!PyObject_TypeCheck(cdescriptor, &CFieldDescriptor_Type)) {
+    PyErr_SetString(PyExc_TypeError, "Not a CFieldDescriptor");
+    Py_DECREF(cdescriptor);
     return NULL;
   }
-  return global_message_factory->GetPrototype(descriptor);
+  return reinterpret_cast<CFieldDescriptor*>(cdescriptor.get())->descriptor;
 }
 
 // If cmessage_list is not NULL, this function releases values into the
@@ -627,39 +676,8 @@ int InternalDeleteRepeatedField(
   return 0;
 }
 
-int InitAttributes(CMessage* self, PyObject* arg, PyObject* kwargs) {
-  ScopedPyObjectPtr descriptor;
-  if (arg == NULL) {
-    descriptor.reset(
-        PyObject_GetAttr(reinterpret_cast<PyObject*>(self), kDESCRIPTOR));
-    if (descriptor == NULL) {
-      return NULL;
-    }
-  } else {
-    descriptor.reset(arg);
-    descriptor.inc();
-  }
-  ScopedPyObjectPtr is_extendable(PyObject_GetAttr(descriptor, kis_extendable));
-  if (is_extendable == NULL) {
-    return NULL;
-  }
-  int retcode = PyObject_IsTrue(is_extendable);
-  if (retcode == -1) {
-    return NULL;
-  }
-  if (retcode) {
-    PyObject* py_extension_dict = PyObject_CallObject(
-        reinterpret_cast<PyObject*>(&ExtensionDict_Type), NULL);
-    if (py_extension_dict == NULL) {
-      return NULL;
-    }
-    ExtensionDict* extension_dict = reinterpret_cast<ExtensionDict*>(
-        py_extension_dict);
-    extension_dict->parent = self;
-    extension_dict->message = self->message;
-    self->extensions = extension_dict;
-  }
-
+// Initializes fields of a message. Used in constructors.
+int InitAttributes(CMessage* self, PyObject* kwargs) {
   if (kwargs == NULL) {
     return 0;
   }
@@ -672,14 +690,12 @@ int InitAttributes(CMessage* self, PyObject* arg, PyObject* kwargs) {
       PyErr_SetString(PyExc_ValueError, "Field name must be a string");
       return -1;
     }
-    PyObject* py_cdescriptor = GetDescriptor(self, name);
-    if (py_cdescriptor == NULL) {
+    const google::protobuf::FieldDescriptor* descriptor = GetFieldDescriptor(self, name);
+    if (descriptor == NULL) {
       PyErr_Format(PyExc_ValueError, "Protocol message has no \"%s\" field.",
                    PyString_AsString(name));
       return -1;
     }
-    const google::protobuf::FieldDescriptor* descriptor =
-        reinterpret_cast<CFieldDescriptor*>(py_cdescriptor)->descriptor;
     if (descriptor->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED) {
       ScopedPyObjectPtr container(GetAttr(self, name));
       if (container == NULL) {
@@ -719,15 +735,19 @@ int InitAttributes(CMessage* self, PyObject* arg, PyObject* kwargs) {
   return 0;
 }
 
-static PyObject* New(PyTypeObject* type, PyObject* args, PyObject* kwargs) {
-  CMessage* self = reinterpret_cast<CMessage*>(type->tp_alloc(type, 0));
+// Allocates an incomplete Python Message: the caller must fill self->message,
+// self->owner and eventually self->parent.
+CMessage* NewEmptyMessage(PyObject* type,
+                          const google::protobuf::Descriptor *descriptor) {
+  CMessage* self = reinterpret_cast<CMessage*>(
+      PyType_GenericAlloc(reinterpret_cast<PyTypeObject*>(type), 0));
   if (self == NULL) {
     return NULL;
   }
 
   self->message = NULL;
   self->parent = NULL;
-  self->parent_field = NULL;
+  self->parent_field_descriptor = NULL;
   self->read_only = false;
   self->extensions = NULL;
 
@@ -735,43 +755,58 @@ static PyObject* New(PyTypeObject* type, PyObject* args, PyObject* kwargs) {
   if (self->composite_fields == NULL) {
     return NULL;
   }
-  return reinterpret_cast<PyObject*>(self);
-}
-
-PyObject* NewEmpty(PyObject* type) {
-  return New(reinterpret_cast<PyTypeObject*>(type), NULL, NULL);
-}
 
-static int Init(CMessage* self, PyObject* args, PyObject* kwargs) {
-  if (kwargs == NULL) {
-    // TODO(anuraag): Set error
-    return -1;
+  // If there are extension_ranges, the message is "extendable". Allocate a
+  // dictionary to store the extension fields.
+  if (descriptor->extension_range_count() > 0) {
+    // TODO(amauryfa): Delay the construction of this dict until extensions are
+    // really used on the object.
+    ExtensionDict* extension_dict = extension_dict::NewExtensionDict(self);
+    if (extension_dict == NULL) {
+      return NULL;
+    }
+    self->extensions = extension_dict;
   }
 
-  PyObject* descriptor = PyTuple_GetItem(args, 0);
-  if (descriptor == NULL || PyTuple_Size(args) != 1) {
-    PyErr_SetString(PyExc_ValueError, "args must contain one arg: descriptor");
-    return -1;
-  }
+  return self;
+}
 
-  ScopedPyObjectPtr py_message_type(PyObject_GetAttr(descriptor, kfull_name));
-  if (py_message_type == NULL) {
-    return -1;
+// The __new__ method of Message classes.
+// Creates a new C++ message and takes ownership.
+static PyObject* New(PyTypeObject* type,
+                     PyObject* unused_args, PyObject* unused_kwargs) {
+  // Retrieve the message descriptor and the default instance (=prototype).
+  const google::protobuf::Descriptor* message_descriptor = GetMessageDescriptor(type);
+  if (message_descriptor == NULL) {
+    return NULL;
   }
-
-  const char* message_type = PyString_AsString(py_message_type.get());
-  const google::protobuf::Message* message = CreateMessage(message_type);
-  if (message == NULL) {
-    return -1;
+  const google::protobuf::Message* default_message =
+      global_message_factory->GetPrototype(message_descriptor);
+  if (default_message == NULL) {
+    PyErr_SetString(PyExc_TypeError, message_descriptor->full_name().c_str());
+    return NULL;
   }
 
-  self->message = message->New();
+  CMessage* self = NewEmptyMessage(reinterpret_cast<PyObject*>(type),
+                                   message_descriptor);
+  if (self == NULL) {
+    return NULL;
+  }
+  self->message = default_message->New();
   self->owner.reset(self->message);
 
-  if (InitAttributes(self, descriptor, kwargs) < 0) {
+  return reinterpret_cast<PyObject*>(self);
+}
+
+// The __init__ method of Message classes.
+// It initializes fields from keywords passed to the constructor.
+static int Init(CMessage* self, PyObject* args, PyObject* kwargs) {
+  if (PyTuple_Size(args) != 0) {
+    PyErr_SetString(PyExc_TypeError, "No positional arguments allowed");
     return -1;
   }
-  return 0;
+
+  return InitAttributes(self, kwargs);
 }
 
 // ---------------------------------------------------------------------
@@ -853,9 +888,7 @@ PyObject* IsInitialized(CMessage* self, PyObject* args) {
 PyObject* HasFieldByDescriptor(
     CMessage* self, const google::protobuf::FieldDescriptor* field_descriptor) {
   google::protobuf::Message* message = self->message;
-  if (!FIELD_BELONGS_TO_MESSAGE(field_descriptor, message)) {
-    PyErr_SetString(PyExc_KeyError,
-                    "Field does not belong to message!");
+  if (!CheckFieldBelongsToMessage(field_descriptor, message)) {
     return NULL;
   }
   if (field_descriptor->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED) {
@@ -1048,7 +1081,7 @@ int ReleaseSubMessage(google::protobuf::Message* message,
   child_cmessage->message = released_message.get();
   child_cmessage->owner.swap(released_message);
   child_cmessage->parent = NULL;
-  child_cmessage->parent_field = NULL;
+  child_cmessage->parent_field_descriptor = NULL;
   child_cmessage->read_only = false;
   return ForEachCompositeField(child_cmessage,
                                SetOwnerVisitor(child_cmessage->owner));
@@ -1090,10 +1123,8 @@ int InternalReleaseFieldByDescriptor(
 
 int InternalReleaseField(CMessage* self, PyObject* composite_field,
                          PyObject* name) {
-  PyObject* cdescriptor = GetDescriptor(self, name);
-  if (cdescriptor != NULL) {
-    const google::protobuf::FieldDescriptor* descriptor =
-        reinterpret_cast<CFieldDescriptor*>(cdescriptor)->descriptor;
+  const google::protobuf::FieldDescriptor* descriptor = GetFieldDescriptor(self, name);
+  if (descriptor != NULL) {
     return InternalReleaseFieldByDescriptor(
         descriptor, composite_field, self->message);
   }
@@ -1104,9 +1135,7 @@ int InternalReleaseField(CMessage* self, PyObject* composite_field,
 PyObject* ClearFieldByDescriptor(
     CMessage* self,
     const google::protobuf::FieldDescriptor* descriptor) {
-  if (!FIELD_BELONGS_TO_MESSAGE(descriptor, self->message)) {
-    PyErr_SetString(PyExc_KeyError,
-                    "Field does not belong to message!");
+  if (!CheckFieldBelongsToMessage(descriptor, self->message)) {
     return NULL;
   }
   AssureWritable(self);
@@ -1177,15 +1206,10 @@ PyObject* Clear(CMessage* self) {
   // fields have been released.
   if (self->extensions != NULL) {
     Py_CLEAR(self->extensions);
-    PyObject* py_extension_dict = PyObject_CallObject(
-        reinterpret_cast<PyObject*>(&ExtensionDict_Type), NULL);
-    if (py_extension_dict == NULL) {
+    ExtensionDict* extension_dict = extension_dict::NewExtensionDict(self);
+    if (extension_dict == NULL) {
       return NULL;
     }
-    ExtensionDict* extension_dict = reinterpret_cast<ExtensionDict*>(
-        py_extension_dict);
-    extension_dict->parent = self;
-    extension_dict->message = self->message;
     self->extensions = extension_dict;
   }
   PyDict_Clear(self->composite_fields);
@@ -1196,8 +1220,8 @@ PyObject* Clear(CMessage* self) {
 // ---------------------------------------------------------------------
 
 static string GetMessageName(CMessage* self) {
-  if (self->parent_field != NULL) {
-    return self->parent_field->descriptor->full_name();
+  if (self->parent_field_descriptor != NULL) {
+    return self->parent_field_descriptor->full_name();
   } else {
     return self->message->GetDescriptor()->full_name();
   }
@@ -1219,7 +1243,7 @@ static PyObject* SerializeToString(CMessage* self, PyObject* args) {
       return NULL;
     }
     PyErr_Format(EncodeError_class, "Message %s is missing required fields: %s",
-                 GetMessageName(self).c_str(), PyString_AsString(joined.get()));
+                 GetMessageName(self).c_str(), PyString_AsString(joined));
     return NULL;
   }
   int size = self->message->ByteSize();
@@ -1361,7 +1385,7 @@ static PyObject* MergeFromString(CMessage* self, PyObject* arg) {
   AssureWritable(self);
   google::protobuf::io::CodedInputStream input(
       reinterpret_cast<const uint8*>(data), data_length);
-  input.SetExtensionRegistry(GetDescriptorPool(), global_message_factory);
+  input.SetExtensionRegistry(descriptor_pool->pool, global_message_factory);
   bool success = self->message->MergePartialFromCodedStream(&input);
   if (success) {
     return PyInt_FromLong(input.CurrentPosition());
@@ -1421,15 +1445,11 @@ static PyObject* RegisterExtension(PyObject* cls,
     return NULL;
   }
 
-  CFieldDescriptor* cdescriptor =
-      extension_dict::InternalGetCDescriptorFromExtension(extension_handle);
-  ScopedPyObjectPtr py_cdescriptor(reinterpret_cast<PyObject*>(cdescriptor));
-  if (cdescriptor == NULL) {
+  const google::protobuf::FieldDescriptor* descriptor =
+      GetExtensionDescriptor(extension_handle);
+  if (descriptor == NULL) {
     return NULL;
   }
-  Py_INCREF(extension_handle);
-  cdescriptor->descriptor_field = extension_handle;
-  const google::protobuf::FieldDescriptor* descriptor = cdescriptor->descriptor;
   // Check if it's a message set
   if (descriptor->is_extension() &&
       descriptor->containing_type()->options().message_set_wire_format() &&
@@ -1608,8 +1628,14 @@ static PyObject* RichCompare(CMessage* self, PyObject* other, int opid) {
   }
   if (opid == Py_EQ || opid == Py_NE) {
     ScopedPyObjectPtr self_fields(ListFields(self));
+    if (!self_fields) {
+      return NULL;
+    }
     ScopedPyObjectPtr other_fields(ListFields(
         reinterpret_cast<CMessage*>(other)));
+    if (!other_fields) {
+      return NULL;
+    }
     return PyObject_RichCompare(self_fields, other_fields, opid);
   } else {
     Py_INCREF(Py_NotImplemented);
@@ -1623,9 +1649,7 @@ PyObject* InternalGetScalar(
   google::protobuf::Message* message = self->message;
   const google::protobuf::Reflection* reflection = message->GetReflection();
 
-  if (!FIELD_BELONGS_TO_MESSAGE(field_descriptor, message)) {
-    PyErr_SetString(
-        PyExc_KeyError, "Field does not belong to message!");
+  if (!CheckFieldBelongsToMessage(field_descriptor, message)) {
     return NULL;
   }
 
@@ -1701,43 +1725,31 @@ PyObject* InternalGetScalar(
   return result;
 }
 
-PyObject* InternalGetSubMessage(CMessage* self,
-                                CFieldDescriptor* cfield_descriptor) {
-  PyObject* field = cfield_descriptor->descriptor_field;
-  ScopedPyObjectPtr message_type(PyObject_GetAttr(field, kmessage_type));
-  if (message_type == NULL) {
-    return NULL;
-  }
-  ScopedPyObjectPtr concrete_class(
-      PyObject_GetAttr(message_type, k_concrete_class));
-  if (concrete_class == NULL) {
+PyObject* InternalGetSubMessage(
+    CMessage* self, const google::protobuf::FieldDescriptor* field_descriptor) {
+  const google::protobuf::Reflection* reflection = self->message->GetReflection();
+  const google::protobuf::Message& sub_message = reflection->GetMessage(
+      *self->message, field_descriptor, global_message_factory);
+
+  PyObject *message_class = cdescriptor_pool::GetMessageClass(
+      descriptor_pool, field_descriptor->message_type());
+  if (message_class == NULL) {
     return NULL;
   }
-  PyObject* py_cmsg = cmessage::NewEmpty(concrete_class);
-  if (py_cmsg == NULL) {
+
+  CMessage* cmsg = cmessage::NewEmptyMessage(message_class,
+                                             sub_message.GetDescriptor());
+  if (cmsg == NULL) {
     return NULL;
   }
-  if (!PyObject_TypeCheck(py_cmsg, &CMessage_Type)) {
-    PyErr_SetString(PyExc_TypeError, "Not a CMessage!");
-  }
-  CMessage* cmsg = reinterpret_cast<CMessage*>(py_cmsg);
 
-  const google::protobuf::FieldDescriptor* field_descriptor =
-      cfield_descriptor->descriptor;
-  const google::protobuf::Reflection* reflection = self->message->GetReflection();
-  const google::protobuf::Message& sub_message = reflection->GetMessage(
-      *self->message, field_descriptor, global_message_factory);
   cmsg->owner = self->owner;
   cmsg->parent = self;
-  cmsg->parent_field = cfield_descriptor;
+  cmsg->parent_field_descriptor = field_descriptor;
   cmsg->read_only = !reflection->HasField(*self->message, field_descriptor);
   cmsg->message = const_cast<google::protobuf::Message*>(&sub_message);
 
-  if (InitAttributes(cmsg, NULL, NULL) < 0) {
-    Py_DECREF(py_cmsg);
-    return NULL;
-  }
-  return py_cmsg;
+  return reinterpret_cast<PyObject*>(cmsg);
 }
 
 int InternalSetScalar(
@@ -1747,9 +1759,7 @@ int InternalSetScalar(
   google::protobuf::Message* message = self->message;
   const google::protobuf::Reflection* reflection = message->GetReflection();
 
-  if (!FIELD_BELONGS_TO_MESSAGE(field_descriptor, message)) {
-    PyErr_SetString(
-        PyExc_KeyError, "Field does not belong to message!");
+  if (!CheckFieldBelongsToMessage(field_descriptor, message)) {
     return -1;
   }
 
@@ -1838,25 +1848,35 @@ PyObject* FromString(PyTypeObject* cls, PyObject* serialized) {
     return NULL;
   }
 
-  if (InitAttributes(cmsg, NULL, NULL) < 0) {
-    Py_DECREF(py_cmsg);
-    return NULL;
-  }
   return py_cmsg;
 }
 
+
+// Finalize the creation of the Message class.
+// Called from its metaclass: GeneratedProtocolMessageType.__init__().
 static PyObject* AddDescriptors(PyTypeObject* cls,
                                 PyObject* descriptor) {
-  if (PyObject_SetAttr(reinterpret_cast<PyObject*>(cls),
-                       k_extensions_by_name, PyDict_New()) < 0) {
-    return NULL;
-  }
-  if (PyObject_SetAttr(reinterpret_cast<PyObject*>(cls),
-                       k_extensions_by_number, PyDict_New()) < 0) {
+  const google::protobuf::Descriptor* message_descriptor =
+      cdescriptor_pool::RegisterMessageClass(
+          descriptor_pool, reinterpret_cast<PyObject*>(cls), descriptor);
+  if (message_descriptor == NULL) {
     return NULL;
   }
 
-  ScopedPyObjectPtr field_descriptors(PyDict_New());
+  // If there are extension_ranges, the message is "extendable", and extension
+  // classes will register themselves in this class.
+  if (message_descriptor->extension_range_count() > 0) {
+    ScopedPyObjectPtr by_name(PyDict_New());
+    if (PyObject_SetAttr(reinterpret_cast<PyObject*>(cls),
+                         k_extensions_by_name, by_name) < 0) {
+      return NULL;
+    }
+    ScopedPyObjectPtr by_number(PyDict_New());
+    if (PyObject_SetAttr(reinterpret_cast<PyObject*>(cls),
+                         k_extensions_by_number, by_number) < 0) {
+      return NULL;
+    }
+  }
 
   ScopedPyObjectPtr fields(PyObject_GetAttrString(descriptor, "fields"));
   if (fields == NULL) {
@@ -1878,19 +1898,14 @@ static PyObject* AddDescriptors(PyTypeObject* cls,
       return NULL;
     }
 
-    PyObject* field_descriptor =
-        cdescriptor_pool::FindFieldByName(descriptor_pool, full_field_name);
+    ScopedPyObjectPtr field_descriptor(
+        cdescriptor_pool::FindFieldByName(descriptor_pool, full_field_name));
     if (field_descriptor == NULL) {
       PyErr_SetString(PyExc_TypeError, "Couldn't find field");
       return NULL;
     }
-    Py_INCREF(field);
     CFieldDescriptor* cfield_descriptor = reinterpret_cast<CFieldDescriptor*>(
-        field_descriptor);
-    cfield_descriptor->descriptor_field = field;
-    if (PyDict_SetItem(field_descriptors, field_name, field_descriptor) < 0) {
-      return NULL;
-    }
+        field_descriptor.get());
 
     // The FieldDescriptor's name field might either be of type bytes or
     // of type unicode, depending on whether the FieldDescriptor was
@@ -1919,8 +1934,6 @@ static PyObject* AddDescriptors(PyTypeObject* cls,
     }
   }
 
-  PyDict_SetItem(cls->tp_dict, k__descriptors, field_descriptors);
-
   // Enum Values
   ScopedPyObjectPtr enum_types(PyObject_GetAttrString(descriptor,
                                                       "enum_types"));
@@ -1994,15 +2007,11 @@ static PyObject* AddDescriptors(PyTypeObject* cls,
                          extension_name, extension_field) == -1) {
       return NULL;
     }
-    ScopedPyObjectPtr py_cfield_descriptor(
-        PyObject_GetAttrString(extension_field, "_cdescriptor"));
-    if (py_cfield_descriptor == NULL) {
+    const google::protobuf::FieldDescriptor* field_descriptor =
+        GetExtensionDescriptor(extension_field);
+    if (field_descriptor == NULL) {
       return NULL;
     }
-    CFieldDescriptor* cfield_descriptor =
-        reinterpret_cast<CFieldDescriptor*>(py_cfield_descriptor.get());
-    Py_INCREF(extension_field);
-    cfield_descriptor->descriptor_field = extension_field;
 
     ScopedPyObjectPtr field_name_upcased(
         PyObject_CallMethod(extension_name, "upper", NULL));
@@ -2015,13 +2024,12 @@ static PyObject* AddDescriptors(PyTypeObject* cls,
       return NULL;
     }
     ScopedPyObjectPtr number(PyInt_FromLong(
-        cfield_descriptor->descriptor->number()));
+        field_descriptor->number()));
     if (number == NULL) {
       return NULL;
     }
     if (PyObject_SetAttr(reinterpret_cast<PyObject*>(cls),
-                         field_number_name, PyInt_FromLong(
-            cfield_descriptor->descriptor->number())) == -1) {
+                         field_number_name, number) == -1) {
       return NULL;
     }
   }
@@ -2039,10 +2047,6 @@ PyObject* DeepCopy(CMessage* self, PyObject* arg) {
     Py_DECREF(clone);
     return NULL;
   }
-  if (InitAttributes(reinterpret_cast<CMessage*>(clone), NULL, NULL) < 0) {
-    Py_DECREF(clone);
-    return NULL;
-  }
   if (MergeFrom(reinterpret_cast<CMessage*>(clone),
                 reinterpret_cast<PyObject*>(self)) == NULL) {
     Py_DECREF(clone);
@@ -2202,48 +2206,30 @@ PyObject* GetAttr(CMessage* self, PyObject* name) {
     return value;
   }
 
-  PyObject* descriptor = GetDescriptor(self, name);
-  if (descriptor != NULL) {
-    CFieldDescriptor* cdescriptor =
-        reinterpret_cast<CFieldDescriptor*>(descriptor);
-    const google::protobuf::FieldDescriptor* field_descriptor = cdescriptor->descriptor;
+  const google::protobuf::FieldDescriptor* field_descriptor = GetFieldDescriptor(
+      self, name);
+  if (field_descriptor != NULL) {
     if (field_descriptor->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED) {
       if (field_descriptor->cpp_type() ==
           google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE) {
-        PyObject* py_container = PyObject_CallObject(
-            reinterpret_cast<PyObject*>(&RepeatedCompositeContainer_Type),
-            NULL);
-        if (py_container == NULL) {
+        PyObject *message_class = cdescriptor_pool::GetMessageClass(
+            descriptor_pool, field_descriptor->message_type());
+        if (message_class == NULL) {
           return NULL;
         }
-        RepeatedCompositeContainer* container =
-            reinterpret_cast<RepeatedCompositeContainer*>(py_container);
-        PyObject* field = cdescriptor->descriptor_field;
-        PyObject* message_type = PyObject_GetAttr(field, kmessage_type);
-        if (message_type == NULL) {
-          return NULL;
-        }
-        PyObject* concrete_class =
-            PyObject_GetAttr(message_type, k_concrete_class);
-        if (concrete_class == NULL) {
+        PyObject* py_container = repeated_composite_container::NewContainer(
+            self, field_descriptor, message_class);
+        if (py_container == NULL) {
           return NULL;
         }
-        container->parent = self;
-        container->parent_field = cdescriptor;
-        container->message = self->message;
-        container->owner = self->owner;
-        container->subclass_init = concrete_class;
-        Py_DECREF(message_type);
         if (PyDict_SetItem(self->composite_fields, name, py_container) < 0) {
           Py_DECREF(py_container);
           return NULL;
         }
         return py_container;
       } else {
-        ScopedPyObjectPtr init_args(PyTuple_Pack(2, self, cdescriptor));
-        PyObject* py_container = PyObject_CallObject(
-            reinterpret_cast<PyObject*>(&RepeatedScalarContainer_Type),
-            init_args);
+        PyObject* py_container = repeated_scalar_container::NewContainer(
+            self, field_descriptor);
         if (py_container == NULL) {
           return NULL;
         }
@@ -2256,7 +2242,7 @@ PyObject* GetAttr(CMessage* self, PyObject* name) {
     } else {
       if (field_descriptor->cpp_type() ==
           google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE) {
-        PyObject* sub_message = InternalGetSubMessage(self, cdescriptor);
+        PyObject* sub_message = InternalGetSubMessage(self, field_descriptor);
         if (PyDict_SetItem(self->composite_fields, name, sub_message) < 0) {
           Py_DECREF(sub_message);
           return NULL;
@@ -2278,12 +2264,10 @@ int SetAttr(CMessage* self, PyObject* name, PyObject* value) {
     return -1;
   }
 
-  PyObject* descriptor = GetDescriptor(self, name);
-  if (descriptor != NULL) {
+  const google::protobuf::FieldDescriptor* field_descriptor =
+      GetFieldDescriptor(self, name);
+  if (field_descriptor != NULL) {
     AssureWritable(self);
-    CFieldDescriptor* cdescriptor =
-        reinterpret_cast<CFieldDescriptor*>(descriptor);
-    const google::protobuf::FieldDescriptor* field_descriptor = cdescriptor->descriptor;
     if (field_descriptor->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED) {
       PyErr_Format(PyExc_AttributeError, "Assignment not allowed to repeated "
                    "field \"%s\" in protocol message object.",
@@ -2401,22 +2385,18 @@ void InitGlobals() {
   kuint64max_py = PyLong_FromUnsignedLongLong(kuint64max);
 
   kDESCRIPTOR = PyString_FromString("DESCRIPTOR");
-  k__descriptors = PyString_FromString("__descriptors");
+  k_cdescriptor = PyString_FromString("_cdescriptor");
   kfull_name = PyString_FromString("full_name");
-  kis_extendable = PyString_FromString("is_extendable");
   kextensions_by_name = PyString_FromString("extensions_by_name");
   k_extensions_by_name = PyString_FromString("_extensions_by_name");
   k_extensions_by_number = PyString_FromString("_extensions_by_number");
-  k_concrete_class = PyString_FromString("_concrete_class");
-  kmessage_type = PyString_FromString("message_type");
   kname = PyString_FromString("name");
   kfields_by_name = PyString_FromString("fields_by_name");
 
-  global_message_factory = new DynamicMessageFactory(GetDescriptorPool());
-  global_message_factory->SetDelegateToGeneratedFactory(true);
+  descriptor_pool = cdescriptor_pool::NewDescriptorPool();
 
-  descriptor_pool = reinterpret_cast<google::protobuf::python::CDescriptorPool*>(
-      Python_NewCDescriptorPool(NULL, NULL));
+  global_message_factory = new DynamicMessageFactory(descriptor_pool->pool);
+  global_message_factory->SetDelegateToGeneratedFactory(true);
 }
 
 bool InitProto2MessageModule(PyObject *m) {
@@ -2427,19 +2407,32 @@ bool InitProto2MessageModule(PyObject *m) {
     return false;
   }
 
-  // All three of these are actually set elsewhere, directly onto the child
-  // protocol buffer message class, but set them here as well to document that
-  // subclasses need to set these.
+  // DESCRIPTOR is set on each protocol buffer message class elsewhere, but set
+  // it here as well to document that subclasses need to set it.
   PyDict_SetItem(google::protobuf::python::CMessage_Type.tp_dict, kDESCRIPTOR, Py_None);
-  PyDict_SetItem(google::protobuf::python::CMessage_Type.tp_dict,
-                 k_extensions_by_name, Py_None);
-  PyDict_SetItem(google::protobuf::python::CMessage_Type.tp_dict,
-                 k_extensions_by_number, Py_None);
+  // Subclasses with message extensions will override _extensions_by_name and
+  // _extensions_by_number with fresh mutable dictionaries in AddDescriptors.
+  // All other classes can share this same immutable mapping.
+  ScopedPyObjectPtr empty_dict(PyDict_New());
+  if (empty_dict == NULL) {
+    return false;
+  }
+  ScopedPyObjectPtr immutable_dict(PyDictProxy_New(empty_dict));
+  if (immutable_dict == NULL) {
+    return false;
+  }
+  if (PyDict_SetItem(google::protobuf::python::CMessage_Type.tp_dict,
+                     k_extensions_by_name, immutable_dict) < 0) {
+    return false;
+  }
+  if (PyDict_SetItem(google::protobuf::python::CMessage_Type.tp_dict,
+                     k_extensions_by_number, immutable_dict) < 0) {
+    return false;
+  }
 
   PyModule_AddObject(m, "Message", reinterpret_cast<PyObject*>(
       &google::protobuf::python::CMessage_Type));
 
-  google::protobuf::python::RepeatedScalarContainer_Type.tp_new = PyType_GenericNew;
   google::protobuf::python::RepeatedScalarContainer_Type.tp_hash =
       PyObject_HashNotImplemented;
   if (PyType_Ready(&google::protobuf::python::RepeatedScalarContainer_Type) < 0) {
@@ -2450,7 +2443,6 @@ bool InitProto2MessageModule(PyObject *m) {
                      reinterpret_cast<PyObject*>(
                          &google::protobuf::python::RepeatedScalarContainer_Type));
 
-  google::protobuf::python::RepeatedCompositeContainer_Type.tp_new = PyType_GenericNew;
   google::protobuf::python::RepeatedCompositeContainer_Type.tp_hash =
       PyObject_HashNotImplemented;
   if (PyType_Ready(&google::protobuf::python::RepeatedCompositeContainer_Type) < 0) {
@@ -2462,7 +2454,6 @@ bool InitProto2MessageModule(PyObject *m) {
       reinterpret_cast<PyObject*>(
           &google::protobuf::python::RepeatedCompositeContainer_Type));
 
-  google::protobuf::python::ExtensionDict_Type.tp_new = PyType_GenericNew;
   google::protobuf::python::ExtensionDict_Type.tp_hash = PyObject_HashNotImplemented;
   if (PyType_Ready(&google::protobuf::python::ExtensionDict_Type) < 0) {
     return false;

python/google/protobuf/pyext/message.h

@@ -49,12 +49,13 @@ namespace protobuf {
 class Message;
 class Reflection;
 class FieldDescriptor;
+class Descriptor;
 
 using internal::shared_ptr;
 
 namespace python {
 
-struct CFieldDescriptor;
+struct PyDescriptorPool;
 struct ExtensionDict;
 
 typedef struct CMessage {
@@ -79,13 +80,11 @@ typedef struct CMessage {
   // to use this pointer will result in a crash.
   struct CMessage* parent;
 
-  // Weak reference to the parent's descriptor that describes this submessage.
+  // Pointer to the parent's descriptor that describes this submessage.
   // Used together with the parent's message when making a default message
   // instance mutable.
-  // TODO(anuraag): With a bit of work on the Python/C++ layer, it should be
-  // possible to make this a direct pointer to a C++ FieldDescriptor, this would
-  // be easier if this implementation replaces upstream.
-  CFieldDescriptor* parent_field;
+  // The pointer is owned by the global DescriptorPool.
+  const google::protobuf::FieldDescriptor* parent_field_descriptor;
 
   // Pointer to the C++ Message object for this CMessage.  The
   // CMessage does not own this pointer.
@@ -113,8 +112,11 @@ extern PyTypeObject CMessage_Type;
 
 namespace cmessage {
 
-// Create a new empty message that can be populated by the parent.
-PyObject* NewEmpty(PyObject* type);
+// Internal function to create a new empty Message Python object, but with empty
+// pointers to the C++ objects.
+// The caller must fill self->message, self->owner and eventually self->parent.
+CMessage* NewEmptyMessage(PyObject* type,
+                          const google::protobuf::Descriptor* descriptor);
 
 // Release a submessage from its proto tree, making it a new top-level messgae.
 // A new message will be created if this is a read-only default instance.
@@ -124,12 +126,16 @@ int ReleaseSubMessage(google::protobuf::Message* message,
                       const google::protobuf::FieldDescriptor* field_descriptor,
                       CMessage* child_cmessage);
 
+// Retrieves the C++ descriptor of a Python Extension descriptor.
+// On error, return NULL with an exception set.
+const google::protobuf::FieldDescriptor* GetExtensionDescriptor(PyObject* extension);
+
 // Initializes a new CMessage instance for a submessage. Only called once per
 // submessage as the result is cached in composite_fields.
 //
 // Corresponds to reflection api method GetMessage.
-PyObject* InternalGetSubMessage(CMessage* self,
-                                CFieldDescriptor* cfield_descriptor);
+PyObject* InternalGetSubMessage(
+    CMessage* self, const google::protobuf::FieldDescriptor* field_descriptor);
 
 // Deletes a range of C++ submessages in a repeated field (following a
 // removal in a RepeatedCompositeContainer).
@@ -190,10 +196,8 @@ PyObject* HasFieldByDescriptor(
 // Corresponds to reflection api method HasField.
 PyObject* HasField(CMessage* self, PyObject* arg);
 
-// Initializes constants/enum values on a message. This is called by
-// RepeatedCompositeContainer and ExtensionDict after calling the constructor.
-// TODO(anuraag): Make it always called from within the constructor since it can
-int InitAttributes(CMessage* self, PyObject* descriptor, PyObject* kwargs);
+// Initializes values of fields on a newly constructed message.
+int InitAttributes(CMessage* self, PyObject* kwargs);
 
 PyObject* MergeFrom(CMessage* self, PyObject* arg);
 
@@ -218,12 +222,14 @@ int AssureWritable(CMessage* self);
 
 }  // namespace cmessage
 
+
+// Retrieve the global descriptor pool owned by the _message module.
+PyDescriptorPool* GetDescriptorPool();
+
+
 /* Is 64bit */
 #define IS_64BIT (SIZEOF_LONG == 8)
 
-#define FIELD_BELONGS_TO_MESSAGE(field_descriptor, message) \
-    ((message)->GetDescriptor() == (field_descriptor)->containing_type())
-
 #define FIELD_IS_REPEATED(field_descriptor)                 \
     ((field_descriptor)->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED)
 
@@ -296,6 +302,11 @@ bool CheckAndSetString(
 PyObject* ToStringObject(
     const google::protobuf::FieldDescriptor* descriptor, string value);
 
+// Check if the passed field descriptor belongs to the given message.
+// If not, return false and set a Python exception (a KeyError)
+bool CheckFieldBelongsToMessage(const google::protobuf::FieldDescriptor* field_descriptor,
+                                const google::protobuf::Message* message);
+
 extern PyObject* PickleError_class;
 
 }  // namespace python

python/google/protobuf/pyext/proto2_api_test.proto

@@ -28,6 +28,8 @@
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+syntax = "proto2";
+
 import "google/protobuf/internal/cpp/proto1_api_test.proto";
 
 package google.protobuf.python.internal;

python/google/protobuf/pyext/python.proto

@@ -33,6 +33,7 @@
 // These message definitions are used to exercises known corner cases
 // in the C++ implementation of the Python API.
 
+syntax = "proto2";
 
 package google.protobuf.python.internal;
 
@@ -63,4 +64,5 @@ message TestAllExtensions {
 
 extend TestAllExtensions {
   optional TestAllTypes.NestedMessage optional_nested_message_extension = 1;
+  repeated TestAllTypes.NestedMessage repeated_nested_message_extension = 2;
 }

python/google/protobuf/pyext/repeated_composite_container.cc

@@ -65,14 +65,14 @@ namespace repeated_composite_container {
 #define GOOGLE_CHECK_ATTACHED(self)             \
   do {                                   \
     GOOGLE_CHECK_NOTNULL((self)->message);      \
-    GOOGLE_CHECK_NOTNULL((self)->parent_field); \
+    GOOGLE_CHECK_NOTNULL((self)->parent_field_descriptor); \
   } while (0);
 
 #define GOOGLE_CHECK_RELEASED(self)             \
   do {                                   \
     GOOGLE_CHECK((self)->owner.get() == NULL);  \
     GOOGLE_CHECK((self)->message == NULL);      \
-    GOOGLE_CHECK((self)->parent_field == NULL); \
+    GOOGLE_CHECK((self)->parent_field_descriptor == NULL); \
     GOOGLE_CHECK((self)->parent == NULL);       \
   } while (0);
 
@@ -122,7 +122,7 @@ static int InternalQuickSort(RepeatedCompositeContainer* self,
 
   google::protobuf::Message* message = self->message;
   const google::protobuf::Reflection* reflection = message->GetReflection();
-  const google::protobuf::FieldDescriptor* descriptor = self->parent_field->descriptor;
+  const google::protobuf::FieldDescriptor* descriptor = self->parent_field_descriptor;
   Py_ssize_t left;
   Py_ssize_t right;
 
@@ -202,7 +202,7 @@ static Py_ssize_t Length(RepeatedCompositeContainer* self) {
   google::protobuf::Message* message = self->message;
   if (message != NULL) {
     return message->GetReflection()->FieldSize(*message,
-                                               self->parent_field->descriptor);
+                                               self->parent_field_descriptor);
   } else {
     // The container has been released (i.e. by a call to Clear() or
     // ClearField() on the parent) and thus there's no message.
@@ -225,19 +225,19 @@ static int UpdateChildMessages(RepeatedCompositeContainer* self) {
   const google::protobuf::Reflection* reflection = message->GetReflection();
   for (Py_ssize_t i = child_length; i < message_length; ++i) {
     const Message& sub_message = reflection->GetRepeatedMessage(
-        *(self->message), self->parent_field->descriptor, i);
-    ScopedPyObjectPtr py_cmsg(cmessage::NewEmpty(self->subclass_init));
-    if (py_cmsg == NULL) {
+        *(self->message), self->parent_field_descriptor, i);
+    CMessage* cmsg = cmessage::NewEmptyMessage(self->subclass_init,
+                                               sub_message.GetDescriptor());
+    ScopedPyObjectPtr py_cmsg(reinterpret_cast<PyObject*>(cmsg));
+    if (cmsg == NULL) {
       return -1;
     }
-    CMessage* cmsg = reinterpret_cast<CMessage*>(py_cmsg.get());
     cmsg->owner = self->owner;
     cmsg->message = const_cast<google::protobuf::Message*>(&sub_message);
     cmsg->parent = self->parent;
-    if (cmessage::InitAttributes(cmsg, NULL, NULL) < 0) {
+    if (PyList_Append(self->child_messages, py_cmsg) < 0) {
       return -1;
     }
-    PyList_Append(self->child_messages, py_cmsg);
   }
   return 0;
 }
@@ -258,23 +258,25 @@ static PyObject* AddToAttached(RepeatedCompositeContainer* self,
   google::protobuf::Message* message = self->message;
   google::protobuf::Message* sub_message =
       message->GetReflection()->AddMessage(message,
-                                           self->parent_field->descriptor);
-  PyObject* py_cmsg = cmessage::NewEmpty(self->subclass_init);
-  if (py_cmsg == NULL) {
+                                           self->parent_field_descriptor);
+  CMessage* cmsg = cmessage::NewEmptyMessage(self->subclass_init,
+                                             sub_message->GetDescriptor());
+  if (cmsg == NULL)
     return NULL;
-  }
-  CMessage* cmsg = reinterpret_cast<CMessage*>(py_cmsg);
 
   cmsg->owner = self->owner;
   cmsg->message = sub_message;
   cmsg->parent = self->parent;
-  // cmessage::InitAttributes must be called after cmsg->message has
-  // been set.
-  if (cmessage::InitAttributes(cmsg, NULL, kwargs) < 0) {
+  if (cmessage::InitAttributes(cmsg, kwargs) < 0) {
+    Py_DECREF(cmsg);
+    return NULL;
+  }
+
+  PyObject* py_cmsg = reinterpret_cast<PyObject*>(cmsg);
+  if (PyList_Append(self->child_messages, py_cmsg) < 0) {
     Py_DECREF(py_cmsg);
     return NULL;
   }
-  PyList_Append(self->child_messages, py_cmsg);
   return py_cmsg;
 }
 
@@ -283,20 +285,16 @@ static PyObject* AddToReleased(RepeatedCompositeContainer* self,
                                PyObject* kwargs) {
   GOOGLE_CHECK_RELEASED(self);
 
-  // Create the CMessage
-  PyObject* py_cmsg = PyObject_CallObject(self->subclass_init, NULL);
+  // Create a new Message detached from the rest.
+  PyObject* py_cmsg = PyEval_CallObjectWithKeywords(
+      self->subclass_init, NULL, kwargs);
   if (py_cmsg == NULL)
     return NULL;
-  CMessage* cmsg = reinterpret_cast<CMessage*>(py_cmsg);
-  if (cmessage::InitAttributes(cmsg, NULL, kwargs) < 0) {
+
+  if (PyList_Append(self->child_messages, py_cmsg) < 0) {
     Py_DECREF(py_cmsg);
     return NULL;
   }
-
-  // The Message got created by the call to subclass_init above and
-  // it set self->owner to the newly allocated message.
-
-  PyList_Append(self->child_messages, py_cmsg);
   return py_cmsg;
 }
 
@@ -354,35 +352,9 @@ PyObject* Subscript(RepeatedCompositeContainer* self, PyObject* slice) {
   if (UpdateChildMessages(self) < 0) {
     return NULL;
   }
-  Py_ssize_t from;
-  Py_ssize_t to;
-  Py_ssize_t step;
-  Py_ssize_t length = Length(self);
-  Py_ssize_t slicelength;
-  if (PySlice_Check(slice)) {
-#if PY_MAJOR_VERSION >= 3
-    if (PySlice_GetIndicesEx(slice,
-#else
-    if (PySlice_GetIndicesEx(reinterpret_cast<PySliceObject*>(slice),
-#endif
-                             length, &from, &to, &step, &slicelength) == -1) {
-      return NULL;
-    }
-    return PyList_GetSlice(self->child_messages, from, to);
-  } else if (PyInt_Check(slice) || PyLong_Check(slice)) {
-    from = to = PyLong_AsLong(slice);
-    if (from < 0) {
-      from = to = length + from;
-    }
-    PyObject* result = PyList_GetItem(self->child_messages, from);
-    if (result == NULL) {
-      return NULL;
-    }
-    Py_INCREF(result);
-    return result;
-  }
-  PyErr_SetString(PyExc_TypeError, "index must be an integer or slice");
-  return NULL;
+  // Just forward the call to the subscript-handling function of the
+  // list containing the child messages.
+  return PyObject_GetItem(self->child_messages, slice);
 }
 
 int AssignSubscript(RepeatedCompositeContainer* self,
@@ -399,7 +371,7 @@ int AssignSubscript(RepeatedCompositeContainer* self,
   // Delete from the underlying Message, if any.
   if (self->message != NULL) {
     if (cmessage::InternalDeleteRepeatedField(self->message,
-                                              self->parent_field->descriptor,
+                                              self->parent_field_descriptor,
                                               slice,
                                               self->child_messages) < 0) {
       return -1;
@@ -512,7 +484,7 @@ static PyObject* SortAttached(RepeatedCompositeContainer* self,
   if (reverse) {
     google::protobuf::Message* message = self->message;
     const google::protobuf::Reflection* reflection = message->GetReflection();
-    const google::protobuf::FieldDescriptor* descriptor = self->parent_field->descriptor;
+    const google::protobuf::FieldDescriptor* descriptor = self->parent_field_descriptor;
 
     // Reverse the Message array.
     for (int i = 0; i < length / 2; ++i)
@@ -554,8 +526,9 @@ static PyObject* Sort(RepeatedCompositeContainer* self,
     }
   }
 
-  if (UpdateChildMessages(self) < 0)
+  if (UpdateChildMessages(self) < 0) {
     return NULL;
+  }
   if (self->message == NULL) {
     return SortReleased(self, args, kwds);
   } else {
@@ -617,7 +590,7 @@ void ReleaseLastTo(const FieldDescriptor* field,
   shared_ptr<Message> released_message(
       ReleaseLast(field, cmessage->message->GetDescriptor(), message));
   cmessage->parent = NULL;
-  cmessage->parent_field = NULL;
+  cmessage->parent_field_descriptor = NULL;
   cmessage->message = released_message.get();
   cmessage->read_only = false;
   cmessage::SetOwner(cmessage, released_message);
@@ -633,7 +606,7 @@ int Release(RepeatedCompositeContainer* self) {
   }
 
   Message* message = self->message;
-  const FieldDescriptor* field = self->parent_field->descriptor;
+  const FieldDescriptor* field = self->parent_field_descriptor;
 
   // The reflection API only lets us release the last message in a
   // repeated field.  Therefore we iterate through the children
@@ -648,7 +621,7 @@ int Release(RepeatedCompositeContainer* self) {
 
   // Detach from containing message.
   self->parent = NULL;
-  self->parent_field = NULL;
+  self->parent_field_descriptor = NULL;
   self->message = NULL;
   self->owner.reset();
 
@@ -670,22 +643,40 @@ int SetOwner(RepeatedCompositeContainer* self,
   return 0;
 }
 
-static int Init(RepeatedCompositeContainer* self,
-                PyObject* args,
-                PyObject* kwargs) {
-  self->message = NULL;
-  self->parent = NULL;
-  self->parent_field = NULL;
-  self->subclass_init = NULL;
+// The private constructor of RepeatedCompositeContainer objects.
+PyObject *NewContainer(
+    CMessage* parent,
+    const google::protobuf::FieldDescriptor* parent_field_descriptor,
+    PyObject *concrete_class) {
+  if (!CheckFieldBelongsToMessage(parent_field_descriptor, parent->message)) {
+    return NULL;
+  }
+
+  RepeatedCompositeContainer* self =
+      reinterpret_cast<RepeatedCompositeContainer*>(
+          PyType_GenericAlloc(&RepeatedCompositeContainer_Type, 0));
+  if (self == NULL) {
+    return NULL;
+  }
+
+  self->message = parent->message;
+  self->parent = parent;
+  self->parent_field_descriptor = parent_field_descriptor;
+  self->owner = parent->owner;
+  Py_INCREF(concrete_class);
+  self->subclass_init = concrete_class;
   self->child_messages = PyList_New(0);
-  return 0;
+
+  return reinterpret_cast<PyObject*>(self);
 }
 
 static void Dealloc(RepeatedCompositeContainer* self) {
   Py_CLEAR(self->child_messages);
+  Py_CLEAR(self->subclass_init);
   // TODO(tibell): Do we need to call delete on these objects to make
   // sure their destructors are called?
   self->owner.reset();
+
   Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self));
 }
 
@@ -755,7 +746,7 @@ PyTypeObject RepeatedCompositeContainer_Type = {
   0,                                   //  tp_descr_get
   0,                                   //  tp_descr_set
   0,                                   //  tp_dictoffset
-  (initproc)repeated_composite_container::Init,  //  tp_init
+  0,                                   //  tp_init
 };
 
 }  // namespace python

python/google/protobuf/pyext/repeated_composite_container.h

@@ -55,7 +55,6 @@ using internal::shared_ptr;
 namespace python {
 
 struct CMessage;
-struct CFieldDescriptor;
 
 // A RepeatedCompositeContainer can be in one of two states: attached
 // or released.
@@ -66,7 +65,7 @@ struct CFieldDescriptor;
 // 'child_messages' are owner by the 'owner'.
 //
 // When in the released state 'message', 'owner', 'parent', and
-// 'parent_field' are NULL.
+// 'parent_field_descriptor' are NULL.
 typedef struct RepeatedCompositeContainer {
   PyObject_HEAD;
 
@@ -82,7 +81,8 @@ typedef struct RepeatedCompositeContainer {
   CMessage* parent;
 
   // A descriptor used to modify the underlying 'message'.
-  CFieldDescriptor* parent_field;
+  // The pointer is owned by the global DescriptorPool.
+  const google::protobuf::FieldDescriptor* parent_field_descriptor;
 
   // Pointer to the C++ Message that contains this container.  The
   // RepeatedCompositeContainer does not own this pointer.
@@ -102,6 +102,13 @@ extern PyTypeObject RepeatedCompositeContainer_Type;
 
 namespace repeated_composite_container {
 
+// Builds a RepeatedCompositeContainer object, from a parent message and a
+// field descriptor.
+PyObject *NewContainer(
+    CMessage* parent,
+    const google::protobuf::FieldDescriptor* parent_field_descriptor,
+    PyObject *concrete_class);
+
 // Returns the number of items in this repeated composite container.
 static Py_ssize_t Length(RepeatedCompositeContainer* self);
 

python/google/protobuf/pyext/repeated_scalar_container.cc

@@ -52,7 +52,7 @@
     #error "Python 3.0 - 3.2 are not supported."
   #else
   #define PyString_AsString(ob) \
-    (PyUnicode_Check(ob)? PyUnicode_AsUTF8(ob): PyBytes_AS_STRING(ob))
+    (PyUnicode_Check(ob)? PyUnicode_AsUTF8(ob): PyBytes_AsString(ob))
   #endif
 #endif
 
@@ -67,7 +67,7 @@ namespace repeated_scalar_container {
 static int InternalAssignRepeatedField(
     RepeatedScalarContainer* self, PyObject* list) {
   self->message->GetReflection()->ClearField(self->message,
-                                             self->parent_field->descriptor);
+                                             self->parent_field_descriptor);
   for (Py_ssize_t i = 0; i < PyList_GET_SIZE(list); ++i) {
     PyObject* value = PyList_GET_ITEM(list, i);
     if (Append(self, value) == NULL) {
@@ -80,7 +80,7 @@ static int InternalAssignRepeatedField(
 static Py_ssize_t Len(RepeatedScalarContainer* self) {
   google::protobuf::Message* message = self->message;
   return message->GetReflection()->FieldSize(*message,
-                                             self->parent_field->descriptor);
+                                             self->parent_field_descriptor);
 }
 
 static int AssignItem(RepeatedScalarContainer* self,
@@ -89,12 +89,7 @@ static int AssignItem(RepeatedScalarContainer* self,
   cmessage::AssureWritable(self->parent);
   google::protobuf::Message* message = self->message;
   const google::protobuf::FieldDescriptor* field_descriptor =
-      self->parent_field->descriptor;
-  if (!FIELD_BELONGS_TO_MESSAGE(field_descriptor, message)) {
-    PyErr_SetString(
-        PyExc_KeyError, "Field does not belong to message!");
-    return -1;
-  }
+      self->parent_field_descriptor;
 
   const google::protobuf::Reflection* reflection = message->GetReflection();
   int field_size = reflection->FieldSize(*message, field_descriptor);
@@ -175,7 +170,7 @@ static int AssignItem(RepeatedScalarContainer* self,
         ScopedPyObjectPtr s(PyObject_Str(arg));
         if (s != NULL) {
           PyErr_Format(PyExc_ValueError, "Unknown enum value: %s",
-                       PyString_AsString(s.get()));
+                       PyString_AsString(s));
         }
         return -1;
       }
@@ -193,7 +188,7 @@ static int AssignItem(RepeatedScalarContainer* self,
 static PyObject* Item(RepeatedScalarContainer* self, Py_ssize_t index) {
   google::protobuf::Message* message = self->message;
   const google::protobuf::FieldDescriptor* field_descriptor =
-      self->parent_field->descriptor;
+      self->parent_field_descriptor;
   const google::protobuf::Reflection* reflection = message->GetReflection();
 
   int field_size = reflection->FieldSize(*message, field_descriptor);
@@ -358,13 +353,7 @@ PyObject* Append(RepeatedScalarContainer* self, PyObject* item) {
   cmessage::AssureWritable(self->parent);
   google::protobuf::Message* message = self->message;
   const google::protobuf::FieldDescriptor* field_descriptor =
-      self->parent_field->descriptor;
-
-  if (!FIELD_BELONGS_TO_MESSAGE(field_descriptor, message)) {
-    PyErr_SetString(
-        PyExc_KeyError, "Field does not belong to message!");
-    return NULL;
-  }
+      self->parent_field_descriptor;
 
   const google::protobuf::Reflection* reflection = message->GetReflection();
   switch (field_descriptor->cpp_type()) {
@@ -422,7 +411,7 @@ PyObject* Append(RepeatedScalarContainer* self, PyObject* item) {
         ScopedPyObjectPtr s(PyObject_Str(item));
         if (s != NULL) {
           PyErr_Format(PyExc_ValueError, "Unknown enum value: %s",
-                       PyString_AsString(s.get()));
+                       PyString_AsString(s));
         }
         return NULL;
       }
@@ -451,7 +440,7 @@ static int AssSubscript(RepeatedScalarContainer* self,
   cmessage::AssureWritable(self->parent);
   google::protobuf::Message* message = self->message;
   const google::protobuf::FieldDescriptor* field_descriptor =
-      self->parent_field->descriptor;
+      self->parent_field_descriptor;
 
 #if PY_MAJOR_VERSION < 3
   if (PyInt_Check(slice)) {
@@ -638,47 +627,25 @@ static PyObject* Sort(RepeatedScalarContainer* self,
   Py_RETURN_NONE;
 }
 
-static int Init(RepeatedScalarContainer* self,
-                PyObject* args,
-                PyObject* kwargs) {
-  PyObject* py_parent;
-  PyObject* py_parent_field;
-  if (!PyArg_UnpackTuple(args, "__init__()", 2, 2, &py_parent,
-                         &py_parent_field)) {
-    return -1;
-  }
-
-  if (!PyObject_TypeCheck(py_parent, &CMessage_Type)) {
-    PyErr_Format(PyExc_TypeError,
-                 "expect %s, but got %s",
-                 CMessage_Type.tp_name,
-                 Py_TYPE(py_parent)->tp_name);
-    return -1;
+// The private constructor of RepeatedScalarContainer objects.
+PyObject *NewContainer(
+    CMessage* parent, const google::protobuf::FieldDescriptor* parent_field_descriptor) {
+  if (!CheckFieldBelongsToMessage(parent_field_descriptor, parent->message)) {
+    return NULL;
   }
 
-  if (!PyObject_TypeCheck(py_parent_field, &CFieldDescriptor_Type)) {
-    PyErr_Format(PyExc_TypeError,
-                 "expect %s, but got %s",
-                 CFieldDescriptor_Type.tp_name,
-                 Py_TYPE(py_parent_field)->tp_name);
-    return -1;
+  RepeatedScalarContainer* self = reinterpret_cast<RepeatedScalarContainer*>(
+      PyType_GenericAlloc(&RepeatedScalarContainer_Type, 0));
+  if (self == NULL) {
+    return NULL;
   }
 
-  CMessage* cmessage = reinterpret_cast<CMessage*>(py_parent);
-  CFieldDescriptor* cdescriptor = reinterpret_cast<CFieldDescriptor*>(
-      py_parent_field);
-
-  if (!FIELD_BELONGS_TO_MESSAGE(cdescriptor->descriptor, cmessage->message)) {
-    PyErr_SetString(
-        PyExc_KeyError, "Field does not belong to message!");
-    return -1;
-  }
+  self->message = parent->message;
+  self->parent = parent;
+  self->parent_field_descriptor = parent_field_descriptor;
+  self->owner = parent->owner;
 
-  self->message = cmessage->message;
-  self->parent = cmessage;
-  self->parent_field = cdescriptor;
-  self->owner = cmessage->owner;
-  return 0;
+  return reinterpret_cast<PyObject*>(self);
 }
 
 // Initializes the underlying Message object of "to" so it becomes a new parent
@@ -699,10 +666,7 @@ static int InitializeAndCopyToParentContainer(
   google::protobuf::Message* new_message = global_message_factory->GetPrototype(
       from->message->GetDescriptor())->New();
   to->parent = NULL;
-  // TODO(anuraag): Document why it's OK to hang on to parent_field,
-  //     even though it's a weak reference. It ought to be enough to
-  //     hold on to the FieldDescriptor only.
-  to->parent_field = from->parent_field;
+  to->parent_field_descriptor = from->parent_field_descriptor;
   to->message = new_message;
   to->owner.reset(new_message);
   if (InternalAssignRepeatedField(to, values) < 0) {
@@ -716,23 +680,17 @@ int Release(RepeatedScalarContainer* self) {
 }
 
 PyObject* DeepCopy(RepeatedScalarContainer* self, PyObject* arg) {
-  ScopedPyObjectPtr init_args(
-      PyTuple_Pack(2, self->parent, self->parent_field));
-  PyObject* clone = PyObject_CallObject(
-      reinterpret_cast<PyObject*>(&RepeatedScalarContainer_Type), init_args);
+  RepeatedScalarContainer* clone = reinterpret_cast<RepeatedScalarContainer*>(
+      PyType_GenericAlloc(&RepeatedScalarContainer_Type, 0));
   if (clone == NULL) {
     return NULL;
   }
-  if (!PyObject_TypeCheck(clone, &RepeatedScalarContainer_Type)) {
-    Py_DECREF(clone);
-    return NULL;
-  }
-  if (InitializeAndCopyToParentContainer(
-          self, reinterpret_cast<RepeatedScalarContainer*>(clone)) < 0) {
+
+  if (InitializeAndCopyToParentContainer(self, clone) < 0) {
     Py_DECREF(clone);
     return NULL;
   }
-  return clone;
+  return reinterpret_cast<PyObject*>(clone);
 }
 
 static void Dealloc(RepeatedScalarContainer* self) {
@@ -817,7 +775,7 @@ PyTypeObject RepeatedScalarContainer_Type = {
   0,                                   //  tp_descr_get
   0,                                   //  tp_descr_set
   0,                                   //  tp_dictoffset
-  (initproc)repeated_scalar_container::Init,  //  tp_init
+  0,                                   //  tp_init
 };
 
 }  // namespace python

python/google/protobuf/pyext/repeated_scalar_container.h

@@ -41,6 +41,7 @@
 #include <google/protobuf/stubs/shared_ptr.h>
 #endif
 
+#include <google/protobuf/descriptor.h>
 
 namespace google {
 namespace protobuf {
@@ -51,7 +52,6 @@ using internal::shared_ptr;
 
 namespace python {
 
-struct CFieldDescriptor;
 struct CMessage;
 
 typedef struct RepeatedScalarContainer {
@@ -73,16 +73,22 @@ typedef struct RepeatedScalarContainer {
   // modifying the container.
   CMessage* parent;
 
-  // Weak reference to the parent's descriptor that describes this
+  // Pointer to the parent's descriptor that describes this
   // field.  Used together with the parent's message when making a
   // default message instance mutable.
-  CFieldDescriptor* parent_field;
+  // The pointer is owned by the global DescriptorPool.
+  const google::protobuf::FieldDescriptor* parent_field_descriptor;
 } RepeatedScalarContainer;
 
 extern PyTypeObject RepeatedScalarContainer_Type;
 
 namespace repeated_scalar_container {
 
+// Builds a RepeatedScalarContainer object, from a parent message and a
+// field descriptor.
+extern PyObject *NewContainer(
+    CMessage* parent, const google::protobuf::FieldDescriptor* parent_field_descriptor);
+
 // Appends the scalar 'item' to the end of the container 'self'.
 //
 // Returns None if successful; returns NULL and sets an exception if

python/google/protobuf/pyext/scoped_pyobject_ptr.h

@@ -33,6 +33,8 @@
 #ifndef GOOGLE_PROTOBUF_PYTHON_CPP_SCOPED_PYOBJECT_PTR_H__
 #define GOOGLE_PROTOBUF_PYTHON_CPP_SCOPED_PYOBJECT_PTR_H__
 
+#include <google/protobuf/stubs/common.h>
+
 #include <Python.h>
 
 namespace google {