Merge pull request #3624 from acozzette/down-integrate

Integrated internal changes from Google

BUILD

@@ -326,6 +326,7 @@ cc_library(
         "src/google/protobuf/compiler/cpp/cpp_map_field.cc",
         "src/google/protobuf/compiler/cpp/cpp_message.cc",
         "src/google/protobuf/compiler/cpp/cpp_message_field.cc",
+        "src/google/protobuf/compiler/cpp/cpp_padding_optimizer.cc",
         "src/google/protobuf/compiler/cpp/cpp_primitive_field.cc",
         "src/google/protobuf/compiler/cpp/cpp_service.cc",
         "src/google/protobuf/compiler/cpp/cpp_string_field.cc",

Makefile.am

@@ -225,6 +225,7 @@ java_EXTRA_DIST=
   java/core/src/main/java/com/google/protobuf/IntArrayList.java                    \
   java/core/src/main/java/com/google/protobuf/Internal.java                        \
   java/core/src/main/java/com/google/protobuf/InvalidProtocolBufferException.java  \
+  java/core/src/main/java/com/google/protobuf/IterableByteBufferInputStream.java   \
   java/core/src/main/java/com/google/protobuf/LazyField.java                       \
   java/core/src/main/java/com/google/protobuf/LazyFieldLite.java                   \
   java/core/src/main/java/com/google/protobuf/LazyStringArrayList.java             \

cmake/libprotoc.cmake

@@ -11,6 +11,7 @@ set(libprotoc_files
   ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/cpp_map_field.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/cpp_message.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/cpp_message_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/cpp_padding_optimizer.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/cpp_primitive_field.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/cpp_service.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/cpp_string_field.cc

conformance/conformance_cpp.cc

@@ -52,6 +52,7 @@ using google::protobuf::util::NewTypeResolverForDescriptorPool;
 using google::protobuf::util::Status;
 using google::protobuf::util::TypeResolver;
 using protobuf_test_messages::proto3::TestAllTypesProto3;
+using protobuf_test_messages::proto2::TestAllTypesProto2;
 using std::string;
 
 static const char kTypeUrlPrefix[] = "type.googleapis.com";

csharp/src/Google.Protobuf/Reflection/Descriptor.cs

@@ -70,88 +70,91 @@ namespace Google.Protobuf.Reflection {
             "IkMKBUxhYmVsEhIKDkxBQkVMX09QVElPTkFMEAESEgoOTEFCRUxfUkVRVUlS",
             "RUQQAhISCg5MQUJFTF9SRVBFQVRFRBADIlQKFE9uZW9mRGVzY3JpcHRvclBy",
             "b3RvEgwKBG5hbWUYASABKAkSLgoHb3B0aW9ucxgCIAEoCzIdLmdvb2dsZS5w",
-            "cm90b2J1Zi5PbmVvZk9wdGlvbnMijAEKE0VudW1EZXNjcmlwdG9yUHJvdG8S",
+            "cm90b2J1Zi5PbmVvZk9wdGlvbnMipAIKE0VudW1EZXNjcmlwdG9yUHJvdG8S",
             "DAoEbmFtZRgBIAEoCRI4CgV2YWx1ZRgCIAMoCzIpLmdvb2dsZS5wcm90b2J1",
             "Zi5FbnVtVmFsdWVEZXNjcmlwdG9yUHJvdG8SLQoHb3B0aW9ucxgDIAEoCzIc",
-            "Lmdvb2dsZS5wcm90b2J1Zi5FbnVtT3B0aW9ucyJsChhFbnVtVmFsdWVEZXNj",
-            "cmlwdG9yUHJvdG8SDAoEbmFtZRgBIAEoCRIOCgZudW1iZXIYAiABKAUSMgoH",
-            "b3B0aW9ucxgDIAEoCzIhLmdvb2dsZS5wcm90b2J1Zi5FbnVtVmFsdWVPcHRp",
-            "b25zIpABChZTZXJ2aWNlRGVzY3JpcHRvclByb3RvEgwKBG5hbWUYASABKAkS",
-            "NgoGbWV0aG9kGAIgAygLMiYuZ29vZ2xlLnByb3RvYnVmLk1ldGhvZERlc2Ny",
-            "aXB0b3JQcm90bxIwCgdvcHRpb25zGAMgASgLMh8uZ29vZ2xlLnByb3RvYnVm",
-            "LlNlcnZpY2VPcHRpb25zIsEBChVNZXRob2REZXNjcmlwdG9yUHJvdG8SDAoE",
-            "bmFtZRgBIAEoCRISCgppbnB1dF90eXBlGAIgASgJEhMKC291dHB1dF90eXBl",
-            "GAMgASgJEi8KB29wdGlvbnMYBCABKAsyHi5nb29nbGUucHJvdG9idWYuTWV0",
-            "aG9kT3B0aW9ucxIfChBjbGllbnRfc3RyZWFtaW5nGAUgASgIOgVmYWxzZRIf",
-            "ChBzZXJ2ZXJfc3RyZWFtaW5nGAYgASgIOgVmYWxzZSLwBQoLRmlsZU9wdGlv",
-            "bnMSFAoMamF2YV9wYWNrYWdlGAEgASgJEhwKFGphdmFfb3V0ZXJfY2xhc3Nu",
-            "YW1lGAggASgJEiIKE2phdmFfbXVsdGlwbGVfZmlsZXMYCiABKAg6BWZhbHNl",
-            "EikKHWphdmFfZ2VuZXJhdGVfZXF1YWxzX2FuZF9oYXNoGBQgASgIQgIYARIl",
-            "ChZqYXZhX3N0cmluZ19jaGVja191dGY4GBsgASgIOgVmYWxzZRJGCgxvcHRp",
-            "bWl6ZV9mb3IYCSABKA4yKS5nb29nbGUucHJvdG9idWYuRmlsZU9wdGlvbnMu",
-            "T3B0aW1pemVNb2RlOgVTUEVFRBISCgpnb19wYWNrYWdlGAsgASgJEiIKE2Nj",
-            "X2dlbmVyaWNfc2VydmljZXMYECABKAg6BWZhbHNlEiQKFWphdmFfZ2VuZXJp",
-            "Y19zZXJ2aWNlcxgRIAEoCDoFZmFsc2USIgoTcHlfZ2VuZXJpY19zZXJ2aWNl",
-            "cxgSIAEoCDoFZmFsc2USIwoUcGhwX2dlbmVyaWNfc2VydmljZXMYEyABKAg6",
-            "BWZhbHNlEhkKCmRlcHJlY2F0ZWQYFyABKAg6BWZhbHNlEh8KEGNjX2VuYWJs",
-            "ZV9hcmVuYXMYHyABKAg6BWZhbHNlEhkKEW9iamNfY2xhc3NfcHJlZml4GCQg",
-            "ASgJEhgKEGNzaGFycF9uYW1lc3BhY2UYJSABKAkSFAoMc3dpZnRfcHJlZml4",
-            "GCcgASgJEhgKEHBocF9jbGFzc19wcmVmaXgYKCABKAkSFQoNcGhwX25hbWVz",
-            "cGFjZRgpIAEoCRJDChR1bmludGVycHJldGVkX29wdGlvbhjnByADKAsyJC5n",
-            "b29nbGUucHJvdG9idWYuVW5pbnRlcnByZXRlZE9wdGlvbiI6CgxPcHRpbWl6",
-            "ZU1vZGUSCQoFU1BFRUQQARINCglDT0RFX1NJWkUQAhIQCgxMSVRFX1JVTlRJ",
-            "TUUQAyoJCOgHEICAgIACSgQIJhAnIvIBCg5NZXNzYWdlT3B0aW9ucxImChdt",
-            "ZXNzYWdlX3NldF93aXJlX2Zvcm1hdBgBIAEoCDoFZmFsc2USLgofbm9fc3Rh",
-            "bmRhcmRfZGVzY3JpcHRvcl9hY2Nlc3NvchgCIAEoCDoFZmFsc2USGQoKZGVw",
-            "cmVjYXRlZBgDIAEoCDoFZmFsc2USEQoJbWFwX2VudHJ5GAcgASgIEkMKFHVu",
-            "aW50ZXJwcmV0ZWRfb3B0aW9uGOcHIAMoCzIkLmdvb2dsZS5wcm90b2J1Zi5V",
-            "bmludGVycHJldGVkT3B0aW9uKgkI6AcQgICAgAJKBAgIEAlKBAgJEAoingMK",
-            "DEZpZWxkT3B0aW9ucxI6CgVjdHlwZRgBIAEoDjIjLmdvb2dsZS5wcm90b2J1",
-            "Zi5GaWVsZE9wdGlvbnMuQ1R5cGU6BlNUUklORxIOCgZwYWNrZWQYAiABKAgS",
-            "PwoGanN0eXBlGAYgASgOMiQuZ29vZ2xlLnByb3RvYnVmLkZpZWxkT3B0aW9u",
-            "cy5KU1R5cGU6CUpTX05PUk1BTBITCgRsYXp5GAUgASgIOgVmYWxzZRIZCgpk",
-            "ZXByZWNhdGVkGAMgASgIOgVmYWxzZRITCgR3ZWFrGAogASgIOgVmYWxzZRJD",
-            "ChR1bmludGVycHJldGVkX29wdGlvbhjnByADKAsyJC5nb29nbGUucHJvdG9i",
-            "dWYuVW5pbnRlcnByZXRlZE9wdGlvbiIvCgVDVHlwZRIKCgZTVFJJTkcQABII",
-            "CgRDT1JEEAESEAoMU1RSSU5HX1BJRUNFEAIiNQoGSlNUeXBlEg0KCUpTX05P",
-            "Uk1BTBAAEg0KCUpTX1NUUklORxABEg0KCUpTX05VTUJFUhACKgkI6AcQgICA",
-            "gAJKBAgEEAUiXgoMT25lb2ZPcHRpb25zEkMKFHVuaW50ZXJwcmV0ZWRfb3B0",
-            "aW9uGOcHIAMoCzIkLmdvb2dsZS5wcm90b2J1Zi5VbmludGVycHJldGVkT3B0",
-            "aW9uKgkI6AcQgICAgAIikwEKC0VudW1PcHRpb25zEhMKC2FsbG93X2FsaWFz",
-            "GAIgASgIEhkKCmRlcHJlY2F0ZWQYAyABKAg6BWZhbHNlEkMKFHVuaW50ZXJw",
-            "cmV0ZWRfb3B0aW9uGOcHIAMoCzIkLmdvb2dsZS5wcm90b2J1Zi5VbmludGVy",
-            "cHJldGVkT3B0aW9uKgkI6AcQgICAgAJKBAgFEAYifQoQRW51bVZhbHVlT3B0",
-            "aW9ucxIZCgpkZXByZWNhdGVkGAEgASgIOgVmYWxzZRJDChR1bmludGVycHJl",
-            "dGVkX29wdGlvbhjnByADKAsyJC5nb29nbGUucHJvdG9idWYuVW5pbnRlcnBy",
-            "ZXRlZE9wdGlvbioJCOgHEICAgIACInsKDlNlcnZpY2VPcHRpb25zEhkKCmRl",
-            "cHJlY2F0ZWQYISABKAg6BWZhbHNlEkMKFHVuaW50ZXJwcmV0ZWRfb3B0aW9u",
-            "GOcHIAMoCzIkLmdvb2dsZS5wcm90b2J1Zi5VbmludGVycHJldGVkT3B0aW9u",
-            "KgkI6AcQgICAgAIirQIKDU1ldGhvZE9wdGlvbnMSGQoKZGVwcmVjYXRlZBgh",
-            "IAEoCDoFZmFsc2USXwoRaWRlbXBvdGVuY3lfbGV2ZWwYIiABKA4yLy5nb29n",
-            "bGUucHJvdG9idWYuTWV0aG9kT3B0aW9ucy5JZGVtcG90ZW5jeUxldmVsOhNJ",
-            "REVNUE9URU5DWV9VTktOT1dOEkMKFHVuaW50ZXJwcmV0ZWRfb3B0aW9uGOcH",
-            "IAMoCzIkLmdvb2dsZS5wcm90b2J1Zi5VbmludGVycHJldGVkT3B0aW9uIlAK",
-            "EElkZW1wb3RlbmN5TGV2ZWwSFwoTSURFTVBPVEVOQ1lfVU5LTk9XThAAEhMK",
-            "D05PX1NJREVfRUZGRUNUUxABEg4KCklERU1QT1RFTlQQAioJCOgHEICAgIAC",
-            "Ip4CChNVbmludGVycHJldGVkT3B0aW9uEjsKBG5hbWUYAiADKAsyLS5nb29n",
-            "bGUucHJvdG9idWYuVW5pbnRlcnByZXRlZE9wdGlvbi5OYW1lUGFydBIYChBp",
-            "ZGVudGlmaWVyX3ZhbHVlGAMgASgJEhoKEnBvc2l0aXZlX2ludF92YWx1ZRgE",
-            "IAEoBBIaChJuZWdhdGl2ZV9pbnRfdmFsdWUYBSABKAMSFAoMZG91YmxlX3Zh",
-            "bHVlGAYgASgBEhQKDHN0cmluZ192YWx1ZRgHIAEoDBIXCg9hZ2dyZWdhdGVf",
-            "dmFsdWUYCCABKAkaMwoITmFtZVBhcnQSEQoJbmFtZV9wYXJ0GAEgAigJEhQK",
-            "DGlzX2V4dGVuc2lvbhgCIAIoCCLVAQoOU291cmNlQ29kZUluZm8SOgoIbG9j",
-            "YXRpb24YASADKAsyKC5nb29nbGUucHJvdG9idWYuU291cmNlQ29kZUluZm8u",
-            "TG9jYXRpb24ahgEKCExvY2F0aW9uEhAKBHBhdGgYASADKAVCAhABEhAKBHNw",
-            "YW4YAiADKAVCAhABEhgKEGxlYWRpbmdfY29tbWVudHMYAyABKAkSGQoRdHJh",
-            "aWxpbmdfY29tbWVudHMYBCABKAkSIQoZbGVhZGluZ19kZXRhY2hlZF9jb21t",
-            "ZW50cxgGIAMoCSKnAQoRR2VuZXJhdGVkQ29kZUluZm8SQQoKYW5ub3RhdGlv",
-            "bhgBIAMoCzItLmdvb2dsZS5wcm90b2J1Zi5HZW5lcmF0ZWRDb2RlSW5mby5B",
-            "bm5vdGF0aW9uGk8KCkFubm90YXRpb24SEAoEcGF0aBgBIAMoBUICEAESEwoL",
-            "c291cmNlX2ZpbGUYAiABKAkSDQoFYmVnaW4YAyABKAUSCwoDZW5kGAQgASgF",
-            "QowBChNjb20uZ29vZ2xlLnByb3RvYnVmQhBEZXNjcmlwdG9yUHJvdG9zSAFa",
-            "PmdpdGh1Yi5jb20vZ29sYW5nL3Byb3RvYnVmL3Byb3RvYy1nZW4tZ28vZGVz",
-            "Y3JpcHRvcjtkZXNjcmlwdG9yogIDR1BCqgIaR29vZ2xlLlByb3RvYnVmLlJl",
-            "ZmxlY3Rpb24="));
+            "Lmdvb2dsZS5wcm90b2J1Zi5FbnVtT3B0aW9ucxJOCg5yZXNlcnZlZF9yYW5n",
+            "ZRgEIAMoCzI2Lmdvb2dsZS5wcm90b2J1Zi5FbnVtRGVzY3JpcHRvclByb3Rv",
+            "LkVudW1SZXNlcnZlZFJhbmdlEhUKDXJlc2VydmVkX25hbWUYBSADKAkaLwoR",
+            "RW51bVJlc2VydmVkUmFuZ2USDQoFc3RhcnQYASABKAUSCwoDZW5kGAIgASgF",
+            "ImwKGEVudW1WYWx1ZURlc2NyaXB0b3JQcm90bxIMCgRuYW1lGAEgASgJEg4K",
+            "Bm51bWJlchgCIAEoBRIyCgdvcHRpb25zGAMgASgLMiEuZ29vZ2xlLnByb3Rv",
+            "YnVmLkVudW1WYWx1ZU9wdGlvbnMikAEKFlNlcnZpY2VEZXNjcmlwdG9yUHJv",
+            "dG8SDAoEbmFtZRgBIAEoCRI2CgZtZXRob2QYAiADKAsyJi5nb29nbGUucHJv",
+            "dG9idWYuTWV0aG9kRGVzY3JpcHRvclByb3RvEjAKB29wdGlvbnMYAyABKAsy",
+            "Hy5nb29nbGUucHJvdG9idWYuU2VydmljZU9wdGlvbnMiwQEKFU1ldGhvZERl",
+            "c2NyaXB0b3JQcm90bxIMCgRuYW1lGAEgASgJEhIKCmlucHV0X3R5cGUYAiAB",
+            "KAkSEwoLb3V0cHV0X3R5cGUYAyABKAkSLwoHb3B0aW9ucxgEIAEoCzIeLmdv",
+            "b2dsZS5wcm90b2J1Zi5NZXRob2RPcHRpb25zEh8KEGNsaWVudF9zdHJlYW1p",
+            "bmcYBSABKAg6BWZhbHNlEh8KEHNlcnZlcl9zdHJlYW1pbmcYBiABKAg6BWZh",
+            "bHNlIvAFCgtGaWxlT3B0aW9ucxIUCgxqYXZhX3BhY2thZ2UYASABKAkSHAoU",
+            "amF2YV9vdXRlcl9jbGFzc25hbWUYCCABKAkSIgoTamF2YV9tdWx0aXBsZV9m",
+            "aWxlcxgKIAEoCDoFZmFsc2USKQodamF2YV9nZW5lcmF0ZV9lcXVhbHNfYW5k",
+            "X2hhc2gYFCABKAhCAhgBEiUKFmphdmFfc3RyaW5nX2NoZWNrX3V0ZjgYGyAB",
+            "KAg6BWZhbHNlEkYKDG9wdGltaXplX2ZvchgJIAEoDjIpLmdvb2dsZS5wcm90",
+            "b2J1Zi5GaWxlT3B0aW9ucy5PcHRpbWl6ZU1vZGU6BVNQRUVEEhIKCmdvX3Bh",
+            "Y2thZ2UYCyABKAkSIgoTY2NfZ2VuZXJpY19zZXJ2aWNlcxgQIAEoCDoFZmFs",
+            "c2USJAoVamF2YV9nZW5lcmljX3NlcnZpY2VzGBEgASgIOgVmYWxzZRIiChNw",
+            "eV9nZW5lcmljX3NlcnZpY2VzGBIgASgIOgVmYWxzZRIjChRwaHBfZ2VuZXJp",
+            "Y19zZXJ2aWNlcxgqIAEoCDoFZmFsc2USGQoKZGVwcmVjYXRlZBgXIAEoCDoF",
+            "ZmFsc2USHwoQY2NfZW5hYmxlX2FyZW5hcxgfIAEoCDoFZmFsc2USGQoRb2Jq",
+            "Y19jbGFzc19wcmVmaXgYJCABKAkSGAoQY3NoYXJwX25hbWVzcGFjZRglIAEo",
+            "CRIUCgxzd2lmdF9wcmVmaXgYJyABKAkSGAoQcGhwX2NsYXNzX3ByZWZpeBgo",
+            "IAEoCRIVCg1waHBfbmFtZXNwYWNlGCkgASgJEkMKFHVuaW50ZXJwcmV0ZWRf",
+            "b3B0aW9uGOcHIAMoCzIkLmdvb2dsZS5wcm90b2J1Zi5VbmludGVycHJldGVk",
+            "T3B0aW9uIjoKDE9wdGltaXplTW9kZRIJCgVTUEVFRBABEg0KCUNPREVfU0la",
+            "RRACEhAKDExJVEVfUlVOVElNRRADKgkI6AcQgICAgAJKBAgmECci8gEKDk1l",
+            "c3NhZ2VPcHRpb25zEiYKF21lc3NhZ2Vfc2V0X3dpcmVfZm9ybWF0GAEgASgI",
+            "OgVmYWxzZRIuCh9ub19zdGFuZGFyZF9kZXNjcmlwdG9yX2FjY2Vzc29yGAIg",
+            "ASgIOgVmYWxzZRIZCgpkZXByZWNhdGVkGAMgASgIOgVmYWxzZRIRCgltYXBf",
+            "ZW50cnkYByABKAgSQwoUdW5pbnRlcnByZXRlZF9vcHRpb24Y5wcgAygLMiQu",
+            "Z29vZ2xlLnByb3RvYnVmLlVuaW50ZXJwcmV0ZWRPcHRpb24qCQjoBxCAgICA",
+            "AkoECAgQCUoECAkQCiKeAwoMRmllbGRPcHRpb25zEjoKBWN0eXBlGAEgASgO",
+            "MiMuZ29vZ2xlLnByb3RvYnVmLkZpZWxkT3B0aW9ucy5DVHlwZToGU1RSSU5H",
+            "Eg4KBnBhY2tlZBgCIAEoCBI/CgZqc3R5cGUYBiABKA4yJC5nb29nbGUucHJv",
+            "dG9idWYuRmllbGRPcHRpb25zLkpTVHlwZToJSlNfTk9STUFMEhMKBGxhenkY",
+            "BSABKAg6BWZhbHNlEhkKCmRlcHJlY2F0ZWQYAyABKAg6BWZhbHNlEhMKBHdl",
+            "YWsYCiABKAg6BWZhbHNlEkMKFHVuaW50ZXJwcmV0ZWRfb3B0aW9uGOcHIAMo",
+            "CzIkLmdvb2dsZS5wcm90b2J1Zi5VbmludGVycHJldGVkT3B0aW9uIi8KBUNU",
+            "eXBlEgoKBlNUUklORxAAEggKBENPUkQQARIQCgxTVFJJTkdfUElFQ0UQAiI1",
+            "CgZKU1R5cGUSDQoJSlNfTk9STUFMEAASDQoJSlNfU1RSSU5HEAESDQoJSlNf",
+            "TlVNQkVSEAIqCQjoBxCAgICAAkoECAQQBSJeCgxPbmVvZk9wdGlvbnMSQwoU",
+            "dW5pbnRlcnByZXRlZF9vcHRpb24Y5wcgAygLMiQuZ29vZ2xlLnByb3RvYnVm",
+            "LlVuaW50ZXJwcmV0ZWRPcHRpb24qCQjoBxCAgICAAiKTAQoLRW51bU9wdGlv",
+            "bnMSEwoLYWxsb3dfYWxpYXMYAiABKAgSGQoKZGVwcmVjYXRlZBgDIAEoCDoF",
+            "ZmFsc2USQwoUdW5pbnRlcnByZXRlZF9vcHRpb24Y5wcgAygLMiQuZ29vZ2xl",
+            "LnByb3RvYnVmLlVuaW50ZXJwcmV0ZWRPcHRpb24qCQjoBxCAgICAAkoECAUQ",
+            "BiJ9ChBFbnVtVmFsdWVPcHRpb25zEhkKCmRlcHJlY2F0ZWQYASABKAg6BWZh",
+            "bHNlEkMKFHVuaW50ZXJwcmV0ZWRfb3B0aW9uGOcHIAMoCzIkLmdvb2dsZS5w",
+            "cm90b2J1Zi5VbmludGVycHJldGVkT3B0aW9uKgkI6AcQgICAgAIiewoOU2Vy",
+            "dmljZU9wdGlvbnMSGQoKZGVwcmVjYXRlZBghIAEoCDoFZmFsc2USQwoUdW5p",
+            "bnRlcnByZXRlZF9vcHRpb24Y5wcgAygLMiQuZ29vZ2xlLnByb3RvYnVmLlVu",
+            "aW50ZXJwcmV0ZWRPcHRpb24qCQjoBxCAgICAAiKtAgoNTWV0aG9kT3B0aW9u",
+            "cxIZCgpkZXByZWNhdGVkGCEgASgIOgVmYWxzZRJfChFpZGVtcG90ZW5jeV9s",
+            "ZXZlbBgiIAEoDjIvLmdvb2dsZS5wcm90b2J1Zi5NZXRob2RPcHRpb25zLklk",
+            "ZW1wb3RlbmN5TGV2ZWw6E0lERU1QT1RFTkNZX1VOS05PV04SQwoUdW5pbnRl",
+            "cnByZXRlZF9vcHRpb24Y5wcgAygLMiQuZ29vZ2xlLnByb3RvYnVmLlVuaW50",
+            "ZXJwcmV0ZWRPcHRpb24iUAoQSWRlbXBvdGVuY3lMZXZlbBIXChNJREVNUE9U",
+            "RU5DWV9VTktOT1dOEAASEwoPTk9fU0lERV9FRkZFQ1RTEAESDgoKSURFTVBP",
+            "VEVOVBACKgkI6AcQgICAgAIingIKE1VuaW50ZXJwcmV0ZWRPcHRpb24SOwoE",
+            "bmFtZRgCIAMoCzItLmdvb2dsZS5wcm90b2J1Zi5VbmludGVycHJldGVkT3B0",
+            "aW9uLk5hbWVQYXJ0EhgKEGlkZW50aWZpZXJfdmFsdWUYAyABKAkSGgoScG9z",
+            "aXRpdmVfaW50X3ZhbHVlGAQgASgEEhoKEm5lZ2F0aXZlX2ludF92YWx1ZRgF",
+            "IAEoAxIUCgxkb3VibGVfdmFsdWUYBiABKAESFAoMc3RyaW5nX3ZhbHVlGAcg",
+            "ASgMEhcKD2FnZ3JlZ2F0ZV92YWx1ZRgIIAEoCRozCghOYW1lUGFydBIRCglu",
+            "YW1lX3BhcnQYASACKAkSFAoMaXNfZXh0ZW5zaW9uGAIgAigIItUBCg5Tb3Vy",
+            "Y2VDb2RlSW5mbxI6Cghsb2NhdGlvbhgBIAMoCzIoLmdvb2dsZS5wcm90b2J1",
+            "Zi5Tb3VyY2VDb2RlSW5mby5Mb2NhdGlvbhqGAQoITG9jYXRpb24SEAoEcGF0",
+            "aBgBIAMoBUICEAESEAoEc3BhbhgCIAMoBUICEAESGAoQbGVhZGluZ19jb21t",
+            "ZW50cxgDIAEoCRIZChF0cmFpbGluZ19jb21tZW50cxgEIAEoCRIhChlsZWFk",
+            "aW5nX2RldGFjaGVkX2NvbW1lbnRzGAYgAygJIqcBChFHZW5lcmF0ZWRDb2Rl",
+            "SW5mbxJBCgphbm5vdGF0aW9uGAEgAygLMi0uZ29vZ2xlLnByb3RvYnVmLkdl",
+            "bmVyYXRlZENvZGVJbmZvLkFubm90YXRpb24aTwoKQW5ub3RhdGlvbhIQCgRw",
+            "YXRoGAEgAygFQgIQARITCgtzb3VyY2VfZmlsZRgCIAEoCRINCgViZWdpbhgD",
+            "IAEoBRILCgNlbmQYBCABKAVCjwEKE2NvbS5nb29nbGUucHJvdG9idWZCEERl",
+            "c2NyaXB0b3JQcm90b3NIAVo+Z2l0aHViLmNvbS9nb2xhbmcvcHJvdG9idWYv",
+            "cHJvdG9jLWdlbi1nby9kZXNjcmlwdG9yO2Rlc2NyaXB0b3L4AQGiAgNHUEKq",
+            "AhpHb29nbGUuUHJvdG9idWYuUmVmbGVjdGlvbg=="));
       descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData,
           new pbr::FileDescriptor[] { },
           new pbr::GeneratedClrTypeInfo(null, new pbr::GeneratedClrTypeInfo[] {
@@ -162,7 +165,7 @@ namespace Google.Protobuf.Reflection {
             new pbr::GeneratedClrTypeInfo(typeof(global::Google.Protobuf.Reflection.ExtensionRangeOptions), global::Google.Protobuf.Reflection.ExtensionRangeOptions.Parser, new[]{ "UninterpretedOption" }, null, null, null),
             new pbr::GeneratedClrTypeInfo(typeof(global::Google.Protobuf.Reflection.FieldDescriptorProto), global::Google.Protobuf.Reflection.FieldDescriptorProto.Parser, new[]{ "Name", "Number", "Label", "Type", "TypeName", "Extendee", "DefaultValue", "OneofIndex", "JsonName", "Options" }, null, new[]{ typeof(global::Google.Protobuf.Reflection.FieldDescriptorProto.Types.Type), typeof(global::Google.Protobuf.Reflection.FieldDescriptorProto.Types.Label) }, null),
             new pbr::GeneratedClrTypeInfo(typeof(global::Google.Protobuf.Reflection.OneofDescriptorProto), global::Google.Protobuf.Reflection.OneofDescriptorProto.Parser, new[]{ "Name", "Options" }, null, null, null),
-            new pbr::GeneratedClrTypeInfo(typeof(global::Google.Protobuf.Reflection.EnumDescriptorProto), global::Google.Protobuf.Reflection.EnumDescriptorProto.Parser, new[]{ "Name", "Value", "Options" }, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Google.Protobuf.Reflection.EnumDescriptorProto), global::Google.Protobuf.Reflection.EnumDescriptorProto.Parser, new[]{ "Name", "Value", "Options", "ReservedRange", "ReservedName" }, null, null, new pbr::GeneratedClrTypeInfo[] { new pbr::GeneratedClrTypeInfo(typeof(global::Google.Protobuf.Reflection.EnumDescriptorProto.Types.EnumReservedRange), global::Google.Protobuf.Reflection.EnumDescriptorProto.Types.EnumReservedRange.Parser, new[]{ "Start", "End" }, null, null, null)}),
             new pbr::GeneratedClrTypeInfo(typeof(global::Google.Protobuf.Reflection.EnumValueDescriptorProto), global::Google.Protobuf.Reflection.EnumValueDescriptorProto.Parser, new[]{ "Name", "Number", "Options" }, null, null, null),
             new pbr::GeneratedClrTypeInfo(typeof(global::Google.Protobuf.Reflection.ServiceDescriptorProto), global::Google.Protobuf.Reflection.ServiceDescriptorProto.Parser, new[]{ "Name", "Method", "Options" }, null, null, null),
             new pbr::GeneratedClrTypeInfo(typeof(global::Google.Protobuf.Reflection.MethodDescriptorProto), global::Google.Protobuf.Reflection.MethodDescriptorProto.Parser, new[]{ "Name", "InputType", "OutputType", "Options", "ClientStreaming", "ServerStreaming" }, null, null, null),
@@ -2144,6 +2147,8 @@ namespace Google.Protobuf.Reflection {
       name_ = other.name_;
       value_ = other.value_.Clone();
       Options = other.options_ != null ? other.Options.Clone() : null;
+      reservedRange_ = other.reservedRange_.Clone();
+      reservedName_ = other.reservedName_.Clone();
     }
 
     [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
@@ -2183,6 +2188,35 @@ namespace Google.Protobuf.Reflection {
       }
     }
 
+    /// <summary>Field number for the "reserved_range" field.</summary>
+    public const int ReservedRangeFieldNumber = 4;
+    private static readonly pb::FieldCodec<global::Google.Protobuf.Reflection.EnumDescriptorProto.Types.EnumReservedRange> _repeated_reservedRange_codec
+        = pb::FieldCodec.ForMessage(34, global::Google.Protobuf.Reflection.EnumDescriptorProto.Types.EnumReservedRange.Parser);
+    private readonly pbc::RepeatedField<global::Google.Protobuf.Reflection.EnumDescriptorProto.Types.EnumReservedRange> reservedRange_ = new pbc::RepeatedField<global::Google.Protobuf.Reflection.EnumDescriptorProto.Types.EnumReservedRange>();
+    /// <summary>
+    /// Range of reserved numeric values. Reserved numeric values may not be used
+    /// by enum values in the same enum declaration. Reserved ranges may not
+    /// overlap.
+    /// </summary>
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public pbc::RepeatedField<global::Google.Protobuf.Reflection.EnumDescriptorProto.Types.EnumReservedRange> ReservedRange {
+      get { return reservedRange_; }
+    }
+
+    /// <summary>Field number for the "reserved_name" field.</summary>
+    public const int ReservedNameFieldNumber = 5;
+    private static readonly pb::FieldCodec<string> _repeated_reservedName_codec
+        = pb::FieldCodec.ForString(42);
+    private readonly pbc::RepeatedField<string> reservedName_ = new pbc::RepeatedField<string>();
+    /// <summary>
+    /// Reserved enum value names, which may not be reused. A given name may only
+    /// be reserved once.
+    /// </summary>
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public pbc::RepeatedField<string> ReservedName {
+      get { return reservedName_; }
+    }
+
     [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
     public override bool Equals(object other) {
       return Equals(other as EnumDescriptorProto);
@@ -2199,6 +2233,8 @@ namespace Google.Protobuf.Reflection {
       if (Name != other.Name) return false;
       if(!value_.Equals(other.value_)) return false;
       if (!object.Equals(Options, other.Options)) return false;
+      if(!reservedRange_.Equals(other.reservedRange_)) return false;
+      if(!reservedName_.Equals(other.reservedName_)) return false;
       return true;
     }
 
@@ -2208,6 +2244,8 @@ namespace Google.Protobuf.Reflection {
       if (Name.Length != 0) hash ^= Name.GetHashCode();
       hash ^= value_.GetHashCode();
       if (options_ != null) hash ^= Options.GetHashCode();
+      hash ^= reservedRange_.GetHashCode();
+      hash ^= reservedName_.GetHashCode();
       return hash;
     }
 
@@ -2227,6 +2265,8 @@ namespace Google.Protobuf.Reflection {
         output.WriteRawTag(26);
         output.WriteMessage(Options);
       }
+      reservedRange_.WriteTo(output, _repeated_reservedRange_codec);
+      reservedName_.WriteTo(output, _repeated_reservedName_codec);
     }
 
     [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
@@ -2239,6 +2279,8 @@ namespace Google.Protobuf.Reflection {
       if (options_ != null) {
         size += 1 + pb::CodedOutputStream.ComputeMessageSize(Options);
       }
+      size += reservedRange_.CalculateSize(_repeated_reservedRange_codec);
+      size += reservedName_.CalculateSize(_repeated_reservedName_codec);
       return size;
     }
 
@@ -2257,6 +2299,8 @@ namespace Google.Protobuf.Reflection {
         }
         Options.MergeFrom(other.Options);
       }
+      reservedRange_.Add(other.reservedRange_);
+      reservedName_.Add(other.reservedName_);
     }
 
     [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
@@ -2282,10 +2326,184 @@ namespace Google.Protobuf.Reflection {
             input.ReadMessage(options_);
             break;
           }
+          case 34: {
+            reservedRange_.AddEntriesFrom(input, _repeated_reservedRange_codec);
+            break;
+          }
+          case 42: {
+            reservedName_.AddEntriesFrom(input, _repeated_reservedName_codec);
+            break;
+          }
         }
       }
     }
 
+    #region Nested types
+    /// <summary>Container for nested types declared in the EnumDescriptorProto message type.</summary>
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public static partial class Types {
+      /// <summary>
+      /// Range of reserved numeric values. Reserved values may not be used by
+      /// entries in the same enum. Reserved ranges may not overlap.
+      ///
+      /// Note that this is distinct from DescriptorProto.ReservedRange in that it
+      /// is inclusive such that it can appropriately represent the entire int32
+      /// domain.
+      /// </summary>
+      internal sealed partial class EnumReservedRange : pb::IMessage<EnumReservedRange> {
+        private static readonly pb::MessageParser<EnumReservedRange> _parser = new pb::MessageParser<EnumReservedRange>(() => new EnumReservedRange());
+        [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+        public static pb::MessageParser<EnumReservedRange> Parser { get { return _parser; } }
+
+        [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+        public static pbr::MessageDescriptor Descriptor {
+          get { return global::Google.Protobuf.Reflection.EnumDescriptorProto.Descriptor.NestedTypes[0]; }
+        }
+
+        [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+        pbr::MessageDescriptor pb::IMessage.Descriptor {
+          get { return Descriptor; }
+        }
+
+        [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+        public EnumReservedRange() {
+          OnConstruction();
+        }
+
+        partial void OnConstruction();
+
+        [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+        public EnumReservedRange(EnumReservedRange other) : this() {
+          start_ = other.start_;
+          end_ = other.end_;
+        }
+
+        [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+        public EnumReservedRange Clone() {
+          return new EnumReservedRange(this);
+        }
+
+        /// <summary>Field number for the "start" field.</summary>
+        public const int StartFieldNumber = 1;
+        private int start_;
+        /// <summary>
+        /// Inclusive.
+        /// </summary>
+        [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+        public int Start {
+          get { return start_; }
+          set {
+            start_ = value;
+          }
+        }
+
+        /// <summary>Field number for the "end" field.</summary>
+        public const int EndFieldNumber = 2;
+        private int end_;
+        /// <summary>
+        /// Inclusive.
+        /// </summary>
+        [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+        public int End {
+          get { return end_; }
+          set {
+            end_ = value;
+          }
+        }
+
+        [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+        public override bool Equals(object other) {
+          return Equals(other as EnumReservedRange);
+        }
+
+        [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+        public bool Equals(EnumReservedRange other) {
+          if (ReferenceEquals(other, null)) {
+            return false;
+          }
+          if (ReferenceEquals(other, this)) {
+            return true;
+          }
+          if (Start != other.Start) return false;
+          if (End != other.End) return false;
+          return true;
+        }
+
+        [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+        public override int GetHashCode() {
+          int hash = 1;
+          if (Start != 0) hash ^= Start.GetHashCode();
+          if (End != 0) hash ^= End.GetHashCode();
+          return hash;
+        }
+
+        [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+        public override string ToString() {
+          return pb::JsonFormatter.ToDiagnosticString(this);
+        }
+
+        [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+        public void WriteTo(pb::CodedOutputStream output) {
+          if (Start != 0) {
+            output.WriteRawTag(8);
+            output.WriteInt32(Start);
+          }
+          if (End != 0) {
+            output.WriteRawTag(16);
+            output.WriteInt32(End);
+          }
+        }
+
+        [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+        public int CalculateSize() {
+          int size = 0;
+          if (Start != 0) {
+            size += 1 + pb::CodedOutputStream.ComputeInt32Size(Start);
+          }
+          if (End != 0) {
+            size += 1 + pb::CodedOutputStream.ComputeInt32Size(End);
+          }
+          return size;
+        }
+
+        [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+        public void MergeFrom(EnumReservedRange other) {
+          if (other == null) {
+            return;
+          }
+          if (other.Start != 0) {
+            Start = other.Start;
+          }
+          if (other.End != 0) {
+            End = other.End;
+          }
+        }
+
+        [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+        public void MergeFrom(pb::CodedInputStream input) {
+          uint tag;
+          while ((tag = input.ReadTag()) != 0) {
+            switch(tag) {
+              default:
+                input.SkipLastField();
+                break;
+              case 8: {
+                Start = input.ReadInt32();
+                break;
+              }
+              case 16: {
+                End = input.ReadInt32();
+                break;
+              }
+            }
+          }
+        }
+
+      }
+
+    }
+    #endregion
+
   }
 
   /// <summary>
@@ -3135,7 +3353,7 @@ namespace Google.Protobuf.Reflection {
     }
 
     /// <summary>Field number for the "php_generic_services" field.</summary>
-    public const int PhpGenericServicesFieldNumber = 19;
+    public const int PhpGenericServicesFieldNumber = 42;
     private bool phpGenericServices_;
     [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
     public bool PhpGenericServices {
@@ -3366,10 +3584,6 @@ namespace Google.Protobuf.Reflection {
         output.WriteRawTag(144, 1);
         output.WriteBool(PyGenericServices);
       }
-      if (PhpGenericServices != false) {
-        output.WriteRawTag(152, 1);
-        output.WriteBool(PhpGenericServices);
-      }
       if (JavaGenerateEqualsAndHash != false) {
         output.WriteRawTag(160, 1);
         output.WriteBool(JavaGenerateEqualsAndHash);
@@ -3406,6 +3620,10 @@ namespace Google.Protobuf.Reflection {
         output.WriteRawTag(202, 2);
         output.WriteString(PhpNamespace);
       }
+      if (PhpGenericServices != false) {
+        output.WriteRawTag(208, 2);
+        output.WriteBool(PhpGenericServices);
+      }
       uninterpretedOption_.WriteTo(output, _repeated_uninterpretedOption_codec);
     }
 
@@ -3572,10 +3790,6 @@ namespace Google.Protobuf.Reflection {
             PyGenericServices = input.ReadBool();
             break;
           }
-          case 152: {
-            PhpGenericServices = input.ReadBool();
-            break;
-          }
           case 160: {
             JavaGenerateEqualsAndHash = input.ReadBool();
             break;
@@ -3612,6 +3826,10 @@ namespace Google.Protobuf.Reflection {
             PhpNamespace = input.ReadString();
             break;
           }
+          case 336: {
+            PhpGenericServices = input.ReadBool();
+            break;
+          }
           case 7994: {
             uninterpretedOption_.AddEntriesFrom(input, _repeated_uninterpretedOption_codec);
             break;

java/core/src/main/java/com/google/protobuf/CodedInputStream.java

@@ -44,6 +44,7 @@ import java.io.InputStream;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Iterator;
 import java.util.List;
 
 /**
@@ -85,6 +86,43 @@ public abstract class CodedInputStream {
     return new StreamDecoder(input, bufferSize);
   }
 
+  /** Create a new CodedInputStream wrapping the given {@code Iterable <ByteBuffer>}. */
+  public static CodedInputStream newInstance(final Iterable<ByteBuffer> input) {
+    if (!UnsafeDirectNioDecoder.isSupported()) {
+      return newInstance(new IterableByteBufferInputStream(input));
+    }
+    return newInstance(input, false);
+  }
+
+  /** Create a new CodedInputStream wrapping the given {@code Iterable <ByteBuffer>}. */
+  static CodedInputStream newInstance(
+      final Iterable<ByteBuffer> bufs, final boolean bufferIsImmutable) {
+    // flag is to check the type of input's ByteBuffers.
+    // flag equals 1: all ByteBuffers have array.
+    // flag equals 2: all ByteBuffers are direct ByteBuffers.
+    // flag equals 3: some ByteBuffers are direct and some have array.
+    // flag greater than 3: other cases.
+    int flag = 0;
+    // Total size of the input
+    int totalSize = 0;
+    for (ByteBuffer buf : bufs) {
+      totalSize += buf.remaining();
+      if (buf.hasArray()) {
+        flag |= 1;
+      } else if (buf.isDirect()) {
+        flag |= 2;
+      } else {
+        flag |= 4;
+      }
+    }
+    if (flag == 2) {
+      return new IterableDirectByteBufferDecoder(bufs, totalSize, bufferIsImmutable);
+    } else {
+      // TODO(yilunchong): add another decoders to deal case 1 and 3.
+      return newInstance(new IterableByteBufferInputStream(bufs));
+    }
+  }
+
   /** Create a new CodedInputStream wrapping the given byte array. */
   public static CodedInputStream newInstance(final byte[] buf) {
     return newInstance(buf, 0, buf.length);
@@ -3022,4 +3060,848 @@ public abstract class CodedInputStream {
       pos = size - tempPos;
     }
   }
+
+  /**
+   * Implementation of {@link CodedInputStream} that uses an {@link Iterable <ByteBuffer>} as the
+   * data source. Requires the use of {@code sun.misc.Unsafe} to perform fast reads on the buffer.
+   */
+  private static final class IterableDirectByteBufferDecoder extends CodedInputStream {
+    /** The object that need to decode. */
+    private Iterable<ByteBuffer> input;
+    /** The {@link Iterator} with type {@link ByteBuffer} of {@code input} */
+    private Iterator<ByteBuffer> iterator;
+    /** The current ByteBuffer; */
+    private ByteBuffer currentByteBuffer;
+    /**
+     * If {@code true}, indicates that all the buffer are backing a {@link ByteString} and are
+     * therefore considered to be an immutable input source.
+     */
+    private boolean immutable;
+    /**
+     * If {@code true}, indicates that calls to read {@link ByteString} or {@code byte[]}
+     * <strong>may</strong> return slices of the underlying buffer, rather than copies.
+     */
+    private boolean enableAliasing;
+    /** The global total message length limit */
+    private int totalBufferSize;
+    /** The amount of available data in the input beyond {@link #currentLimit}. */
+    private int bufferSizeAfterCurrentLimit;
+    /** The absolute position of the end of the current message. */
+    private int currentLimit = Integer.MAX_VALUE;
+    /** The last tag that was read from this stream. */
+    private int lastTag;
+    /** Total Bytes have been Read from the {@link Iterable} {@link ByteBuffer} */
+    private int totalBytesRead;
+    /** The start position offset of the whole message, used as to reset the totalBytesRead */
+    private int startOffset;
+    /** The current position for current ByteBuffer */
+    private long currentByteBufferPos;
+
+    private long currentByteBufferStartPos;
+    /**
+     * If the current ByteBuffer is unsafe-direct based, currentAddress is the start address of this
+     * ByteBuffer; otherwise should be zero.
+     */
+    private long currentAddress;
+    /** The limit position for current ByteBuffer */
+    private long currentByteBufferLimit;
+
+    /**
+     * The constructor of {@code Iterable<ByteBuffer>} decoder.
+     *
+     * @param inputBufs The input data.
+     * @param size The total size of the input data.
+     * @param immutableFlag whether the input data is immutable.
+     */
+    private IterableDirectByteBufferDecoder(
+        Iterable<ByteBuffer> inputBufs, int size, boolean immutableFlag) {
+      totalBufferSize = size;
+      input = inputBufs;
+      iterator = input.iterator();
+      immutable = immutableFlag;
+      startOffset = totalBytesRead = 0;
+      if (size == 0) {
+        currentByteBuffer = EMPTY_BYTE_BUFFER;
+        currentByteBufferPos = 0;
+        currentByteBufferStartPos = 0;
+        currentByteBufferLimit = 0;
+        currentAddress = 0;
+      } else {
+        tryGetNextByteBuffer();
+      }
+    }
+
+    /** To get the next ByteBuffer from {@code input}, and then update the parameters */
+    private void getNextByteBuffer() throws InvalidProtocolBufferException {
+      if (!iterator.hasNext()) {
+        throw InvalidProtocolBufferException.truncatedMessage();
+      }
+      tryGetNextByteBuffer();
+    }
+
+    private void tryGetNextByteBuffer() {
+      currentByteBuffer = iterator.next();
+      totalBytesRead += (int) (currentByteBufferPos - currentByteBufferStartPos);
+      currentByteBufferPos = currentByteBuffer.position();
+      currentByteBufferStartPos = currentByteBufferPos;
+      currentByteBufferLimit = currentByteBuffer.limit();
+      currentAddress = UnsafeUtil.addressOffset(currentByteBuffer);
+      currentByteBufferPos += currentAddress;
+      currentByteBufferStartPos += currentAddress;
+      currentByteBufferLimit += currentAddress;
+    }
+
+    @Override
+    public int readTag() throws IOException {
+      if (isAtEnd()) {
+        lastTag = 0;
+        return 0;
+      }
+
+      lastTag = readRawVarint32();
+      if (WireFormat.getTagFieldNumber(lastTag) == 0) {
+        // If we actually read zero (or any tag number corresponding to field
+        // number zero), that's not a valid tag.
+        throw InvalidProtocolBufferException.invalidTag();
+      }
+      return lastTag;
+    }
+
+    @Override
+    public void checkLastTagWas(final int value) throws InvalidProtocolBufferException {
+      if (lastTag != value) {
+        throw InvalidProtocolBufferException.invalidEndTag();
+      }
+    }
+
+    @Override
+    public int getLastTag() {
+      return lastTag;
+    }
+
+    @Override
+    public boolean skipField(final int tag) throws IOException {
+      switch (WireFormat.getTagWireType(tag)) {
+        case WireFormat.WIRETYPE_VARINT:
+          skipRawVarint();
+          return true;
+        case WireFormat.WIRETYPE_FIXED64:
+          skipRawBytes(FIXED64_SIZE);
+          return true;
+        case WireFormat.WIRETYPE_LENGTH_DELIMITED:
+          skipRawBytes(readRawVarint32());
+          return true;
+        case WireFormat.WIRETYPE_START_GROUP:
+          skipMessage();
+          checkLastTagWas(
+              WireFormat.makeTag(WireFormat.getTagFieldNumber(tag), WireFormat.WIRETYPE_END_GROUP));
+          return true;
+        case WireFormat.WIRETYPE_END_GROUP:
+          return false;
+        case WireFormat.WIRETYPE_FIXED32:
+          skipRawBytes(FIXED32_SIZE);
+          return true;
+        default:
+          throw InvalidProtocolBufferException.invalidWireType();
+      }
+    }
+
+    @Override
+    public boolean skipField(final int tag, final CodedOutputStream output) throws IOException {
+      switch (WireFormat.getTagWireType(tag)) {
+        case WireFormat.WIRETYPE_VARINT:
+          {
+            long value = readInt64();
+            output.writeRawVarint32(tag);
+            output.writeUInt64NoTag(value);
+            return true;
+          }
+        case WireFormat.WIRETYPE_FIXED64:
+          {
+            long value = readRawLittleEndian64();
+            output.writeRawVarint32(tag);
+            output.writeFixed64NoTag(value);
+            return true;
+          }
+        case WireFormat.WIRETYPE_LENGTH_DELIMITED:
+          {
+            ByteString value = readBytes();
+            output.writeRawVarint32(tag);
+            output.writeBytesNoTag(value);
+            return true;
+          }
+        case WireFormat.WIRETYPE_START_GROUP:
+          {
+            output.writeRawVarint32(tag);
+            skipMessage(output);
+            int endtag =
+                WireFormat.makeTag(
+                    WireFormat.getTagFieldNumber(tag), WireFormat.WIRETYPE_END_GROUP);
+            checkLastTagWas(endtag);
+            output.writeRawVarint32(endtag);
+            return true;
+          }
+        case WireFormat.WIRETYPE_END_GROUP:
+          {
+            return false;
+          }
+        case WireFormat.WIRETYPE_FIXED32:
+          {
+            int value = readRawLittleEndian32();
+            output.writeRawVarint32(tag);
+            output.writeFixed32NoTag(value);
+            return true;
+          }
+        default:
+          throw InvalidProtocolBufferException.invalidWireType();
+      }
+    }
+
+    @Override
+    public void skipMessage() throws IOException {
+      while (true) {
+        final int tag = readTag();
+        if (tag == 0 || !skipField(tag)) {
+          return;
+        }
+      }
+    }
+
+    @Override
+    public void skipMessage(CodedOutputStream output) throws IOException {
+      while (true) {
+        final int tag = readTag();
+        if (tag == 0 || !skipField(tag, output)) {
+          return;
+        }
+      }
+    }
+
+    // -----------------------------------------------------------------
+
+    @Override
+    public double readDouble() throws IOException {
+      return Double.longBitsToDouble(readRawLittleEndian64());
+    }
+
+    @Override
+    public float readFloat() throws IOException {
+      return Float.intBitsToFloat(readRawLittleEndian32());
+    }
+
+    @Override
+    public long readUInt64() throws IOException {
+      return readRawVarint64();
+    }
+
+    @Override
+    public long readInt64() throws IOException {
+      return readRawVarint64();
+    }
+
+    @Override
+    public int readInt32() throws IOException {
+      return readRawVarint32();
+    }
+
+    @Override
+    public long readFixed64() throws IOException {
+      return readRawLittleEndian64();
+    }
+
+    @Override
+    public int readFixed32() throws IOException {
+      return readRawLittleEndian32();
+    }
+
+    @Override
+    public boolean readBool() throws IOException {
+      return readRawVarint64() != 0;
+    }
+
+    @Override
+    public String readString() throws IOException {
+      final int size = readRawVarint32();
+      if (size > 0 && size <= currentByteBufferLimit - currentByteBufferPos) {
+        byte[] bytes = new byte[size];
+        UnsafeUtil.copyMemory(currentByteBufferPos, bytes, 0, size);
+        String result = new String(bytes, UTF_8);
+        currentByteBufferPos += size;
+        return result;
+      } else if (size > 0 && size <= remaining()) {
+        // TODO(yilunchong): To use an underlying bytes[] instead of allocating a new bytes[]
+        byte[] bytes = new byte[size];
+        readRawBytesTo(bytes, 0, size);
+        String result = new String(bytes, UTF_8);
+        return result;
+      }
+
+      if (size == 0) {
+        return "";
+      }
+      if (size < 0) {
+        throw InvalidProtocolBufferException.negativeSize();
+      }
+      throw InvalidProtocolBufferException.truncatedMessage();
+    }
+
+    @Override
+    public String readStringRequireUtf8() throws IOException {
+      final int size = readRawVarint32();
+      if (size > 0 && size <= currentByteBufferLimit - currentByteBufferPos) {
+        byte[] bytes = new byte[size];
+        UnsafeUtil.copyMemory(currentByteBufferPos, bytes, 0, size);
+        if (!Utf8.isValidUtf8(bytes)) {
+          throw InvalidProtocolBufferException.invalidUtf8();
+        }
+        String result = new String(bytes, UTF_8);
+        currentByteBufferPos += size;
+        return result;
+      }
+      if (size >= 0 && size <= remaining()) {
+        byte[] bytes = new byte[size];
+        readRawBytesTo(bytes, 0, size);
+        if (!Utf8.isValidUtf8(bytes)) {
+          throw InvalidProtocolBufferException.invalidUtf8();
+        }
+        String result = new String(bytes, UTF_8);
+        return result;
+      }
+
+      if (size == 0) {
+        return "";
+      }
+      if (size <= 0) {
+        throw InvalidProtocolBufferException.negativeSize();
+      }
+      throw InvalidProtocolBufferException.truncatedMessage();
+    }
+
+    @Override
+    public void readGroup(
+        final int fieldNumber,
+        final MessageLite.Builder builder,
+        final ExtensionRegistryLite extensionRegistry)
+        throws IOException {
+      if (recursionDepth >= recursionLimit) {
+        throw InvalidProtocolBufferException.recursionLimitExceeded();
+      }
+      ++recursionDepth;
+      builder.mergeFrom(this, extensionRegistry);
+      checkLastTagWas(WireFormat.makeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP));
+      --recursionDepth;
+    }
+
+
+    @Override
+    public <T extends MessageLite> T readGroup(
+        final int fieldNumber,
+        final Parser<T> parser,
+        final ExtensionRegistryLite extensionRegistry)
+        throws IOException {
+      if (recursionDepth >= recursionLimit) {
+        throw InvalidProtocolBufferException.recursionLimitExceeded();
+      }
+      ++recursionDepth;
+      T result = parser.parsePartialFrom(this, extensionRegistry);
+      checkLastTagWas(WireFormat.makeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP));
+      --recursionDepth;
+      return result;
+    }
+
+    @Deprecated
+    @Override
+    public void readUnknownGroup(final int fieldNumber, final MessageLite.Builder builder)
+        throws IOException {
+      readGroup(fieldNumber, builder, ExtensionRegistryLite.getEmptyRegistry());
+    }
+
+    @Override
+    public void readMessage(
+        final MessageLite.Builder builder, final ExtensionRegistryLite extensionRegistry)
+        throws IOException {
+      final int length = readRawVarint32();
+      if (recursionDepth >= recursionLimit) {
+        throw InvalidProtocolBufferException.recursionLimitExceeded();
+      }
+      final int oldLimit = pushLimit(length);
+      ++recursionDepth;
+      builder.mergeFrom(this, extensionRegistry);
+      checkLastTagWas(0);
+      --recursionDepth;
+      popLimit(oldLimit);
+    }
+
+
+    @Override
+    public <T extends MessageLite> T readMessage(
+        final Parser<T> parser, final ExtensionRegistryLite extensionRegistry) throws IOException {
+      int length = readRawVarint32();
+      if (recursionDepth >= recursionLimit) {
+        throw InvalidProtocolBufferException.recursionLimitExceeded();
+      }
+      final int oldLimit = pushLimit(length);
+      ++recursionDepth;
+      T result = parser.parsePartialFrom(this, extensionRegistry);
+      checkLastTagWas(0);
+      --recursionDepth;
+      popLimit(oldLimit);
+      return result;
+    }
+
+    @Override
+    public ByteString readBytes() throws IOException {
+      final int size = readRawVarint32();
+      if (size > 0 && size <= currentByteBufferLimit - currentByteBufferPos) {
+        if (immutable && enableAliasing) {
+          final int idx = (int) (currentByteBufferPos - currentAddress);
+          final ByteString result = ByteString.wrap(slice(idx, idx + size));
+          currentByteBufferPos += size;
+          return result;
+        } else {
+          byte[] bytes;
+          bytes = new byte[size];
+          UnsafeUtil.copyMemory(currentByteBufferPos, bytes, 0, size);
+          currentByteBufferPos += size;
+          return ByteString.wrap(bytes);
+        }
+      } else if (size > 0 && size <= remaining()) {
+        byte[] temp = new byte[size];
+        readRawBytesTo(temp, 0, size);
+        return ByteString.wrap(temp);
+      }
+
+      if (size == 0) {
+        return ByteString.EMPTY;
+      }
+      if (size < 0) {
+        throw InvalidProtocolBufferException.negativeSize();
+      }
+      throw InvalidProtocolBufferException.truncatedMessage();
+    }
+
+    @Override
+    public byte[] readByteArray() throws IOException {
+      return readRawBytes(readRawVarint32());
+    }
+
+    @Override
+    public ByteBuffer readByteBuffer() throws IOException {
+      final int size = readRawVarint32();
+      if (size > 0 && size <= currentRemaining()) {
+        if (!immutable && enableAliasing) {
+          currentByteBufferPos += size;
+          return slice(
+              (int) (currentByteBufferPos - currentAddress - size),
+              (int) (currentByteBufferPos - currentAddress));
+        } else {
+          byte[] bytes = new byte[size];
+          UnsafeUtil.copyMemory(currentByteBufferPos, bytes, 0, size);
+          currentByteBufferPos += size;
+          return ByteBuffer.wrap(bytes);
+        }
+      } else if (size > 0 && size <= remaining()) {
+        byte[] temp = new byte[size];
+        readRawBytesTo(temp, 0, size);
+        return ByteBuffer.wrap(temp);
+      }
+
+      if (size == 0) {
+        return EMPTY_BYTE_BUFFER;
+      }
+      if (size < 0) {
+        throw InvalidProtocolBufferException.negativeSize();
+      }
+      throw InvalidProtocolBufferException.truncatedMessage();
+    }
+
+    @Override
+    public int readUInt32() throws IOException {
+      return readRawVarint32();
+    }
+
+    @Override
+    public int readEnum() throws IOException {
+      return readRawVarint32();
+    }
+
+    @Override
+    public int readSFixed32() throws IOException {
+      return readRawLittleEndian32();
+    }
+
+    @Override
+    public long readSFixed64() throws IOException {
+      return readRawLittleEndian64();
+    }
+
+    @Override
+    public int readSInt32() throws IOException {
+      return decodeZigZag32(readRawVarint32());
+    }
+
+    @Override
+    public long readSInt64() throws IOException {
+      return decodeZigZag64(readRawVarint64());
+    }
+
+    @Override
+    public int readRawVarint32() throws IOException {
+      fastpath:
+      {
+        long tempPos = currentByteBufferPos;
+
+        if (currentByteBufferLimit == currentByteBufferPos) {
+          break fastpath;
+        }
+
+        int x;
+        if ((x = UnsafeUtil.getByte(tempPos++)) >= 0) {
+          currentByteBufferPos++;
+          return x;
+        } else if (currentByteBufferLimit - currentByteBufferPos < 10) {
+          break fastpath;
+        } else if ((x ^= (UnsafeUtil.getByte(tempPos++) << 7)) < 0) {
+          x ^= (~0 << 7);
+        } else if ((x ^= (UnsafeUtil.getByte(tempPos++) << 14)) >= 0) {
+          x ^= (~0 << 7) ^ (~0 << 14);
+        } else if ((x ^= (UnsafeUtil.getByte(tempPos++) << 21)) < 0) {
+          x ^= (~0 << 7) ^ (~0 << 14) ^ (~0 << 21);
+        } else {
+          int y = UnsafeUtil.getByte(tempPos++);
+          x ^= y << 28;
+          x ^= (~0 << 7) ^ (~0 << 14) ^ (~0 << 21) ^ (~0 << 28);
+          if (y < 0
+              && UnsafeUtil.getByte(tempPos++) < 0
+              && UnsafeUtil.getByte(tempPos++) < 0
+              && UnsafeUtil.getByte(tempPos++) < 0
+              && UnsafeUtil.getByte(tempPos++) < 0
+              && UnsafeUtil.getByte(tempPos++) < 0) {
+            break fastpath; // Will throw malformedVarint()
+          }
+        }
+        currentByteBufferPos = tempPos;
+        return x;
+      }
+      return (int) readRawVarint64SlowPath();
+    }
+
+    @Override
+    public long readRawVarint64() throws IOException {
+      fastpath:
+      {
+        long tempPos = currentByteBufferPos;
+
+        if (currentByteBufferLimit == currentByteBufferPos) {
+          break fastpath;
+        }
+
+        long x;
+        int y;
+        if ((y = UnsafeUtil.getByte(tempPos++)) >= 0) {
+          currentByteBufferPos++;
+          return y;
+        } else if (currentByteBufferLimit - currentByteBufferPos < 10) {
+          break fastpath;
+        } else if ((y ^= (UnsafeUtil.getByte(tempPos++) << 7)) < 0) {
+          x = y ^ (~0 << 7);
+        } else if ((y ^= (UnsafeUtil.getByte(tempPos++) << 14)) >= 0) {
+          x = y ^ ((~0 << 7) ^ (~0 << 14));
+        } else if ((y ^= (UnsafeUtil.getByte(tempPos++) << 21)) < 0) {
+          x = y ^ ((~0 << 7) ^ (~0 << 14) ^ (~0 << 21));
+        } else if ((x = y ^ ((long) UnsafeUtil.getByte(tempPos++) << 28)) >= 0L) {
+          x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28);
+        } else if ((x ^= ((long) UnsafeUtil.getByte(tempPos++) << 35)) < 0L) {
+          x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28) ^ (~0L << 35);
+        } else if ((x ^= ((long) UnsafeUtil.getByte(tempPos++) << 42)) >= 0L) {
+          x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28) ^ (~0L << 35) ^ (~0L << 42);
+        } else if ((x ^= ((long) UnsafeUtil.getByte(tempPos++) << 49)) < 0L) {
+          x ^=
+              (~0L << 7)
+                  ^ (~0L << 14)
+                  ^ (~0L << 21)
+                  ^ (~0L << 28)
+                  ^ (~0L << 35)
+                  ^ (~0L << 42)
+                  ^ (~0L << 49);
+        } else {
+          x ^= ((long) UnsafeUtil.getByte(tempPos++) << 56);
+          x ^=
+              (~0L << 7)
+                  ^ (~0L << 14)
+                  ^ (~0L << 21)
+                  ^ (~0L << 28)
+                  ^ (~0L << 35)
+                  ^ (~0L << 42)
+                  ^ (~0L << 49)
+                  ^ (~0L << 56);
+          if (x < 0L) {
+            if (UnsafeUtil.getByte(tempPos++) < 0L) {
+              break fastpath; // Will throw malformedVarint()
+            }
+          }
+        }
+        currentByteBufferPos = tempPos;
+        return x;
+      }
+      return readRawVarint64SlowPath();
+    }
+
+    @Override
+    long readRawVarint64SlowPath() throws IOException {
+      long result = 0;
+      for (int shift = 0; shift < 64; shift += 7) {
+        final byte b = readRawByte();
+        result |= (long) (b & 0x7F) << shift;
+        if ((b & 0x80) == 0) {
+          return result;
+        }
+      }
+      throw InvalidProtocolBufferException.malformedVarint();
+    }
+
+    @Override
+    public int readRawLittleEndian32() throws IOException {
+      if (currentRemaining() >= FIXED32_SIZE) {
+        long tempPos = currentByteBufferPos;
+        currentByteBufferPos += FIXED32_SIZE;
+        return (((UnsafeUtil.getByte(tempPos) & 0xff))
+            | ((UnsafeUtil.getByte(tempPos + 1) & 0xff) << 8)
+            | ((UnsafeUtil.getByte(tempPos + 2) & 0xff) << 16)
+            | ((UnsafeUtil.getByte(tempPos + 3) & 0xff) << 24));
+      }
+      return ((readRawByte() & 0xff)
+          | ((readRawByte() & 0xff) << 8)
+          | ((readRawByte() & 0xff) << 16)
+          | ((readRawByte() & 0xff) << 24));
+    }
+
+    @Override
+    public long readRawLittleEndian64() throws IOException {
+      if (currentRemaining() >= FIXED64_SIZE) {
+        long tempPos = currentByteBufferPos;
+        currentByteBufferPos += FIXED64_SIZE;
+        return (((UnsafeUtil.getByte(tempPos) & 0xffL))
+            | ((UnsafeUtil.getByte(tempPos + 1) & 0xffL) << 8)
+            | ((UnsafeUtil.getByte(tempPos + 2) & 0xffL) << 16)
+            | ((UnsafeUtil.getByte(tempPos + 3) & 0xffL) << 24)
+            | ((UnsafeUtil.getByte(tempPos + 4) & 0xffL) << 32)
+            | ((UnsafeUtil.getByte(tempPos + 5) & 0xffL) << 40)
+            | ((UnsafeUtil.getByte(tempPos + 6) & 0xffL) << 48)
+            | ((UnsafeUtil.getByte(tempPos + 7) & 0xffL) << 56));
+      }
+      return ((readRawByte() & 0xffL)
+          | ((readRawByte() & 0xffL) << 8)
+          | ((readRawByte() & 0xffL) << 16)
+          | ((readRawByte() & 0xffL) << 24)
+          | ((readRawByte() & 0xffL) << 32)
+          | ((readRawByte() & 0xffL) << 40)
+          | ((readRawByte() & 0xffL) << 48)
+          | ((readRawByte() & 0xffL) << 56));
+    }
+
+    @Override
+    public void enableAliasing(boolean enabled) {
+      this.enableAliasing = enabled;
+    }
+
+    @Override
+    public void resetSizeCounter() {
+      startOffset = (int) (totalBytesRead + currentByteBufferPos - currentByteBufferStartPos);
+    }
+
+    @Override
+    public int pushLimit(int byteLimit) throws InvalidProtocolBufferException {
+      if (byteLimit < 0) {
+        throw InvalidProtocolBufferException.negativeSize();
+      }
+      byteLimit += getTotalBytesRead();
+      final int oldLimit = currentLimit;
+      if (byteLimit > oldLimit) {
+        throw InvalidProtocolBufferException.truncatedMessage();
+      }
+      currentLimit = byteLimit;
+
+      recomputeBufferSizeAfterLimit();
+
+      return oldLimit;
+    }
+
+    private void recomputeBufferSizeAfterLimit() {
+      totalBufferSize += bufferSizeAfterCurrentLimit;
+      final int bufferEnd = totalBufferSize - startOffset;
+      if (bufferEnd > currentLimit) {
+        // Limit is in current buffer.
+        bufferSizeAfterCurrentLimit = bufferEnd - currentLimit;
+        totalBufferSize -= bufferSizeAfterCurrentLimit;
+      } else {
+        bufferSizeAfterCurrentLimit = 0;
+      }
+    }
+
+    @Override
+    public void popLimit(final int oldLimit) {
+      currentLimit = oldLimit;
+      recomputeBufferSizeAfterLimit();
+    }
+
+    @Override
+    public int getBytesUntilLimit() {
+      if (currentLimit == Integer.MAX_VALUE) {
+        return -1;
+      }
+
+      return currentLimit - getTotalBytesRead();
+    }
+
+    @Override
+    public boolean isAtEnd() throws IOException {
+      return totalBytesRead + currentByteBufferPos - currentByteBufferStartPos == totalBufferSize;
+    }
+
+    @Override
+    public int getTotalBytesRead() {
+      return (int)
+          (totalBytesRead - startOffset + currentByteBufferPos - currentByteBufferStartPos);
+    }
+
+    @Override
+    public byte readRawByte() throws IOException {
+      if (currentRemaining() == 0) {
+        getNextByteBuffer();
+      }
+      return UnsafeUtil.getByte(currentByteBufferPos++);
+    }
+
+    @Override
+    public byte[] readRawBytes(final int length) throws IOException {
+      if (length >= 0 && length <= currentRemaining()) {
+        byte[] bytes = new byte[length];
+        UnsafeUtil.copyMemory(currentByteBufferPos, bytes, 0, length);
+        currentByteBufferPos += length;
+        return bytes;
+      }
+      if (length >= 0 && length <= remaining()) {
+        byte[] bytes = new byte[length];
+        readRawBytesTo(bytes, 0, length);
+        return bytes;
+      }
+
+      if (length <= 0) {
+        if (length == 0) {
+          return EMPTY_BYTE_ARRAY;
+        } else {
+          throw InvalidProtocolBufferException.negativeSize();
+        }
+      }
+
+      throw InvalidProtocolBufferException.truncatedMessage();
+    }
+
+    /**
+     * Try to get raw bytes from {@code input} with the size of {@code length} and copy to {@code
+     * bytes} array. If the size is bigger than the number of remaining bytes in the input, then
+     * throw {@code truncatedMessage} exception.
+     *
+     * @param bytes
+     * @param offset
+     * @param length
+     * @throws IOException
+     */
+    private void readRawBytesTo(byte[] bytes, int offset, final int length) throws IOException {
+      if (length >= 0 && length <= remaining()) {
+        int l = length;
+        while (l > 0) {
+          if (currentRemaining() == 0) {
+            getNextByteBuffer();
+          }
+          int bytesToCopy = Math.min(l, (int) currentRemaining());
+          UnsafeUtil.copyMemory(currentByteBufferPos, bytes, length - l + offset, bytesToCopy);
+          l -= bytesToCopy;
+          currentByteBufferPos += bytesToCopy;
+        }
+        return;
+      }
+
+      if (length <= 0) {
+        if (length == 0) {
+          return;
+        } else {
+          throw InvalidProtocolBufferException.negativeSize();
+        }
+      }
+      throw InvalidProtocolBufferException.truncatedMessage();
+    }
+
+    @Override
+    public void skipRawBytes(final int length) throws IOException {
+      if (length >= 0
+          && length
+              <= (totalBufferSize
+                  - totalBytesRead
+                  - currentByteBufferPos
+                  + currentByteBufferStartPos)) {
+        // We have all the bytes we need already.
+        int l = length;
+        while (l > 0) {
+          if (currentRemaining() == 0) {
+            getNextByteBuffer();
+          }
+          int rl = Math.min(l, (int) currentRemaining());
+          l -= rl;
+          currentByteBufferPos += rl;
+        }
+        return;
+      }
+
+      if (length < 0) {
+        throw InvalidProtocolBufferException.negativeSize();
+      }
+      throw InvalidProtocolBufferException.truncatedMessage();
+    }
+
+    // TODO: optimize to fastpath
+    private void skipRawVarint() throws IOException {
+      for (int i = 0; i < MAX_VARINT_SIZE; i++) {
+        if (readRawByte() >= 0) {
+          return;
+        }
+      }
+      throw InvalidProtocolBufferException.malformedVarint();
+    }
+
+    /**
+     * Try to get the number of remaining bytes in {@code input}.
+     *
+     * @return the number of remaining bytes in {@code input}.
+     */
+    private int remaining() {
+      return (int)
+          (totalBufferSize - totalBytesRead - currentByteBufferPos + currentByteBufferStartPos);
+    }
+
+    /**
+     * Try to get the number of remaining bytes in {@code currentByteBuffer}.
+     *
+     * @return the number of remaining bytes in {@code currentByteBuffer}
+     */
+    private long currentRemaining() {
+      return (currentByteBufferLimit - currentByteBufferPos);
+    }
+
+    private ByteBuffer slice(int begin, int end) throws IOException {
+      int prevPos = currentByteBuffer.position();
+      int prevLimit = currentByteBuffer.limit();
+      try {
+        currentByteBuffer.position(begin);
+        currentByteBuffer.limit(end);
+        return currentByteBuffer.slice();
+      } catch (IllegalArgumentException e) {
+        throw InvalidProtocolBufferException.truncatedMessage();
+      } finally {
+        currentByteBuffer.position(prevPos);
+        currentByteBuffer.limit(prevLimit);
+      }
+    }
+  }
 }

java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java

@@ -220,7 +220,7 @@ public abstract class GeneratedMessageLite<
 
   @Override
   public final boolean isInitialized() {
-    return dynamicMethod(MethodToInvoke.IS_INITIALIZED, Boolean.TRUE) != null;
+    return isInitialized((MessageType) this, Boolean.TRUE);
   }
 
   @Override
@@ -240,6 +240,8 @@ public abstract class GeneratedMessageLite<
   public static enum MethodToInvoke {
     // Rely on/modify instance state
     IS_INITIALIZED,
+    GET_MEMOIZED_IS_INITIALIZED,
+    SET_MEMOIZED_IS_INITIALIZED,
     VISIT,
     MERGE_FROM_STREAM,
     MAKE_IMMUTABLE,
@@ -256,25 +258,30 @@ public abstract class GeneratedMessageLite<
    * Theses different kinds of operations are required to implement message-level operations for
    * builders in the runtime. This method bundles those operations to reduce the generated methods
    * count.
+   *
    * <ul>
-   * <li>{@code MERGE_FROM_STREAM} is parameterized with an {@link CodedInputStream} and
-   * {@link ExtensionRegistryLite}. It consumes the input stream, parsing the contents into the
-   * returned protocol buffer. If parsing throws an {@link InvalidProtocolBufferException}, the
-   * implementation wraps it in a RuntimeException.
-   * <li>{@code NEW_INSTANCE} returns a new instance of the protocol buffer that has not yet been
-   * made immutable. See {@code MAKE_IMMUTABLE}.
-   * <li>{@code IS_INITIALIZED} is parameterized with a {@code Boolean} detailing whether to
-   * memoize. It returns {@code null} for false and the default instance for true. We optionally
-   * memoize to support the Builder case, where memoization is not desired.
-   * <li>{@code NEW_BUILDER} returns a {@code BuilderType} instance.
-   * <li>{@code VISIT} is parameterized with a {@code Visitor} and a {@code MessageType} and
-   * recursively iterates through the fields side by side between this and the instance.
-   * <li>{@code MAKE_IMMUTABLE} sets all internal fields to an immutable state.
+   *   <li>{@code MERGE_FROM_STREAM} is parameterized with an {@link CodedInputStream} and {@link
+   *       ExtensionRegistryLite}. It consumes the input stream, parsing the contents into the
+   *       returned protocol buffer. If parsing throws an {@link InvalidProtocolBufferException},
+   *       the implementation wraps it in a RuntimeException.
+   *   <li>{@code NEW_INSTANCE} returns a new instance of the protocol buffer that has not yet been
+   *       made immutable. See {@code MAKE_IMMUTABLE}.
+   *   <li>{@code IS_INITIALIZED} returns {@code null} for false and the default instance for true.
+   *       It doesn't use or modify any memoized value.
+   *   <li>{@code GET_MEMOIZED_IS_INITIALIZED} returns the memoized {@code isInitialized} byte
+   *       value.
+   *   <li>{@code SET_MEMOIZED_IS_INITIALIZED} sets the memoized {@code isInitilaized} byte value to
+   *       1 if the first parameter is not null, or to 0 if the first parameter is null.
+   *   <li>{@code NEW_BUILDER} returns a {@code BuilderType} instance.
+   *   <li>{@code VISIT} is parameterized with a {@code Visitor} and a {@code MessageType} and
+   *       recursively iterates through the fields side by side between this and the instance.
+   *   <li>{@code MAKE_IMMUTABLE} sets all internal fields to an immutable state.
    * </ul>
+   *
    * This method, plus the implementation of the Builder, enables the Builder class to be proguarded
    * away entirely on Android.
-   * <p>
-   * For use by generated code only.
+   *
+   * <p>For use by generated code only.
    */
   protected abstract Object dynamicMethod(MethodToInvoke method, Object arg0, Object arg1);
 
@@ -297,9 +304,9 @@ public abstract class GeneratedMessageLite<
     unknownFields = visitor.visitUnknownFields(unknownFields, other.unknownFields);
   }
 
+
   /**
-   * Merge some unknown fields into the {@link UnknownFieldSetLite} for this
-   * message.
+   * Merge some unknown fields into the {@link UnknownFieldSetLite} for this message.
    *
    * <p>For use by generated code only.
    */
@@ -1403,7 +1410,21 @@ public abstract class GeneratedMessageLite<
    */
   protected static final <T extends GeneratedMessageLite<T, ?>> boolean isInitialized(
       T message, boolean shouldMemoize) {
-    return message.dynamicMethod(MethodToInvoke.IS_INITIALIZED, shouldMemoize) != null;
+    byte memoizedIsInitialized =
+        (Byte) message.dynamicMethod(MethodToInvoke.GET_MEMOIZED_IS_INITIALIZED);
+    if (memoizedIsInitialized == 1) {
+      return true;
+    }
+    if (memoizedIsInitialized == 0) {
+      return false;
+    }
+    boolean isInitialized =
+        message.dynamicMethod(MethodToInvoke.IS_INITIALIZED, Boolean.FALSE) != null;
+    if (shouldMemoize) {
+      message.dynamicMethod(
+          MethodToInvoke.SET_MEMOIZED_IS_INITIALIZED, isInitialized ? message : null);
+    }
+    return isInitialized;
   }
 
   protected static final <T extends GeneratedMessageLite<T, ?>> void makeImmutable(T message) {

java/core/src/main/java/com/google/protobuf/IterableByteBufferInputStream.java

@@ -0,0 +1,150 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package com.google.protobuf;
+
+import static com.google.protobuf.Internal.EMPTY_BYTE_BUFFER;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+
+class IterableByteBufferInputStream extends InputStream {
+  /** The {@link Iterator} with type {@link ByteBuffer} of {@code input} */
+  private Iterator<ByteBuffer> iterator;
+  /** The current ByteBuffer; */
+  private ByteBuffer currentByteBuffer;
+  /** The number of ByteBuffers in the input data. */
+  private int dataSize;
+  /**
+   * Current {@code ByteBuffer}'s index
+   *
+   * <p>If index equals dataSize, then all the data in the InputStream has been consumed
+   */
+  private int currentIndex;
+  /** The current position for current ByteBuffer */
+  private int currentByteBufferPos;
+  /** Whether current ByteBuffer has an array */
+  private boolean hasArray;
+  /**
+   * If the current ByteBuffer is unsafe-direct based, currentArray is null; otherwise should be the
+   * array inside ByteBuffer.
+   */
+  private byte[] currentArray;
+  /** Current ByteBuffer's array offset */
+  private int currentArrayOffset;
+  /**
+   * If the current ByteBuffer is unsafe-direct based, currentAddress is the start address of this
+   * ByteBuffer; otherwise should be zero.
+   */
+  private long currentAddress;
+
+  IterableByteBufferInputStream(Iterable<ByteBuffer> data) {
+    iterator = data.iterator();
+    dataSize = 0;
+    for (ByteBuffer unused : data) {
+      dataSize++;
+    }
+    currentIndex = -1;
+
+    if (!getNextByteBuffer()) {
+      currentByteBuffer = EMPTY_BYTE_BUFFER;
+      currentIndex = 0;
+      currentByteBufferPos = 0;
+      currentAddress = 0;
+    }
+  }
+
+  private boolean getNextByteBuffer() {
+    currentIndex++;
+    if (!iterator.hasNext()) {
+      return false;
+    }
+    currentByteBuffer = iterator.next();
+    currentByteBufferPos = currentByteBuffer.position();
+    if (currentByteBuffer.hasArray()) {
+      hasArray = true;
+      currentArray = currentByteBuffer.array();
+      currentArrayOffset = currentByteBuffer.arrayOffset();
+    } else {
+      hasArray = false;
+      currentAddress = UnsafeUtil.addressOffset(currentByteBuffer);
+      currentArray = null;
+    }
+    return true;
+  }
+
+  private void updateCurrentByteBufferPos(int numberOfBytesRead) {
+    currentByteBufferPos += numberOfBytesRead;
+    if (currentByteBufferPos == currentByteBuffer.limit()) {
+      getNextByteBuffer();
+    }
+  }
+
+  @Override
+  public int read() throws IOException {
+    if (currentIndex == dataSize) {
+      return -1;
+    }
+    if (hasArray) {
+      int result = currentArray[currentByteBufferPos + currentArrayOffset] & 0xFF;
+      updateCurrentByteBufferPos(1);
+      return result;
+    } else {
+      int result = UnsafeUtil.getByte(currentByteBufferPos + currentAddress) & 0xFF;
+      updateCurrentByteBufferPos(1);
+      return result;
+    }
+  }
+
+  @Override
+  public int read(byte[] output, int offset, int length) throws IOException {
+    if (currentIndex == dataSize) {
+      return -1;
+    }
+    int remaining = currentByteBuffer.limit() - currentByteBufferPos;
+    if (length > remaining) {
+      length = remaining;
+    }
+    if (hasArray) {
+      System.arraycopy(
+          currentArray, currentByteBufferPos + currentArrayOffset, output, offset, length);
+      updateCurrentByteBufferPos(length);
+    } else {
+      int prevPos = currentByteBuffer.position();
+      currentByteBuffer.position(currentByteBufferPos);
+      currentByteBuffer.get(output, offset, length);
+      currentByteBuffer.position(prevPos);
+      updateCurrentByteBufferPos(length);
+    }
+    return length;
+  }
+}

java/core/src/main/java/com/google/protobuf/SmallSortedMap.java

@@ -540,8 +540,8 @@ class SmallSortedMap<K extends Comparable<K>, V> extends AbstractMap<K, V> {
 
     @Override
     public boolean hasNext() {
-      return (pos + 1) < entryList.size() ||
-          getOverflowIterator().hasNext();
+      return (pos + 1) < entryList.size()
+          || (!overflowEntries.isEmpty() && getOverflowIterator().hasNext());
     }
 
     @Override

java/core/src/main/java/com/google/protobuf/TextFormat.java

@@ -279,9 +279,21 @@ public final class TextFormat {
         generator.print(String.format((Locale) null, "0x%016x", (Long) value));
         break;
       case WireFormat.WIRETYPE_LENGTH_DELIMITED:
-        generator.print("\"");
-        generator.print(escapeBytes((ByteString) value));
-        generator.print("\"");
+        try {
+          // Try to parse and print the field as an embedded message
+          UnknownFieldSet message = UnknownFieldSet.parseFrom((ByteString) value);
+          generator.print("{");
+          generator.eol();
+          generator.indent();
+          Printer.DEFAULT.printUnknownFields(message, generator);
+          generator.outdent();
+          generator.print("}");
+        } catch (InvalidProtocolBufferException e) {
+          // If not parseable as a message, print as a String
+          generator.print("\"");
+          generator.print(escapeBytes((ByteString) value));
+          generator.print("\"");
+        }
         break;
       case WireFormat.WIRETYPE_START_GROUP:
         Printer.DEFAULT.printUnknownFields((UnknownFieldSet) value, generator);

java/core/src/main/java/com/google/protobuf/UnknownFieldSetLite.java

@@ -81,7 +81,7 @@ public final class UnknownFieldSetLite {
     System.arraycopy(second.objects, 0, objects, first.count, second.count);
     return new UnknownFieldSetLite(count, tags, objects, true /* isMutable */);
   }
-  
+
   /**
    * The number of elements in the set.
    */
@@ -323,6 +323,7 @@ public final class UnknownFieldSetLite {
 
   // Package private for unsafe experimental runtime.
   void storeField(int tag, Object value) {
+    checkMutable();
     ensureCapacity();
     
     tags[count] = tag;

java/core/src/main/java/com/google/protobuf/UnsafeUtil.java

@@ -252,10 +252,6 @@ final class UnsafeUtil {
     MEMORY_ACCESSOR.putLong(address, value);
   }
 
-  static void copyMemory(long srcAddress, long targetAddress, long length) {
-    MEMORY_ACCESSOR.copyMemory(srcAddress, targetAddress, length);
-  }
-
   /**
    * Gets the offset of the {@code address} field of the given direct {@link ByteBuffer}.
    */
@@ -478,8 +474,6 @@ final class UnsafeUtil {
 
     public abstract void putLong(long address, long value);
 
-    public abstract void copyMemory(long srcAddress, long targetAddress, long length);
-
     public abstract Object getStaticObject(Field field);
     
     public abstract void copyMemory(long srcOffset, byte[] target, long targetIndex, long length);
@@ -562,11 +556,6 @@ final class UnsafeUtil {
     public void putDouble(Object target, long offset, double value) {
       unsafe.putDouble(target, offset, value);
     }
-
-    @Override
-    public void copyMemory(long srcAddress, long targetAddress, long length) {
-      unsafe.copyMemory(srcAddress, targetAddress, length);
-    }
     
     @Override 
     public void copyMemory(long srcOffset, byte[] target, long targetIndex, long length) {

java/core/src/test/java/com/google/protobuf/CodedInputStreamTest.java

@@ -41,6 +41,7 @@ import java.io.FilterInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.ByteBuffer;
+import java.util.ArrayList;
 import java.util.Arrays;
 import junit.framework.TestCase;
 
@@ -50,6 +51,9 @@ import junit.framework.TestCase;
  * @author kenton@google.com Kenton Varda
  */
 public class CodedInputStreamTest extends TestCase {
+  
+  private static final int DEFAULT_BLOCK_SIZE = 4096; 
+  
   private enum InputType {
     ARRAY {
       @Override
@@ -77,7 +81,43 @@ public class CodedInputStreamTest extends TestCase {
       CodedInputStream newDecoder(byte[] data, int blockSize) {
         return CodedInputStream.newInstance(new SmallBlockInputStream(data, blockSize));
       }
+    },
+    ITER_DIRECT {
+      @Override
+      CodedInputStream newDecoder(byte[] data, int blockSize) {
+        if (blockSize > DEFAULT_BLOCK_SIZE) {
+          blockSize = DEFAULT_BLOCK_SIZE;
+        }
+        ArrayList <ByteBuffer> input = new ArrayList <ByteBuffer>();
+        for (int i = 0; i < data.length; i += blockSize) {
+          int rl = Math.min(blockSize, data.length - i); 
+          ByteBuffer rb = ByteBuffer.allocateDirect(rl); 
+          rb.put(data, i, rl);
+          rb.flip();
+          input.add(rb);
+        }
+        return CodedInputStream.newInstance(input);
+      }
+    },
+    STREAM_ITER_DIRECT {
+      @Override
+      CodedInputStream newDecoder(byte[] data, int blockSize) {
+        if (blockSize > DEFAULT_BLOCK_SIZE) {
+          blockSize = DEFAULT_BLOCK_SIZE;
+        }
+        ArrayList <ByteBuffer> input = new ArrayList <ByteBuffer>();
+        for (int i = 0; i < data.length; i += blockSize) {
+          int rl = Math.min(blockSize, data.length - i); 
+          ByteBuffer rb = ByteBuffer.allocateDirect(rl); 
+          rb.put(data, i, rl);
+          rb.flip();
+          input.add(rb);
+        }
+        return CodedInputStream.newInstance(new IterableByteBufferInputStream(input));
+      }
     };
+    
+    
 
     CodedInputStream newDecoder(byte[] data) {
       return newDecoder(data, data.length);
@@ -994,7 +1034,9 @@ public class CodedInputStreamTest extends TestCase {
     byte[] data = byteArrayStream.toByteArray();
 
     for (InputType inputType : InputType.values()) {
-      if (inputType == InputType.STREAM) {
+      if (inputType == InputType.STREAM 
+          || inputType == InputType.STREAM_ITER_DIRECT 
+          || inputType == InputType.ITER_DIRECT) {
         // Aliasing doesn't apply to stream-backed CIS.
         continue;
       }
@@ -1019,7 +1061,7 @@ public class CodedInputStreamTest extends TestCase {
       assertEquals(inputType.name(), (byte) 89, result.get());
 
       // Enable aliasing
-      inputStream = inputType.newDecoder(data);
+      inputStream = inputType.newDecoder(data, data.length);
       inputStream.enableAliasing(true);
       result = inputStream.readByteBuffer();
       assertEquals(inputType.name(), 0, result.capacity());

java/core/src/test/java/com/google/protobuf/GeneratedMessageTest.java

@@ -32,19 +32,14 @@ package com.google.protobuf;
 
 import com.google.protobuf.Descriptors.Descriptor;
 import com.google.protobuf.Descriptors.FieldDescriptor;
-import com.google.protobuf.UnittestLite.TestAllExtensionsLite;
 import com.google.protobuf.test.UnittestImport;
 import protobuf_unittest.EnumWithNoOuter;
 import protobuf_unittest.MessageWithNoOuter;
 import protobuf_unittest.MultipleFilesTestProto;
 import protobuf_unittest.NestedExtension.MyNestedExtension;
-import protobuf_unittest.NestedExtensionLite.MyNestedExtensionLite;
 import protobuf_unittest.NonNestedExtension;
 import protobuf_unittest.NonNestedExtension.MessageToBeExtended;
 import protobuf_unittest.NonNestedExtension.MyNonNestedExtension;
-import protobuf_unittest.NonNestedExtensionLite;
-import protobuf_unittest.NonNestedExtensionLite.MessageLiteToBeExtended;
-import protobuf_unittest.NonNestedExtensionLite.MyNonNestedExtensionLite;
 import protobuf_unittest.OuterClassNameTest2OuterClass;
 import protobuf_unittest.OuterClassNameTest3OuterClass;
 import protobuf_unittest.OuterClassNameTestOuterClass;
@@ -711,70 +706,6 @@ public class GeneratedMessageTest extends TestCase {
         1, (int) merged.getExtension(UnittestProto.optionalInt32Extension));
   }
 
-  // =================================================================
-  // Lite Extensions.
-
-  // We test lite extensions directly because they have a separate
-  // implementation from full extensions.  In contrast, we do not test
-  // lite fields directly since they are implemented exactly the same as
-  // regular fields.
-
-  public void testLiteExtensionMessageOrBuilder() throws Exception {
-    TestAllExtensionsLite.Builder builder = TestAllExtensionsLite.newBuilder();
-    TestUtilLite.setAllExtensions(builder);
-    TestUtil.assertAllExtensionsSet(builder);
-
-    TestAllExtensionsLite message = builder.build();
-    TestUtil.assertAllExtensionsSet(message);
-  }
-
-  public void testLiteExtensionRepeatedSetters() throws Exception {
-    TestAllExtensionsLite.Builder builder = TestAllExtensionsLite.newBuilder();
-    TestUtilLite.setAllExtensions(builder);
-    TestUtilLite.modifyRepeatedExtensions(builder);
-    TestUtil.assertRepeatedExtensionsModified(builder);
-
-    TestAllExtensionsLite message = builder.build();
-    TestUtil.assertRepeatedExtensionsModified(message);
-  }
-
-  public void testLiteExtensionDefaults() throws Exception {
-    TestUtil.assertExtensionsClear(TestAllExtensionsLite.getDefaultInstance());
-    TestUtil.assertExtensionsClear(TestAllExtensionsLite.newBuilder().build());
-  }
-
-  public void testClearLiteExtension() throws Exception {
-    // clearExtension() is not actually used in TestUtil, so try it manually.
-    assertFalse(
-      TestAllExtensionsLite.newBuilder()
-        .setExtension(UnittestLite.optionalInt32ExtensionLite, 1)
-        .clearExtension(UnittestLite.optionalInt32ExtensionLite)
-        .hasExtension(UnittestLite.optionalInt32ExtensionLite));
-    assertEquals(0,
-      TestAllExtensionsLite.newBuilder()
-        .addExtension(UnittestLite.repeatedInt32ExtensionLite, 1)
-        .clearExtension(UnittestLite.repeatedInt32ExtensionLite)
-        .getExtensionCount(UnittestLite.repeatedInt32ExtensionLite));
-  }
-
-  public void testLiteExtensionCopy() throws Exception {
-    TestAllExtensionsLite original = TestUtilLite.getAllLiteExtensionsSet();
-    TestAllExtensionsLite copy =
-        TestAllExtensionsLite.newBuilder(original).build();
-    TestUtil.assertAllExtensionsSet(copy);
-  }
-
-  public void testLiteExtensionMergeFrom() throws Exception {
-    TestAllExtensionsLite original =
-      TestAllExtensionsLite.newBuilder()
-        .setExtension(UnittestLite.optionalInt32ExtensionLite, 1).build();
-    TestAllExtensionsLite merged =
-        TestAllExtensionsLite.newBuilder().mergeFrom(original).build();
-    assertTrue(merged.hasExtension(UnittestLite.optionalInt32ExtensionLite));
-    assertEquals(
-        1, (int) merged.getExtension(UnittestLite.optionalInt32ExtensionLite));
-  }
-
   // =================================================================
   // multiple_files_test
 
@@ -942,16 +873,6 @@ public class GeneratedMessageTest extends TestCase {
                  MyNestedExtension.recursiveExtension.getDescriptor().getName());
   }
 
-  public void testNonNestedExtensionLiteInitialization() {
-    assertTrue(NonNestedExtensionLite.nonNestedExtensionLite
-               .getMessageDefaultInstance() instanceof MyNonNestedExtensionLite);
-  }
-
-  public void testNestedExtensionLiteInitialization() {
-    assertTrue(MyNestedExtensionLite.recursiveExtensionLite
-               .getMessageDefaultInstance() instanceof MessageLiteToBeExtended);
-  }
-
   public void testInvalidations() throws Exception {
     GeneratedMessage.enableAlwaysUseFieldBuildersForTesting();
     TestAllTypes.NestedMessage nestedMessage1 =

java/core/src/test/java/com/google/protobuf/LiteTest.java

@@ -1453,6 +1453,36 @@ public class LiteTest extends TestCase {
             UnittestLite.optionalFixed32ExtensionLite));
   }
 
+  // Builder.mergeFrom() should keep existing extensions.
+  public void testBuilderMergeFromWithExtensions() throws Exception {
+    TestAllExtensionsLite message =
+        TestAllExtensionsLite.newBuilder()
+            .addExtension(UnittestLite.repeatedInt32ExtensionLite, 12)
+            .build();
+
+    ExtensionRegistryLite registry = ExtensionRegistryLite.newInstance();
+    UnittestLite.registerAllExtensions(registry);
+
+    TestAllExtensionsLite.Builder builder = TestAllExtensionsLite.newBuilder();
+    builder.mergeFrom(message.toByteArray(), registry);
+    builder.mergeFrom(message.toByteArray(), registry);
+    TestAllExtensionsLite result = builder.build();
+    assertEquals(2, result.getExtensionCount(UnittestLite.repeatedInt32ExtensionLite));
+    assertEquals(12, result.getExtension(UnittestLite.repeatedInt32ExtensionLite, 0).intValue());
+    assertEquals(12, result.getExtension(UnittestLite.repeatedInt32ExtensionLite, 1).intValue());
+  }
+
+  // Builder.mergeFrom() should keep existing unknown fields.
+  public void testBuilderMergeFromWithUnknownFields() throws Exception {
+    TestAllTypesLite message = TestAllTypesLite.newBuilder().addRepeatedInt32(1).build();
+
+    NestedMessage.Builder builder = NestedMessage.newBuilder();
+    builder.mergeFrom(message.toByteArray());
+    builder.mergeFrom(message.toByteArray());
+    NestedMessage result = builder.build();
+    assertEquals(message.getSerializedSize() * 2, result.getSerializedSize());
+  }
+
   public void testToStringDefaultInstance() throws Exception {
     assertToStringEquals("", TestAllTypesLite.getDefaultInstance());
   }
@@ -2592,4 +2622,12 @@ public class LiteTest extends TestCase {
       return list.iterator();
     }
   }
+
+  public void testNullExtensionRegistry() throws Exception {
+    try {
+      TestAllTypesLite.parseFrom(new byte[] {}, null);
+      fail();
+    } catch (NullPointerException expected) {
+    }
+  }
 }

java/core/src/test/java/com/google/protobuf/MessageTest.java

@@ -74,6 +74,14 @@ public class MessageTest extends TestCase {
       "repeated_string: \"qux\"\n" +
       "repeated_string: \"bar\"\n";
 
+  public void testParsingWithNullExtensionRegistry() throws Exception {
+    try {
+      TestAllTypes.parseFrom(new byte[] {}, null);
+      fail();
+    } catch (NullPointerException expected) {
+    }
+  }
+
   public void testMergeFrom() throws Exception {
     TestAllTypes result =
       TestAllTypes.newBuilder(MERGE_DEST)

java/core/src/test/java/com/google/protobuf/ParserTest.java

@@ -30,9 +30,6 @@
 
 package com.google.protobuf;
 
-import com.google.protobuf.UnittestLite.TestAllTypesLite;
-import com.google.protobuf.UnittestLite.TestPackedExtensionsLite;
-import com.google.protobuf.UnittestLite.TestParsingMergeLite;
 import protobuf_unittest.UnittestOptimizeFor;
 import protobuf_unittest.UnittestOptimizeFor.TestOptimizedForSize;
 import protobuf_unittest.UnittestOptimizeFor.TestRequiredOptimizedForSize;
@@ -183,22 +180,12 @@ public class ParserTest extends TestCase {
                           TestUtil.getExtensionRegistry());
   }
 
-  public void testParseExtensionsLite() throws Exception {
-    assertRoundTripEquals(
-        TestUtilLite.getAllLiteExtensionsSet(), TestUtilLite.getExtensionRegistryLite());
-  }
-
   public void testParsePacked() throws Exception {
     assertRoundTripEquals(TestUtil.getPackedSet());
     assertRoundTripEquals(TestUtil.getPackedExtensionsSet(),
                           TestUtil.getExtensionRegistry());
   }
 
-  public void testParsePackedLite() throws Exception {
-    assertRoundTripEquals(
-        TestUtilLite.getLitePackedExtensionsSet(), TestUtilLite.getExtensionRegistryLite());
-  }
-
   public void testParseDelimitedTo() throws Exception {
     // Write normal Message.
     TestAllTypes normalMessage = TestUtil.getAllSet();
@@ -211,26 +198,6 @@ public class ParserTest extends TestCase {
     assertMessageEquals(normalMessage, normalMessage.getParserForType().parseDelimitedFrom(input));
   }
 
-  public void testParseDelimitedToLite() throws Exception {
-    // Write MessageLite with packed extension fields.
-    TestPackedExtensionsLite packedMessage = TestUtilLite.getLitePackedExtensionsSet();
-    ByteArrayOutputStream output = new ByteArrayOutputStream();
-    packedMessage.writeDelimitedTo(output);
-    packedMessage.writeDelimitedTo(output);
-
-    InputStream input = new ByteArrayInputStream(output.toByteArray());
-    assertMessageEquals(
-        packedMessage,
-        packedMessage
-            .getParserForType()
-            .parseDelimitedFrom(input, TestUtilLite.getExtensionRegistryLite()));
-    assertMessageEquals(
-        packedMessage,
-        packedMessage
-            .getParserForType()
-            .parseDelimitedFrom(input, TestUtilLite.getExtensionRegistryLite()));
-  }
-
   public void testParseUnknownFields() throws Exception {
     // All fields will be treated as unknown fields in emptyMessage.
     TestEmptyMessage emptyMessage =
@@ -263,14 +230,6 @@ public class ParserTest extends TestCase {
     assertEquals("hello", allTypes.getOptionalString());
   }
 
-  /** Helper method for {@link #testParsingMergeLite()}.*/
-  private void assertMessageMerged(TestAllTypesLite allTypes)
-      throws Exception {
-    assertEquals(3, allTypes.getOptionalInt32());
-    assertEquals(2, allTypes.getOptionalInt64());
-    assertEquals("hello", allTypes.getOptionalString());
-  }
-
   public void testParsingMerge() throws Exception {
     // Build messages.
     TestAllTypes.Builder builder = TestAllTypes.newBuilder();
@@ -332,67 +291,6 @@ public class ParserTest extends TestCase {
         TestParsingMerge.repeatedExt));
   }
 
-  public void testParsingMergeLite() throws Exception {
-    // Build messages.
-    TestAllTypesLite.Builder builder = TestAllTypesLite.newBuilder();
-    TestAllTypesLite msg1 = builder.setOptionalInt32(1).build();
-    builder.clear();
-    TestAllTypesLite msg2 = builder.setOptionalInt64(2).build();
-    builder.clear();
-    TestAllTypesLite msg3 = builder.setOptionalInt32(3)
-        .setOptionalString("hello").build();
-
-    // Build groups.
-    TestParsingMergeLite.RepeatedFieldsGenerator.Group1 optionalG1 =
-        TestParsingMergeLite.RepeatedFieldsGenerator.Group1.newBuilder()
-        .setField1(msg1).build();
-    TestParsingMergeLite.RepeatedFieldsGenerator.Group1 optionalG2 =
-        TestParsingMergeLite.RepeatedFieldsGenerator.Group1.newBuilder()
-        .setField1(msg2).build();
-    TestParsingMergeLite.RepeatedFieldsGenerator.Group1 optionalG3 =
-        TestParsingMergeLite.RepeatedFieldsGenerator.Group1.newBuilder()
-        .setField1(msg3).build();
-    TestParsingMergeLite.RepeatedFieldsGenerator.Group2 repeatedG1 =
-        TestParsingMergeLite.RepeatedFieldsGenerator.Group2.newBuilder()
-        .setField1(msg1).build();
-    TestParsingMergeLite.RepeatedFieldsGenerator.Group2 repeatedG2 =
-        TestParsingMergeLite.RepeatedFieldsGenerator.Group2.newBuilder()
-        .setField1(msg2).build();
-    TestParsingMergeLite.RepeatedFieldsGenerator.Group2 repeatedG3 =
-        TestParsingMergeLite.RepeatedFieldsGenerator.Group2.newBuilder()
-        .setField1(msg3).build();
-
-    // Assign and serialize RepeatedFieldsGenerator.
-    ByteString data = TestParsingMergeLite.RepeatedFieldsGenerator.newBuilder()
-        .addField1(msg1).addField1(msg2).addField1(msg3)
-        .addField2(msg1).addField2(msg2).addField2(msg3)
-        .addField3(msg1).addField3(msg2).addField3(msg3)
-        .addGroup1(optionalG1).addGroup1(optionalG2).addGroup1(optionalG3)
-        .addGroup2(repeatedG1).addGroup2(repeatedG2).addGroup2(repeatedG3)
-        .addExt1(msg1).addExt1(msg2).addExt1(msg3)
-        .addExt2(msg1).addExt2(msg2).addExt2(msg3)
-        .build().toByteString();
-
-    // Parse TestParsingMergeLite.
-    ExtensionRegistry registry = ExtensionRegistry.newInstance();
-    UnittestLite.registerAllExtensions(registry);
-    TestParsingMergeLite parsingMerge = TestParsingMergeLite.parser().parseFrom(data, registry);
-
-    // Required and optional fields should be merged.
-    assertMessageMerged(parsingMerge.getRequiredAllTypes());
-    assertMessageMerged(parsingMerge.getOptionalAllTypes());
-    assertMessageMerged(
-        parsingMerge.getOptionalGroup().getOptionalGroupAllTypes());
-    assertMessageMerged(parsingMerge.getExtension(
-        TestParsingMergeLite.optionalExt));
-
-    // Repeated fields should not be merged.
-    assertEquals(3, parsingMerge.getRepeatedAllTypesCount());
-    assertEquals(3, parsingMerge.getRepeatedGroupCount());
-    assertEquals(3, parsingMerge.getExtensionCount(
-        TestParsingMergeLite.repeatedExt));
-  }
-
   public void testParseDelimitedFrom_firstByteInterrupted_preservesCause() {
     try {
       TestUtil.getAllSet().parseDelimitedFrom(

java/core/src/test/java/com/google/protobuf/TextFormatTest.java

@@ -168,6 +168,7 @@ public class TextFormatTest extends TestCase {
 
   // Creates an example unknown field set.
   private UnknownFieldSet makeUnknownFieldSet() {
+
     return UnknownFieldSet.newBuilder()
         .addField(5,
             UnknownFieldSet.Field.newBuilder()
@@ -175,6 +176,12 @@ public class TextFormatTest extends TestCase {
             .addFixed32(2)
             .addFixed64(3)
             .addLengthDelimited(ByteString.copyFromUtf8("4"))
+            .addLengthDelimited(UnknownFieldSet.newBuilder()
+                .addField(12,
+                    UnknownFieldSet.Field.newBuilder()
+                        .addVarint(6)
+                        .build())
+                .build().toByteString())
             .addGroup(
                 UnknownFieldSet.newBuilder()
                 .addField(10,
@@ -207,20 +214,23 @@ public class TextFormatTest extends TestCase {
         .build();
 
     assertEquals(
-      "5: 1\n" +
-      "5: 0x00000002\n" +
-      "5: 0x0000000000000003\n" +
-      "5: \"4\"\n" +
-      "5 {\n" +
-      "  10: 5\n" +
-      "}\n" +
-      "8: 1\n" +
-      "8: 2\n" +
-      "8: 3\n" +
-      "15: 12379813812177893520\n" +
-      "15: 0xabcd1234\n" +
-      "15: 0xabcdef1234567890\n",
-      TextFormat.printToString(message));
+        "5: 1\n"
+            + "5: 0x00000002\n"
+            + "5: 0x0000000000000003\n"
+            + "5: \"4\"\n"
+            + "5: {\n"
+            + "  12: 6\n"
+            + "}\n"
+            + "5 {\n"
+            + "  10: 5\n"
+            + "}\n"
+            + "8: 1\n"
+            + "8: 2\n"
+            + "8: 3\n"
+            + "15: 12379813812177893520\n"
+            + "15: 0xabcd1234\n"
+            + "15: 0xabcdef1234567890\n",
+        TextFormat.printToString(message));
   }
 
   public void testPrintField() throws Exception {
@@ -861,7 +871,7 @@ public class TextFormatTest extends TestCase {
   }
 
   public void testShortDebugString_unknown() {
-    assertEquals("5: 1 5: 0x00000002 5: 0x0000000000000003 5: \"4\" 5 { 10: 5 }"
+    assertEquals("5: 1 5: 0x00000002 5: 0x0000000000000003 5: \"4\" 5: { 12: 6 } 5 { 10: 5 }"
         + " 8: 1 8: 2 8: 3 15: 12379813812177893520 15: 0xabcd1234 15:"
         + " 0xabcdef1234567890",
         TextFormat.shortDebugString(makeUnknownFieldSet()));

java/core/src/test/java/com/google/protobuf/UnknownFieldSetLiteTest.java

@@ -32,11 +32,20 @@ package com.google.protobuf;
 
 import com.google.protobuf.UnittestLite.TestAllExtensionsLite;
 import com.google.protobuf.UnittestLite.TestAllTypesLite;
+import protobuf_unittest.UnittestProto;
+import protobuf_unittest.UnittestProto.ForeignEnum;
+import protobuf_unittest.UnittestProto.TestAllExtensions;
+import protobuf_unittest.UnittestProto.TestAllTypes;
+import protobuf_unittest.UnittestProto.TestEmptyMessage;
+import protobuf_unittest.UnittestProto.TestPackedExtensions;
+import protobuf_unittest.UnittestProto.TestPackedTypes;
 import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash;
 import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.Bar;
 import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.Foo;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.Map;
 import junit.framework.TestCase;
 
 /**
@@ -45,7 +54,44 @@ import junit.framework.TestCase;
  * @author dweis@google.com (Daniel Weis)
  */
 public class UnknownFieldSetLiteTest extends TestCase {
-  
+  @Override
+  public void setUp() throws Exception {
+    allFields = TestUtil.getAllSet();
+    allFieldsData = allFields.toByteString();
+    emptyMessage = TestEmptyMessage.parseFrom(allFieldsData);
+    unknownFields = emptyMessage.getUnknownFields();
+  }
+
+  TestAllTypes allFields;
+  ByteString allFieldsData;
+
+  // Constructs a protocol buffer which contains fields with all the same
+  // numbers as allFieldsData except that each field is some other wire
+  // type.
+  private ByteString getBizarroData() throws Exception {
+    UnknownFieldSet.Builder bizarroFields = UnknownFieldSet.newBuilder();
+
+    UnknownFieldSet.Field varintField = UnknownFieldSet.Field.newBuilder().addVarint(1).build();
+    UnknownFieldSet.Field fixed32Field = UnknownFieldSet.Field.newBuilder().addFixed32(1).build();
+
+    for (Map.Entry<Integer, UnknownFieldSet.Field> entry : unknownFields.asMap().entrySet()) {
+      if (entry.getValue().getVarintList().isEmpty()) {
+        // Original field is not a varint, so use a varint.
+        bizarroFields.addField(entry.getKey(), varintField);
+      } else {
+        // Original field *is* a varint, so use something else.
+        bizarroFields.addField(entry.getKey(), fixed32Field);
+      }
+    }
+
+    return bizarroFields.build().toByteString();
+  }
+
+  // An empty message that has been parsed from allFieldsData.  So, it has
+  // unknown fields of every type.
+  TestEmptyMessage emptyMessage;
+  UnknownFieldSet unknownFields;
+
   public void testDefaultInstance() {
     UnknownFieldSetLite unknownFields = UnknownFieldSetLite.getDefaultInstance();
 
@@ -331,4 +377,203 @@ public class UnknownFieldSetLiteTest extends TestCase {
     }
     return ByteString.copyFrom(byteArrayOutputStream.toByteArray());
   }
+
+  public void testSerializeLite() throws Exception {
+    UnittestLite.TestEmptyMessageLite emptyMessageLite =
+        UnittestLite.TestEmptyMessageLite.parseFrom(allFieldsData);
+    assertEquals(allFieldsData.size(), emptyMessageLite.getSerializedSize());
+    ByteString data = emptyMessageLite.toByteString();
+    TestAllTypes message = TestAllTypes.parseFrom(data);
+    TestUtil.assertAllFieldsSet(message);
+    assertEquals(allFieldsData, data);
+  }
+
+  public void testAllExtensionsLite() throws Exception {
+    TestAllExtensions allExtensions = TestUtil.getAllExtensionsSet();
+    ByteString allExtensionsData = allExtensions.toByteString();
+    UnittestLite.TestEmptyMessageLite emptyMessageLite =
+        UnittestLite.TestEmptyMessageLite.parser().parseFrom(allExtensionsData);
+    ByteString data = emptyMessageLite.toByteString();
+    TestAllExtensions message = TestAllExtensions.parseFrom(data, TestUtil.getExtensionRegistry());
+    TestUtil.assertAllExtensionsSet(message);
+    assertEquals(allExtensionsData, data);
+  }
+
+  public void testAllPackedFieldsLite() throws Exception {
+    TestPackedTypes allPackedFields = TestUtil.getPackedSet();
+    ByteString allPackedData = allPackedFields.toByteString();
+    UnittestLite.TestEmptyMessageLite emptyMessageLite =
+        UnittestLite.TestEmptyMessageLite.parseFrom(allPackedData);
+    ByteString data = emptyMessageLite.toByteString();
+    TestPackedTypes message = TestPackedTypes.parseFrom(data, TestUtil.getExtensionRegistry());
+    TestUtil.assertPackedFieldsSet(message);
+    assertEquals(allPackedData, data);
+  }
+
+  public void testAllPackedExtensionsLite() throws Exception {
+    TestPackedExtensions allPackedExtensions = TestUtil.getPackedExtensionsSet();
+    ByteString allPackedExtensionsData = allPackedExtensions.toByteString();
+    UnittestLite.TestEmptyMessageLite emptyMessageLite =
+        UnittestLite.TestEmptyMessageLite.parseFrom(allPackedExtensionsData);
+    ByteString data = emptyMessageLite.toByteString();
+    TestPackedExtensions message =
+        TestPackedExtensions.parseFrom(data, TestUtil.getExtensionRegistry());
+    TestUtil.assertPackedExtensionsSet(message);
+    assertEquals(allPackedExtensionsData, data);
+  }
+
+  public void testCopyFromLite() throws Exception {
+    UnittestLite.TestEmptyMessageLite emptyMessageLite =
+        UnittestLite.TestEmptyMessageLite.parseFrom(allFieldsData);
+    UnittestLite.TestEmptyMessageLite emptyMessageLite2 =
+        UnittestLite.TestEmptyMessageLite.newBuilder().mergeFrom(emptyMessageLite).build();
+    assertEquals(emptyMessageLite.toByteString(), emptyMessageLite2.toByteString());
+  }
+
+  public void testMergeFromLite() throws Exception {
+    TestAllTypes message1 =
+        TestAllTypes.newBuilder()
+            .setOptionalInt32(1)
+            .setOptionalString("foo")
+            .addRepeatedString("bar")
+            .setOptionalNestedEnum(TestAllTypes.NestedEnum.BAZ)
+            .build();
+
+    TestAllTypes message2 =
+        TestAllTypes.newBuilder()
+            .setOptionalInt64(2)
+            .setOptionalString("baz")
+            .addRepeatedString("qux")
+            .setOptionalForeignEnum(ForeignEnum.FOREIGN_BAZ)
+            .build();
+
+    ByteString data1 = message1.toByteString();
+    UnittestLite.TestEmptyMessageLite emptyMessageLite1 =
+        UnittestLite.TestEmptyMessageLite.parseFrom(data1);
+    ByteString data2 = message2.toByteString();
+    UnittestLite.TestEmptyMessageLite emptyMessageLite2 =
+        UnittestLite.TestEmptyMessageLite.parseFrom(data2);
+
+    message1 = TestAllTypes.newBuilder(message1).mergeFrom(message2).build();
+    emptyMessageLite1 =
+        UnittestLite.TestEmptyMessageLite.newBuilder(emptyMessageLite1)
+            .mergeFrom(emptyMessageLite2)
+            .build();
+
+    data1 = emptyMessageLite1.toByteString();
+    message2 = TestAllTypes.parseFrom(data1);
+
+    assertEquals(message1, message2);
+  }
+
+  public void testWrongTypeTreatedAsUnknownLite() throws Exception {
+    // Test that fields of the wrong wire type are treated like unknown fields
+    // when parsing.
+
+    ByteString bizarroData = getBizarroData();
+    TestAllTypes allTypesMessage = TestAllTypes.parseFrom(bizarroData);
+    UnittestLite.TestEmptyMessageLite emptyMessageLite =
+        UnittestLite.TestEmptyMessageLite.parseFrom(bizarroData);
+    ByteString data = emptyMessageLite.toByteString();
+    TestAllTypes allTypesMessage2 = TestAllTypes.parseFrom(data);
+
+    assertEquals(allTypesMessage.toString(), allTypesMessage2.toString());
+  }
+
+  public void testUnknownExtensionsLite() throws Exception {
+    // Make sure fields are properly parsed to the UnknownFieldSet even when
+    // they are declared as extension numbers.
+
+    UnittestLite.TestEmptyMessageWithExtensionsLite message =
+        UnittestLite.TestEmptyMessageWithExtensionsLite.parseFrom(allFieldsData);
+
+    assertEquals(allFieldsData, message.toByteString());
+  }
+
+  public void testWrongExtensionTypeTreatedAsUnknownLite() throws Exception {
+    // Test that fields of the wrong wire type are treated like unknown fields
+    // when parsing extensions.
+
+    ByteString bizarroData = getBizarroData();
+    TestAllExtensions allExtensionsMessage = TestAllExtensions.parseFrom(bizarroData);
+    UnittestLite.TestEmptyMessageLite emptyMessageLite =
+        UnittestLite.TestEmptyMessageLite.parseFrom(bizarroData);
+
+    // All fields should have been interpreted as unknown, so the byte strings
+    // should be the same.
+    assertEquals(emptyMessageLite.toByteString(), allExtensionsMessage.toByteString());
+  }
+
+  public void testParseUnknownEnumValueLite() throws Exception {
+    Descriptors.FieldDescriptor singularField =
+        TestAllTypes.getDescriptor().findFieldByName("optional_nested_enum");
+    Descriptors.FieldDescriptor repeatedField =
+        TestAllTypes.getDescriptor().findFieldByName("repeated_nested_enum");
+    assertNotNull(singularField);
+    assertNotNull(repeatedField);
+
+    ByteString data =
+        UnknownFieldSet.newBuilder()
+            .addField(
+                singularField.getNumber(),
+                UnknownFieldSet.Field.newBuilder()
+                    .addVarint(TestAllTypes.NestedEnum.BAR.getNumber())
+                    .addVarint(5) // not valid
+                    .build())
+            .addField(
+                repeatedField.getNumber(),
+                UnknownFieldSet.Field.newBuilder()
+                    .addVarint(TestAllTypes.NestedEnum.FOO.getNumber())
+                    .addVarint(4) // not valid
+                    .addVarint(TestAllTypes.NestedEnum.BAZ.getNumber())
+                    .addVarint(6) // not valid
+                    .build())
+            .build()
+            .toByteString();
+
+    UnittestLite.TestEmptyMessageLite emptyMessageLite =
+        UnittestLite.TestEmptyMessageLite.parseFrom(data);
+    data = emptyMessageLite.toByteString();
+
+    {
+      TestAllTypes message = TestAllTypes.parseFrom(data);
+      assertEquals(TestAllTypes.NestedEnum.BAR, message.getOptionalNestedEnum());
+      assertEquals(
+          Arrays.asList(TestAllTypes.NestedEnum.FOO, TestAllTypes.NestedEnum.BAZ),
+          message.getRepeatedNestedEnumList());
+      assertEquals(
+          Arrays.asList(5L),
+          message.getUnknownFields().getField(singularField.getNumber()).getVarintList());
+      assertEquals(
+          Arrays.asList(4L, 6L),
+          message.getUnknownFields().getField(repeatedField.getNumber()).getVarintList());
+    }
+
+    {
+      TestAllExtensions message =
+          TestAllExtensions.parseFrom(data, TestUtil.getExtensionRegistry());
+      assertEquals(
+          TestAllTypes.NestedEnum.BAR,
+          message.getExtension(UnittestProto.optionalNestedEnumExtension));
+      assertEquals(
+          Arrays.asList(TestAllTypes.NestedEnum.FOO, TestAllTypes.NestedEnum.BAZ),
+          message.getExtension(UnittestProto.repeatedNestedEnumExtension));
+      assertEquals(
+          Arrays.asList(5L),
+          message.getUnknownFields().getField(singularField.getNumber()).getVarintList());
+      assertEquals(
+          Arrays.asList(4L, 6L),
+          message.getUnknownFields().getField(repeatedField.getNumber()).getVarintList());
+    }
+  }
+
+  public void testClearLite() throws Exception {
+    UnittestLite.TestEmptyMessageLite emptyMessageLite1 =
+        UnittestLite.TestEmptyMessageLite.parseFrom(allFieldsData);
+    UnittestLite.TestEmptyMessageLite emptyMessageLite2 =
+        UnittestLite.TestEmptyMessageLite.newBuilder().mergeFrom(emptyMessageLite1).clear().build();
+    assertEquals(0, emptyMessageLite2.getSerializedSize());
+    ByteString data = emptyMessageLite2.toByteString();
+    assertEquals(0, data.size());
+  }
 }

java/core/src/test/java/com/google/protobuf/UnknownFieldSetTest.java

@@ -445,208 +445,4 @@ public class UnknownFieldSetTest extends TestCase {
   }
 
   // =================================================================
-
-  public void testSerializeLite() throws Exception {
-    UnittestLite.TestEmptyMessageLite emptyMessageLite =
-        UnittestLite.TestEmptyMessageLite.parseFrom(allFieldsData);
-    assertEquals(allFieldsData.size(), emptyMessageLite.getSerializedSize());
-    ByteString data = emptyMessageLite.toByteString();
-    TestAllTypes message = TestAllTypes.parseFrom(data);
-    TestUtil.assertAllFieldsSet(message);
-    assertEquals(allFieldsData, data);
-  }
-
-  public void testAllExtensionsLite() throws Exception {
-    TestAllExtensions allExtensions = TestUtil.getAllExtensionsSet();
-    ByteString allExtensionsData = allExtensions.toByteString();
-    UnittestLite.TestEmptyMessageLite emptyMessageLite =
-        UnittestLite.TestEmptyMessageLite.parser().parseFrom(allExtensionsData);
-    ByteString data = emptyMessageLite.toByteString();
-    TestAllExtensions message =
-        TestAllExtensions.parseFrom(data, TestUtil.getExtensionRegistry());
-    TestUtil.assertAllExtensionsSet(message);
-    assertEquals(allExtensionsData, data);
-  }
-
-  public void testAllPackedFieldsLite() throws Exception {
-    TestPackedTypes allPackedFields = TestUtil.getPackedSet();
-    ByteString allPackedData = allPackedFields.toByteString();
-    UnittestLite.TestEmptyMessageLite emptyMessageLite =
-        UnittestLite.TestEmptyMessageLite.parseFrom(allPackedData);
-    ByteString data = emptyMessageLite.toByteString();
-    TestPackedTypes message =
-        TestPackedTypes.parseFrom(data, TestUtil.getExtensionRegistry());
-    TestUtil.assertPackedFieldsSet(message);
-    assertEquals(allPackedData, data);
-  }
-
-  public void testAllPackedExtensionsLite() throws Exception {
-    TestPackedExtensions allPackedExtensions = TestUtil.getPackedExtensionsSet();
-    ByteString allPackedExtensionsData = allPackedExtensions.toByteString();
-    UnittestLite.TestEmptyMessageLite emptyMessageLite =
-        UnittestLite.TestEmptyMessageLite.parseFrom(allPackedExtensionsData);
-    ByteString data = emptyMessageLite.toByteString();
-    TestPackedExtensions message =
-        TestPackedExtensions.parseFrom(data, TestUtil.getExtensionRegistry());
-    TestUtil.assertPackedExtensionsSet(message);
-    assertEquals(allPackedExtensionsData, data);
-  }
-
-  public void testCopyFromLite() throws Exception {
-    UnittestLite.TestEmptyMessageLite emptyMessageLite =
-        UnittestLite.TestEmptyMessageLite.parseFrom(allFieldsData);
-    UnittestLite.TestEmptyMessageLite emptyMessageLite2 =
-        UnittestLite.TestEmptyMessageLite.newBuilder()
-        .mergeFrom(emptyMessageLite).build();
-    assertEquals(emptyMessageLite.toByteString(), emptyMessageLite2.toByteString());
-  }
-
-  public void testMergeFromLite() throws Exception {
-    TestAllTypes message1 = TestAllTypes.newBuilder()
-        .setOptionalInt32(1)
-        .setOptionalString("foo")
-        .addRepeatedString("bar")
-        .setOptionalNestedEnum(TestAllTypes.NestedEnum.BAZ)
-        .build();
-
-    TestAllTypes message2 = TestAllTypes.newBuilder()
-        .setOptionalInt64(2)
-        .setOptionalString("baz")
-        .addRepeatedString("qux")
-        .setOptionalForeignEnum(ForeignEnum.FOREIGN_BAZ)
-        .build();
-
-    ByteString data1 = message1.toByteString();
-    UnittestLite.TestEmptyMessageLite emptyMessageLite1 =
-        UnittestLite.TestEmptyMessageLite.parseFrom(data1);
-    ByteString data2 = message2.toByteString();
-    UnittestLite.TestEmptyMessageLite emptyMessageLite2 =
-        UnittestLite.TestEmptyMessageLite.parseFrom(data2);
-
-    message1 = TestAllTypes.newBuilder(message1).mergeFrom(message2).build();
-    emptyMessageLite1 = UnittestLite.TestEmptyMessageLite.newBuilder(emptyMessageLite1)
-                        .mergeFrom(emptyMessageLite2).build();
-
-    data1 = emptyMessageLite1.toByteString();
-    message2 = TestAllTypes.parseFrom(data1);
-
-    assertEquals(message1, message2);
-  }
-
-  public void testWrongTypeTreatedAsUnknownLite() throws Exception {
-    // Test that fields of the wrong wire type are treated like unknown fields
-    // when parsing.
-
-    ByteString bizarroData = getBizarroData();
-    TestAllTypes allTypesMessage = TestAllTypes.parseFrom(bizarroData);
-    UnittestLite.TestEmptyMessageLite emptyMessageLite =
-        UnittestLite.TestEmptyMessageLite.parseFrom(bizarroData);
-    ByteString data = emptyMessageLite.toByteString();
-    TestAllTypes allTypesMessage2 = TestAllTypes.parseFrom(data);
-
-    assertEquals(allTypesMessage.toString(), allTypesMessage2.toString());
-  }
-
-  public void testUnknownExtensionsLite() throws Exception {
-    // Make sure fields are properly parsed to the UnknownFieldSet even when
-    // they are declared as extension numbers.
-
-    UnittestLite.TestEmptyMessageWithExtensionsLite message =
-      UnittestLite.TestEmptyMessageWithExtensionsLite.parseFrom(allFieldsData);
-
-    assertEquals(allFieldsData, message.toByteString());
-  }
-
-  public void testWrongExtensionTypeTreatedAsUnknownLite() throws Exception {
-    // Test that fields of the wrong wire type are treated like unknown fields
-    // when parsing extensions.
-
-    ByteString bizarroData = getBizarroData();
-    TestAllExtensions allExtensionsMessage =
-      TestAllExtensions.parseFrom(bizarroData);
-    UnittestLite.TestEmptyMessageLite emptyMessageLite =
-        UnittestLite.TestEmptyMessageLite.parseFrom(bizarroData);
-
-    // All fields should have been interpreted as unknown, so the byte strings
-    // should be the same.
-    assertEquals(emptyMessageLite.toByteString(),
-                 allExtensionsMessage.toByteString());
-  }
-
-  public void testParseUnknownEnumValueLite() throws Exception {
-    Descriptors.FieldDescriptor singularField =
-      TestAllTypes.getDescriptor().findFieldByName("optional_nested_enum");
-    Descriptors.FieldDescriptor repeatedField =
-      TestAllTypes.getDescriptor().findFieldByName("repeated_nested_enum");
-    assertNotNull(singularField);
-    assertNotNull(repeatedField);
-
-    ByteString data =
-      UnknownFieldSet.newBuilder()
-        .addField(singularField.getNumber(),
-          UnknownFieldSet.Field.newBuilder()
-            .addVarint(TestAllTypes.NestedEnum.BAR.getNumber())
-            .addVarint(5)   // not valid
-            .build())
-        .addField(repeatedField.getNumber(),
-          UnknownFieldSet.Field.newBuilder()
-            .addVarint(TestAllTypes.NestedEnum.FOO.getNumber())
-            .addVarint(4)   // not valid
-            .addVarint(TestAllTypes.NestedEnum.BAZ.getNumber())
-            .addVarint(6)   // not valid
-            .build())
-        .build()
-        .toByteString();
-
-    UnittestLite.TestEmptyMessageLite emptyMessageLite =
-        UnittestLite.TestEmptyMessageLite.parseFrom(data);
-    data = emptyMessageLite.toByteString();
-
-    {
-      TestAllTypes message = TestAllTypes.parseFrom(data);
-      assertEquals(TestAllTypes.NestedEnum.BAR,
-                   message.getOptionalNestedEnum());
-      assertEquals(
-        Arrays.asList(TestAllTypes.NestedEnum.FOO, TestAllTypes.NestedEnum.BAZ),
-        message.getRepeatedNestedEnumList());
-      assertEquals(Arrays.asList(5L),
-                   message.getUnknownFields()
-                          .getField(singularField.getNumber())
-                          .getVarintList());
-      assertEquals(Arrays.asList(4L, 6L),
-                   message.getUnknownFields()
-                          .getField(repeatedField.getNumber())
-                          .getVarintList());
-    }
-
-    {
-      TestAllExtensions message =
-        TestAllExtensions.parseFrom(data, TestUtil.getExtensionRegistry());
-      assertEquals(TestAllTypes.NestedEnum.BAR,
-        message.getExtension(UnittestProto.optionalNestedEnumExtension));
-      assertEquals(
-        Arrays.asList(TestAllTypes.NestedEnum.FOO, TestAllTypes.NestedEnum.BAZ),
-        message.getExtension(UnittestProto.repeatedNestedEnumExtension));
-      assertEquals(Arrays.asList(5L),
-                   message.getUnknownFields()
-                          .getField(singularField.getNumber())
-                          .getVarintList());
-      assertEquals(Arrays.asList(4L, 6L),
-                   message.getUnknownFields()
-                          .getField(repeatedField.getNumber())
-                          .getVarintList());
-    }
-  }
-
-  public void testClearLite() throws Exception {
-    UnittestLite.TestEmptyMessageLite emptyMessageLite1 =
-        UnittestLite.TestEmptyMessageLite.parseFrom(allFieldsData);
-    UnittestLite.TestEmptyMessageLite emptyMessageLite2 =
-        UnittestLite.TestEmptyMessageLite.newBuilder()
-        .mergeFrom(emptyMessageLite1).clear().build();
-    assertEquals(0, emptyMessageLite2.getSerializedSize());
-    ByteString data = emptyMessageLite2.toByteString();
-    assertEquals(0, data.size());
-  }
-
 }

java/core/src/test/java/com/google/protobuf/WireFormatTest.java

@@ -30,8 +30,6 @@
 
 package com.google.protobuf;
 
-import com.google.protobuf.UnittestLite.TestAllExtensionsLite;
-import com.google.protobuf.UnittestLite.TestPackedExtensionsLite;
 import protobuf_unittest.UnittestMset.RawMessageSet;
 import protobuf_unittest.UnittestMset.TestMessageSetExtension1;
 import protobuf_unittest.UnittestMset.TestMessageSetExtension2;
@@ -125,32 +123,6 @@ public class WireFormatTest extends TestCase {
     TestUtil.assertPackedFieldsSet(message2);
   }
 
-  public void testSerializeExtensionsLite() throws Exception {
-    // TestAllTypes and TestAllExtensions should have compatible wire formats,
-    // so if we serialize a TestAllExtensions then parse it as TestAllTypes
-    // it should work.
-
-    TestAllExtensionsLite message = TestUtilLite.getAllLiteExtensionsSet();
-    ByteString rawBytes = message.toByteString();
-    assertEquals(rawBytes.size(), message.getSerializedSize());
-
-    TestAllTypes message2 = TestAllTypes.parseFrom(rawBytes);
-
-    TestUtil.assertAllFieldsSet(message2);
-  }
-
-  public void testSerializePackedExtensionsLite() throws Exception {
-    // TestPackedTypes and TestPackedExtensions should have compatible wire
-    // formats; check that they serialize to the same string.
-    TestPackedExtensionsLite message = TestUtilLite.getLitePackedExtensionsSet();
-    ByteString rawBytes = message.toByteString();
-
-    TestPackedTypes message2 = TestUtil.getPackedSet();
-    ByteString rawBytes2 = message2.toByteString();
-
-    assertEquals(rawBytes, rawBytes2);
-  }
-
   public void testParseExtensions() throws Exception {
     // TestAllTypes and TestAllExtensions should have compatible wire formats,
     // so if we serialize a TestAllTypes then parse it as TestAllExtensions
@@ -180,48 +152,6 @@ public class WireFormatTest extends TestCase {
     TestUtil.assertPackedExtensionsSet(message2);
   }
 
-  public void testParseExtensionsLite() throws Exception {
-    // TestAllTypes and TestAllExtensions should have compatible wire formats,
-    // so if we serialize a TestAllTypes then parse it as TestAllExtensions
-    // it should work.
-
-    TestAllTypes message = TestUtil.getAllSet();
-    ByteString rawBytes = message.toByteString();
-
-    ExtensionRegistryLite registry_lite = TestUtilLite.getExtensionRegistryLite();
-
-    TestAllExtensionsLite message2 =
-      TestAllExtensionsLite.parseFrom(rawBytes, registry_lite);
-
-    TestUtil.assertAllExtensionsSet(message2);
-
-    // Try again using a full extension registry.
-    ExtensionRegistry registry = TestUtil.getExtensionRegistry();
-
-    TestAllExtensionsLite message3 =
-      TestAllExtensionsLite.parseFrom(rawBytes, registry);
-
-    TestUtil.assertAllExtensionsSet(message3);
-  }
-
-  public void testParsePackedExtensionsLite() throws Exception {
-    // Ensure that packed extensions can be properly parsed.
-    TestPackedExtensionsLite message = TestUtilLite.getLitePackedExtensionsSet();
-    ByteString rawBytes = message.toByteString();
-
-    ExtensionRegistryLite registry = TestUtilLite.getExtensionRegistryLite();
-
-    TestPackedExtensionsLite message2 =
-        TestPackedExtensionsLite.parseFrom(rawBytes, registry);
-
-    TestUtil.assertPackedExtensionsSet(message2);
-  }
-
-  public void testExtensionsSerializedSize() throws Exception {
-    assertNotSame(TestUtil.getAllSet().getSerializedSize(),
-                  TestUtil.getAllExtensionsSet().getSerializedSize());
-  }
-
   public void testSerializeDelimited() throws Exception {
     ByteArrayOutputStream output = new ByteArrayOutputStream();
     TestUtil.getAllSet().writeDelimitedTo(output);

java/util/src/main/java/com/google/protobuf/util/JsonFormat.java

@@ -1522,7 +1522,8 @@ public class JsonFormat {
       for (int i = 0; i < array.size(); ++i) {
         Object value = parseFieldValue(field, array.get(i), builder);
         if (value == null) {
-          throw new InvalidProtocolBufferException("Repeated field elements cannot be null");
+          throw new InvalidProtocolBufferException(
+              "Repeated field elements cannot be null in field: " + field.getFullName());
         }
         builder.addRepeatedField(field, value);
       }

js/binary/decoder.js

@@ -47,6 +47,7 @@ goog.provide('jspb.BinaryDecoder');
 goog.provide('jspb.BinaryIterator');
 
 goog.require('goog.asserts');
+goog.require('goog.crypt');
 goog.require('jspb.utils');
 
 

js/binary/utils.js

@@ -38,6 +38,7 @@
 goog.provide('jspb.utils');
 
 goog.require('goog.asserts');
+goog.require('goog.crypt');
 goog.require('goog.crypt.base64');
 goog.require('goog.string');
 goog.require('jspb.BinaryConstants');

js/binary/utils_test.js

@@ -36,6 +36,7 @@
  * @author aappleby@google.com (Austin Appleby)
  */
 
+goog.require('goog.crypt');
 goog.require('goog.crypt.base64');
 goog.require('goog.testing.asserts');
 goog.require('jspb.BinaryConstants');

js/compatibility_tests/v3.1.0/message_test.js

@@ -643,12 +643,7 @@ describe('Message test suite', function() {
 
   it('testInitialization_emptyArray', function() {
     var msg = new proto.jspb.test.HasExtensions([]);
-    if (jspb.Message.MINIMIZE_MEMORY_ALLOCATIONS) {
-      assertArrayEquals([], msg.toArray());
-    } else {
-      // Extension object is created past all regular fields.
-      assertArrayEquals([,,, {}], msg.toArray());
-    }
+    assertArrayEquals([], msg.toArray());
   });
 
   it('testInitialization_justExtensionObject', function() {

js/message.js

@@ -889,6 +889,100 @@ jspb.Message.setField = function(msg, fieldNumber, value) {
 };
 
 
+/**
+ * Sets the value of a non-extension integer field of a proto3
+ * @param {!jspb.Message} msg A jspb proto.
+ * @param {number} fieldNumber The field number.
+ * @param {number} value New value
+ * @protected
+ */
+jspb.Message.setProto3IntField = function(msg, fieldNumber, value) {
+  jspb.Message.setFieldIgnoringDefault_(msg, fieldNumber, value, 0);
+};
+
+
+/**
+ * Sets the value of a non-extension floating point field of a proto3
+ * @param {!jspb.Message} msg A jspb proto.
+ * @param {number} fieldNumber The field number.
+ * @param {number} value New value
+ * @protected
+ */
+jspb.Message.setProto3FloatField = function(msg, fieldNumber, value) {
+  jspb.Message.setFieldIgnoringDefault_(msg, fieldNumber, value, 0.0);
+};
+
+
+/**
+ * Sets the value of a non-extension boolean field of a proto3
+ * @param {!jspb.Message} msg A jspb proto.
+ * @param {number} fieldNumber The field number.
+ * @param {boolean} value New value
+ * @protected
+ */
+jspb.Message.setProto3BooleanField = function(msg, fieldNumber, value) {
+  jspb.Message.setFieldIgnoringDefault_(msg, fieldNumber, value, false);
+};
+
+
+/**
+ * Sets the value of a non-extension String field of a proto3
+ * @param {!jspb.Message} msg A jspb proto.
+ * @param {number} fieldNumber The field number.
+ * @param {string} value New value
+ * @protected
+ */
+jspb.Message.setProto3StringField = function(msg, fieldNumber, value) {
+  jspb.Message.setFieldIgnoringDefault_(msg, fieldNumber, value, "");
+};
+
+
+/**
+ * Sets the value of a non-extension Bytes field of a proto3
+ * @param {!jspb.Message} msg A jspb proto.
+ * @param {number} fieldNumber The field number.
+ * @param {!Uint8Array|string} value New value
+ * @protected
+ */
+jspb.Message.setProto3BytesField = function(msg, fieldNumber, value) {
+  jspb.Message.setFieldIgnoringDefault_(msg, fieldNumber, value, "");
+};
+
+
+/**
+ * Sets the value of a non-extension enum field of a proto3
+ * @param {!jspb.Message} msg A jspb proto.
+ * @param {number} fieldNumber The field number.
+ * @param {number} value New value
+ * @protected
+ */
+jspb.Message.setProto3EnumField = function(msg, fieldNumber, value) {
+  jspb.Message.setFieldIgnoringDefault_(msg, fieldNumber, value, 0);
+};
+
+
+
+/**
+ * Sets the value of a non-extension primitive field, with proto3 (non-nullable
+ * primitives) semantics of ignoring values that are equal to the type's
+ * default.
+ * @template T
+ * @param {!jspb.Message} msg A jspb proto.
+ * @param {number} fieldNumber The field number.
+ * @param {!Uint8Array|string|number|boolean|undefined} value New value
+ * @param {!Uint8Array|string|number|boolean} defaultValue The default value.
+ * @private
+ */
+jspb.Message.setFieldIgnoringDefault_ = function(
+    msg, fieldNumber, value, defaultValue) {
+  if (value != defaultValue) {
+    jspb.Message.setField(msg, fieldNumber, value);
+  } else {
+    msg.array[jspb.Message.getIndex_(msg, fieldNumber)] = null;
+  }
+};
+
+
 /**
  * Adds a value to a repeated, primitive field.
  * @param {!jspb.Message} msg A jspb proto.

js/proto3_test.js

@@ -72,6 +72,37 @@ function bytesCompare(arr, expected) {
 
 
 describe('proto3Test', function() {
+
+  /**
+   * Test default values don't affect equality test.
+   */
+  it('testEqualsProto3', function() {
+    var msg1 = new proto.jspb.test.TestProto3();
+    var msg2 = new proto.jspb.test.TestProto3();
+    msg2.setOptionalString('');
+
+    assertTrue(jspb.Message.equals(msg1, msg2));
+  });
+
+
+  /**
+   * Test setting when a field has default semantics.
+   */
+  it('testSetProto3ToValueAndBackToDefault', function() {
+    var msg = new proto.jspb.test.TestProto3();
+
+    // Setting should work normally.
+    msg.setOptionalString('optionalString');
+    assertEquals(msg.getOptionalString(), 'optionalString');
+
+    // Clearing should work too ...
+    msg.setOptionalString('');
+    assertEquals(msg.getOptionalString(), '');
+
+    // ... and shouldn't affect the equality with a brand new message.
+    assertTrue(jspb.Message.equals(msg, new proto.jspb.test.TestProto3()));
+  });
+
   /**
    * Test defaults for proto3 message fields.
    */
@@ -225,52 +256,65 @@ describe('proto3Test', function() {
    * Test that oneofs continue to have a notion of field presence.
    */
   it('testOneofs', function() {
+    // Default instance.
     var msg = new proto.jspb.test.TestProto3();
-
     assertEquals(msg.getOneofUint32(), 0);
     assertEquals(msg.getOneofForeignMessage(), undefined);
     assertEquals(msg.getOneofString(), '');
     assertEquals(msg.getOneofBytes(), '');
+
     assertFalse(msg.hasOneofUint32());
+    assertFalse(msg.hasOneofForeignMessage());
     assertFalse(msg.hasOneofString());
     assertFalse(msg.hasOneofBytes());
 
+    // Integer field.
     msg.setOneofUint32(42);
     assertEquals(msg.getOneofUint32(), 42);
     assertEquals(msg.getOneofForeignMessage(), undefined);
     assertEquals(msg.getOneofString(), '');
     assertEquals(msg.getOneofBytes(), '');
+
     assertTrue(msg.hasOneofUint32());
+    assertFalse(msg.hasOneofForeignMessage());
     assertFalse(msg.hasOneofString());
     assertFalse(msg.hasOneofBytes());
 
-
+    // Sub-message field.
     var submsg = new proto.jspb.test.ForeignMessage();
     msg.setOneofForeignMessage(submsg);
     assertEquals(msg.getOneofUint32(), 0);
     assertEquals(msg.getOneofForeignMessage(), submsg);
     assertEquals(msg.getOneofString(), '');
     assertEquals(msg.getOneofBytes(), '');
+
     assertFalse(msg.hasOneofUint32());
+    assertTrue(msg.hasOneofForeignMessage());
     assertFalse(msg.hasOneofString());
     assertFalse(msg.hasOneofBytes());
 
+    // String field.
     msg.setOneofString('hello');
     assertEquals(msg.getOneofUint32(), 0);
     assertEquals(msg.getOneofForeignMessage(), undefined);
     assertEquals(msg.getOneofString(), 'hello');
     assertEquals(msg.getOneofBytes(), '');
+
     assertFalse(msg.hasOneofUint32());
+    assertFalse(msg.hasOneofForeignMessage());
     assertTrue(msg.hasOneofString());
     assertFalse(msg.hasOneofBytes());
 
+    // Bytes field.
     msg.setOneofBytes(goog.crypt.base64.encodeString('\u00FF\u00FF'));
     assertEquals(msg.getOneofUint32(), 0);
     assertEquals(msg.getOneofForeignMessage(), undefined);
     assertEquals(msg.getOneofString(), '');
     assertEquals(msg.getOneofBytes_asB64(),
         goog.crypt.base64.encodeString('\u00FF\u00FF'));
+
     assertFalse(msg.hasOneofUint32());
+    assertFalse(msg.hasOneofForeignMessage());
     assertFalse(msg.hasOneofString());
     assertTrue(msg.hasOneofBytes());
   });

php/src/GPBMetadata/Google/Protobuf/Internal/Descriptor.php

@@ -110,6 +110,13 @@ class Descriptor
             ->optional('name', \Google\Protobuf\Internal\GPBType::STRING, 1)
             ->repeated('value', \Google\Protobuf\Internal\GPBType::MESSAGE, 2, 'google.protobuf.internal.EnumValueDescriptorProto')
             ->optional('options', \Google\Protobuf\Internal\GPBType::MESSAGE, 3, 'google.protobuf.internal.EnumOptions')
+            ->repeated('reserved_range', \Google\Protobuf\Internal\GPBType::MESSAGE, 4, 'google.protobuf.internal.EnumDescriptorProto.EnumReservedRange')
+            ->repeated('reserved_name', \Google\Protobuf\Internal\GPBType::STRING, 5)
+            ->finalizeToPool();
+
+        $pool->addMessage('google.protobuf.internal.EnumDescriptorProto.EnumReservedRange', \Google\Protobuf\Internal\EnumDescriptorProto_EnumReservedRange::class)
+            ->optional('start', \Google\Protobuf\Internal\GPBType::INT32, 1)
+            ->optional('end', \Google\Protobuf\Internal\GPBType::INT32, 2)
             ->finalizeToPool();
 
         $pool->addMessage('google.protobuf.internal.EnumValueDescriptorProto', \Google\Protobuf\Internal\EnumValueDescriptorProto::class)
@@ -144,7 +151,7 @@ class Descriptor
             ->optional('cc_generic_services', \Google\Protobuf\Internal\GPBType::BOOL, 16)
             ->optional('java_generic_services', \Google\Protobuf\Internal\GPBType::BOOL, 17)
             ->optional('py_generic_services', \Google\Protobuf\Internal\GPBType::BOOL, 18)
-            ->optional('php_generic_services', \Google\Protobuf\Internal\GPBType::BOOL, 19)
+            ->optional('php_generic_services', \Google\Protobuf\Internal\GPBType::BOOL, 42)
             ->optional('deprecated', \Google\Protobuf\Internal\GPBType::BOOL, 23)
             ->optional('cc_enable_arenas', \Google\Protobuf\Internal\GPBType::BOOL, 31)
             ->optional('objc_class_prefix', \Google\Protobuf\Internal\GPBType::STRING, 36)

php/src/Google/Protobuf/Internal/EnumDescriptorProto.php

@@ -32,6 +32,23 @@ class EnumDescriptorProto extends \Google\Protobuf\Internal\Message
      */
     private $options = null;
     private $has_options = false;
+    /**
+     * Range of reserved numeric values. Reserved numeric values may not be used
+     * by enum values in the same enum declaration. Reserved ranges may not
+     * overlap.
+     *
+     * Generated from protobuf field <code>repeated .google.protobuf.EnumDescriptorProto.EnumReservedRange reserved_range = 4;</code>
+     */
+    private $reserved_range;
+    private $has_reserved_range = false;
+    /**
+     * Reserved enum value names, which may not be reused. A given name may only
+     * be reserved once.
+     *
+     * Generated from protobuf field <code>repeated string reserved_name = 5;</code>
+     */
+    private $reserved_name;
+    private $has_reserved_name = false;
 
     public function __construct() {
         \GPBMetadata\Google\Protobuf\Internal\Descriptor::initOnce();
@@ -122,5 +139,75 @@ class EnumDescriptorProto extends \Google\Protobuf\Internal\Message
         return $this->has_options;
     }
 
+    /**
+     * Range of reserved numeric values. Reserved numeric values may not be used
+     * by enum values in the same enum declaration. Reserved ranges may not
+     * overlap.
+     *
+     * Generated from protobuf field <code>repeated .google.protobuf.EnumDescriptorProto.EnumReservedRange reserved_range = 4;</code>
+     * @return \Google\Protobuf\Internal\RepeatedField
+     */
+    public function getReservedRange()
+    {
+        return $this->reserved_range;
+    }
+
+    /**
+     * Range of reserved numeric values. Reserved numeric values may not be used
+     * by enum values in the same enum declaration. Reserved ranges may not
+     * overlap.
+     *
+     * Generated from protobuf field <code>repeated .google.protobuf.EnumDescriptorProto.EnumReservedRange reserved_range = 4;</code>
+     * @param \Google\Protobuf\Internal\EnumDescriptorProto_EnumReservedRange[]|\Google\Protobuf\Internal\RepeatedField $var
+     * @return $this
+     */
+    public function setReservedRange($var)
+    {
+        $arr = GPBUtil::checkRepeatedField($var, \Google\Protobuf\Internal\GPBType::MESSAGE, \Google\Protobuf\Internal\EnumDescriptorProto_EnumReservedRange::class);
+        $this->reserved_range = $arr;
+        $this->has_reserved_range = true;
+
+        return $this;
+    }
+
+    public function hasReservedRange()
+    {
+        return $this->has_reserved_range;
+    }
+
+    /**
+     * Reserved enum value names, which may not be reused. A given name may only
+     * be reserved once.
+     *
+     * Generated from protobuf field <code>repeated string reserved_name = 5;</code>
+     * @return \Google\Protobuf\Internal\RepeatedField
+     */
+    public function getReservedName()
+    {
+        return $this->reserved_name;
+    }
+
+    /**
+     * Reserved enum value names, which may not be reused. A given name may only
+     * be reserved once.
+     *
+     * Generated from protobuf field <code>repeated string reserved_name = 5;</code>
+     * @param string[]|\Google\Protobuf\Internal\RepeatedField $var
+     * @return $this
+     */
+    public function setReservedName($var)
+    {
+        $arr = GPBUtil::checkRepeatedField($var, \Google\Protobuf\Internal\GPBType::STRING);
+        $this->reserved_name = $arr;
+        $this->has_reserved_name = true;
+
+        return $this;
+    }
+
+    public function hasReservedName()
+    {
+        return $this->has_reserved_name;
+    }
+
 }
 

php/src/Google/Protobuf/Internal/FileOptions.php

@@ -109,7 +109,7 @@ class FileOptions extends \Google\Protobuf\Internal\Message
     private $py_generic_services = false;
     private $has_py_generic_services = false;
     /**
-     * Generated from protobuf field <code>optional bool php_generic_services = 19 [default = false];</code>
+     * Generated from protobuf field <code>optional bool php_generic_services = 42 [default = false];</code>
      */
     private $php_generic_services = false;
     private $has_php_generic_services = false;
@@ -553,7 +553,7 @@ class FileOptions extends \Google\Protobuf\Internal\Message
     }
 
     /**
-     * Generated from protobuf field <code>optional bool php_generic_services = 19 [default = false];</code>
+     * Generated from protobuf field <code>optional bool php_generic_services = 42 [default = false];</code>
      * @return bool
      */
     public function getPhpGenericServices()
@@ -562,7 +562,7 @@ class FileOptions extends \Google\Protobuf\Internal\Message
     }
 
     /**
-     * Generated from protobuf field <code>optional bool php_generic_services = 19 [default = false];</code>
+     * Generated from protobuf field <code>optional bool php_generic_services = 42 [default = false];</code>
      * @param bool $var
      * @return $this
      */

python/google/protobuf/descriptor_database.py

@@ -107,6 +107,7 @@ class DescriptorDatabase(object):
 
     'some.package.name.Message'
     'some.package.name.Message.NestedEnum'
+    'some.package.name.Message.some_field'
 
     The file descriptor proto containing the specified symbol must be added to
     this database using the Add method or else an error will be raised.
@@ -120,8 +121,16 @@ class DescriptorDatabase(object):
     Raises:
       KeyError if no file contains the specified symbol.
     """
-
-    return self._file_desc_protos_by_symbol[symbol]
+    try:
+      return self._file_desc_protos_by_symbol[symbol]
+    except KeyError:
+      # Fields, enum values, and nested extensions are not in
+      # _file_desc_protos_by_symbol. Try to find the top level
+      # descriptor. Non-existent nested symbol under a valid top level
+      # descriptor can also be found. The behavior is the same with
+      # protobuf C++.
+      top_level, _, _ = symbol.rpartition('.')
+      return self._file_desc_protos_by_symbol[top_level]
 
 
 def _ExtractSymbols(desc_proto, package):

python/google/protobuf/descriptor_pool.py

@@ -127,9 +127,6 @@ class DescriptorPool(object):
     self._service_descriptors = {}
     self._file_descriptors = {}
     self._toplevel_extensions = {}
-    # TODO(jieluo): Remove _file_desc_by_toplevel_extension when
-    # FieldDescriptor.file is added in code gen.
-    self._file_desc_by_toplevel_extension = {}
     # We store extensions in two two-level mappings: The first key is the
     # descriptor of the message being extended, the second key is the extension
     # full name or its tag number.
@@ -255,11 +252,6 @@ class DescriptorPool(object):
     """
 
     self._AddFileDescriptor(file_desc)
-    # TODO(jieluo): This is a temporary solution for FieldDescriptor.file.
-    # Remove it when FieldDescriptor.file is added in code gen.
-    for extension in file_desc.extensions_by_name.values():
-      self._file_desc_by_toplevel_extension[
-          extension.full_name] = file_desc
 
   def _AddFileDescriptor(self, file_desc):
     """Adds a FileDescriptor to the pool, non-recursively.
@@ -339,7 +331,7 @@ class DescriptorPool(object):
       pass
 
     try:
-      return self._file_desc_by_toplevel_extension[symbol]
+      return self._toplevel_extensions[symbol].file
     except KeyError:
       pass
 
@@ -405,6 +397,23 @@ class DescriptorPool(object):
     message_descriptor = self.FindMessageTypeByName(message_name)
     return message_descriptor.fields_by_name[field_name]
 
+  def FindOneofByName(self, full_name):
+    """Loads the named oneof descriptor from the pool.
+
+    Args:
+      full_name: The full name of the oneof descriptor to load.
+
+    Returns:
+      The oneof descriptor for the named oneof.
+
+    Raises:
+      KeyError: if the oneof cannot be found in the pool.
+    """
+    full_name = _NormalizeFullyQualifiedName(full_name)
+    message_name, _, oneof_name = full_name.rpartition('.')
+    message_descriptor = self.FindMessageTypeByName(message_name)
+    return message_descriptor.oneofs_by_name[oneof_name]
+
   def FindExtensionByName(self, full_name):
     """Loads the named extension descriptor from the pool.
 

python/google/protobuf/internal/api_implementation.py

@@ -61,10 +61,15 @@ if _api_version < 0:  # Still unspecified?
     del _use_fast_cpp_protos
     _api_version = 2
   except ImportError:
-    if _proto_extension_modules_exist_in_build:
-      if sys.version_info[0] >= 3:  # Python 3 defaults to C++ impl v2.
-        _api_version = 2
-      # TODO(b/17427486): Make Python 2 default to C++ impl v2.
+    try:
+      # pylint: disable=g-import-not-at-top
+      from google.protobuf.internal import use_pure_python
+      del use_pure_python  # Avoids a pylint error and namespace pollution.
+    except ImportError:
+      if _proto_extension_modules_exist_in_build:
+        if sys.version_info[0] >= 3:  # Python 3 defaults to C++ impl v2.
+          _api_version = 2
+          # TODO(b/17427486): Make Python 2 default to C++ impl v2.
 
 _default_implementation_type = (
     'python' if _api_version <= 0 else 'cpp')
@@ -137,3 +142,29 @@ def Version():
 # For internal use only
 def IsPythonDefaultSerializationDeterministic():
   return _python_deterministic_proto_serialization
+
+# DO NOT USE: For migration and testing only. Will be removed when Proto3
+# defaults to preserve unknowns.
+if _implementation_type == 'cpp':
+  try:
+    # pylint: disable=g-import-not-at-top
+    from google.protobuf.pyext import _message
+
+    def GetPythonProto3PreserveUnknownsDefault():
+      return _message.GetPythonProto3PreserveUnknownsDefault()
+
+    def SetPythonProto3PreserveUnknownsDefault(preserve):
+      _message.SetPythonProto3PreserveUnknownsDefault(preserve)
+  except ImportError:
+    # Unrecognized cpp implementation. Skipping the unknown fields APIs.
+    pass
+else:
+  _python_proto3_preserve_unknowns_default = False
+
+  def GetPythonProto3PreserveUnknownsDefault():
+    return _python_proto3_preserve_unknowns_default
+
+  def SetPythonProto3PreserveUnknownsDefault(preserve):
+    global _python_proto3_preserve_unknowns_default
+    _python_proto3_preserve_unknowns_default = preserve
+

python/google/protobuf/internal/descriptor_database_test.py

@@ -39,6 +39,7 @@ try:
 except ImportError:
   import unittest
 
+from google.protobuf import unittest_pb2
 from google.protobuf import descriptor_pb2
 from google.protobuf.internal import factory_test2_pb2
 from google.protobuf import descriptor_database
@@ -54,16 +55,49 @@ class DescriptorDatabaseTest(unittest.TestCase):
 
     self.assertEqual(file_desc_proto, db.FindFileByName(
         'google/protobuf/internal/factory_test2.proto'))
+    # Can find message type.
     self.assertEqual(file_desc_proto, db.FindFileContainingSymbol(
         'google.protobuf.python.internal.Factory2Message'))
+    # Can find nested message type.
     self.assertEqual(file_desc_proto, db.FindFileContainingSymbol(
         'google.protobuf.python.internal.Factory2Message.NestedFactory2Message'))
+    # Can find enum type.
     self.assertEqual(file_desc_proto, db.FindFileContainingSymbol(
         'google.protobuf.python.internal.Factory2Enum'))
+    # Can find nested enum type.
     self.assertEqual(file_desc_proto, db.FindFileContainingSymbol(
         'google.protobuf.python.internal.Factory2Message.NestedFactory2Enum'))
     self.assertEqual(file_desc_proto, db.FindFileContainingSymbol(
         'google.protobuf.python.internal.MessageWithNestedEnumOnly.NestedEnum'))
+    # Can find field.
+    self.assertEqual(file_desc_proto, db.FindFileContainingSymbol(
+        'google.protobuf.python.internal.Factory2Message.list_field'))
+    # Can find enum value.
+    self.assertEqual(file_desc_proto, db.FindFileContainingSymbol(
+        'google.protobuf.python.internal.Factory2Enum.FACTORY_2_VALUE_0'))
+    # Can find top level extension.
+    self.assertEqual(file_desc_proto, db.FindFileContainingSymbol(
+        'google.protobuf.python.internal.another_field'))
+    # Can find nested extension inside a message.
+    self.assertEqual(file_desc_proto, db.FindFileContainingSymbol(
+        'google.protobuf.python.internal.Factory2Message.one_more_field'))
+
+    # Can find service.
+    file_desc_proto2 = descriptor_pb2.FileDescriptorProto.FromString(
+        unittest_pb2.DESCRIPTOR.serialized_pb)
+    db.Add(file_desc_proto2)
+    self.assertEqual(file_desc_proto2, db.FindFileContainingSymbol(
+        'protobuf_unittest.TestService'))
+
+    # Non-existent field under a valid top level symbol can also be
+    # found. The behavior is the same with protobuf C++.
+    self.assertEqual(file_desc_proto2, db.FindFileContainingSymbol(
+        'protobuf_unittest.TestAllTypes.none_field'))
+
+    self.assertRaises(KeyError,
+                      db.FindFileContainingSymbol,
+                      'protobuf_unittest.NoneMessage')
+
 
 if __name__ == '__main__':
   unittest.main()

python/google/protobuf/internal/descriptor_pool_test.py

@@ -60,26 +60,8 @@ from google.protobuf import message_factory
 from google.protobuf import symbol_database
 
 
-class DescriptorPoolTest(unittest.TestCase):
 
-  def setUp(self):
-    # TODO(jieluo): Should make the pool which is created by
-    # serialized_pb same with generated pool.
-    # TODO(jieluo): More test coverage for the generated pool.
-    self.pool = descriptor_pool.DescriptorPool()
-    self.factory_test1_fd = descriptor_pb2.FileDescriptorProto.FromString(
-        factory_test1_pb2.DESCRIPTOR.serialized_pb)
-    self.factory_test2_fd = descriptor_pb2.FileDescriptorProto.FromString(
-        factory_test2_pb2.DESCRIPTOR.serialized_pb)
-    self.pool.Add(self.factory_test1_fd)
-    self.pool.Add(self.factory_test2_fd)
-
-    self.pool.Add(descriptor_pb2.FileDescriptorProto.FromString(
-        unittest_import_public_pb2.DESCRIPTOR.serialized_pb))
-    self.pool.Add(descriptor_pb2.FileDescriptorProto.FromString(
-        unittest_import_pb2.DESCRIPTOR.serialized_pb))
-    self.pool.Add(descriptor_pb2.FileDescriptorProto.FromString(
-        unittest_pb2.DESCRIPTOR.serialized_pb))
+class DescriptorPoolTestBase(object):
 
   def testFindFileByName(self):
     name1 = 'google/protobuf/internal/factory_test1.proto'
@@ -137,14 +119,6 @@ class DescriptorPoolTest(unittest.TestCase):
     self.assertEqual('google/protobuf/unittest.proto',
                      file_desc5.name)
 
-    # Tests the generated pool.
-    assert descriptor_pool.Default().FindFileContainingSymbol(
-        'google.protobuf.python.internal.Factory2Message.one_more_field')
-    assert descriptor_pool.Default().FindFileContainingSymbol(
-        'google.protobuf.python.internal.another_field')
-    assert descriptor_pool.Default().FindFileContainingSymbol(
-        'protobuf_unittest.TestService')
-
   def testFindFileContainingSymbolFailure(self):
     with self.assertRaises(KeyError):
       self.pool.FindFileContainingSymbol('Does not exist')
@@ -231,6 +205,27 @@ class DescriptorPoolTest(unittest.TestCase):
                        msg2.fields_by_name[name].containing_oneof)
       self.assertIn(msg2.fields_by_name[name], msg2.oneofs[0].fields)
 
+  def testFindTypeErrors(self):
+    self.assertRaises(TypeError, self.pool.FindExtensionByNumber, '')
+
+    # TODO(jieluo): Fix python to raise correct errors.
+    if api_implementation.Type() == 'cpp':
+      self.assertRaises(TypeError, self.pool.FindMethodByName, 0)
+      self.assertRaises(KeyError, self.pool.FindMethodByName, '')
+      error_type = TypeError
+    else:
+      error_type = AttributeError
+    self.assertRaises(error_type, self.pool.FindMessageTypeByName, 0)
+    self.assertRaises(error_type, self.pool.FindFieldByName, 0)
+    self.assertRaises(error_type, self.pool.FindExtensionByName, 0)
+    self.assertRaises(error_type, self.pool.FindEnumTypeByName, 0)
+    self.assertRaises(error_type, self.pool.FindOneofByName, 0)
+    self.assertRaises(error_type, self.pool.FindServiceByName, 0)
+    self.assertRaises(error_type, self.pool.FindFileContainingSymbol, 0)
+    if api_implementation.Type() == 'python':
+      error_type = KeyError
+    self.assertRaises(error_type, self.pool.FindFileByName, 0)
+
   def testFindMessageTypeByNameFailure(self):
     with self.assertRaises(KeyError):
       self.pool.FindMessageTypeByName('Does not exist')
@@ -270,6 +265,11 @@ class DescriptorPoolTest(unittest.TestCase):
       self.pool.FindEnumTypeByName('Does not exist')
 
   def testFindFieldByName(self):
+    if isinstance(self, SecondaryDescriptorFromDescriptorDB):
+      if api_implementation.Type() == 'cpp':
+        # TODO(jieluo): Fix cpp extension to find field correctly
+        # when descriptor pool is using an underlying database.
+        return
     field = self.pool.FindFieldByName(
         'google.protobuf.python.internal.Factory1Message.list_value')
     self.assertEqual(field.name, 'list_value')
@@ -279,7 +279,24 @@ class DescriptorPoolTest(unittest.TestCase):
     with self.assertRaises(KeyError):
       self.pool.FindFieldByName('Does not exist')
 
+  def testFindOneofByName(self):
+    if isinstance(self, SecondaryDescriptorFromDescriptorDB):
+      if api_implementation.Type() == 'cpp':
+        # TODO(jieluo): Fix cpp extension to find oneof correctly
+        # when descriptor pool is using an underlying database.
+        return
+    oneof = self.pool.FindOneofByName(
+        'google.protobuf.python.internal.Factory2Message.oneof_field')
+    self.assertEqual(oneof.name, 'oneof_field')
+    with self.assertRaises(KeyError):
+      self.pool.FindOneofByName('Does not exist')
+
   def testFindExtensionByName(self):
+    if isinstance(self, SecondaryDescriptorFromDescriptorDB):
+      if api_implementation.Type() == 'cpp':
+        # TODO(jieluo): Fix cpp extension to find extension correctly
+        # when descriptor pool is using an underlying database.
+        return
     # An extension defined in a message.
     extension = self.pool.FindExtensionByName(
         'google.protobuf.python.internal.Factory2Message.one_more_field')
@@ -352,6 +369,8 @@ class DescriptorPoolTest(unittest.TestCase):
   def testFindService(self):
     service = self.pool.FindServiceByName('protobuf_unittest.TestService')
     self.assertEqual(service.full_name, 'protobuf_unittest.TestService')
+    with self.assertRaises(KeyError):
+      self.pool.FindServiceByName('Does not exist')
 
   def testUserDefinedDB(self):
     db = descriptor_database.DescriptorDatabase()
@@ -361,24 +380,17 @@ class DescriptorPoolTest(unittest.TestCase):
     self.testFindMessageTypeByName()
 
   def testAddSerializedFile(self):
+    if isinstance(self, SecondaryDescriptorFromDescriptorDB):
+      if api_implementation.Type() == 'cpp':
+        # Cpp extension cannot call Add on a DescriptorPool
+        # that uses a DescriptorDatabase.
+        # TODO(jieluo): Fix python and cpp extension diff.
+        return
     self.pool = descriptor_pool.DescriptorPool()
     self.pool.AddSerializedFile(self.factory_test1_fd.SerializeToString())
     self.pool.AddSerializedFile(self.factory_test2_fd.SerializeToString())
     self.testFindMessageTypeByName()
 
-  def testComplexNesting(self):
-    more_messages_desc = descriptor_pb2.FileDescriptorProto.FromString(
-        more_messages_pb2.DESCRIPTOR.serialized_pb)
-    test1_desc = descriptor_pb2.FileDescriptorProto.FromString(
-        descriptor_pool_test1_pb2.DESCRIPTOR.serialized_pb)
-    test2_desc = descriptor_pb2.FileDescriptorProto.FromString(
-        descriptor_pool_test2_pb2.DESCRIPTOR.serialized_pb)
-    self.pool.Add(more_messages_desc)
-    self.pool.Add(test1_desc)
-    self.pool.Add(test2_desc)
-    TEST1_FILE.CheckFile(self, self.pool)
-    TEST2_FILE.CheckFile(self, self.pool)
-
 
   def testEnumDefaultValue(self):
     """Test the default value of enums which don't start at zero."""
@@ -397,6 +409,12 @@ class DescriptorPoolTest(unittest.TestCase):
     self.assertIs(file_descriptor, descriptor_pool_test1_pb2.DESCRIPTOR)
     _CheckDefaultValue(file_descriptor)
 
+    if isinstance(self, SecondaryDescriptorFromDescriptorDB):
+      if api_implementation.Type() == 'cpp':
+        # Cpp extension cannot call Add on a DescriptorPool
+        # that uses a DescriptorDatabase.
+        # TODO(jieluo): Fix python and cpp extension diff.
+        return
     # Then check the dynamic pool and its internal DescriptorDatabase.
     descriptor_proto = descriptor_pb2.FileDescriptorProto.FromString(
         descriptor_pool_test1_pb2.DESCRIPTOR.serialized_pb)
@@ -444,6 +462,110 @@ class DescriptorPoolTest(unittest.TestCase):
             unittest_pb2.TestAllTypes.DESCRIPTOR.full_name))
     _CheckDefaultValues(message_class())
 
+  def testAddFileDescriptor(self):
+    if isinstance(self, SecondaryDescriptorFromDescriptorDB):
+      if api_implementation.Type() == 'cpp':
+        # Cpp extension cannot call Add on a DescriptorPool
+        # that uses a DescriptorDatabase.
+        # TODO(jieluo): Fix python and cpp extension diff.
+        return
+    file_desc = descriptor_pb2.FileDescriptorProto(name='some/file.proto')
+    self.pool.Add(file_desc)
+    self.pool.AddSerializedFile(file_desc.SerializeToString())
+
+  def testComplexNesting(self):
+    if isinstance(self, SecondaryDescriptorFromDescriptorDB):
+      if api_implementation.Type() == 'cpp':
+        # Cpp extension cannot call Add on a DescriptorPool
+        # that uses a DescriptorDatabase.
+        # TODO(jieluo): Fix python and cpp extension diff.
+        return
+    more_messages_desc = descriptor_pb2.FileDescriptorProto.FromString(
+        more_messages_pb2.DESCRIPTOR.serialized_pb)
+    test1_desc = descriptor_pb2.FileDescriptorProto.FromString(
+        descriptor_pool_test1_pb2.DESCRIPTOR.serialized_pb)
+    test2_desc = descriptor_pb2.FileDescriptorProto.FromString(
+        descriptor_pool_test2_pb2.DESCRIPTOR.serialized_pb)
+    self.pool.Add(more_messages_desc)
+    self.pool.Add(test1_desc)
+    self.pool.Add(test2_desc)
+    TEST1_FILE.CheckFile(self, self.pool)
+    TEST2_FILE.CheckFile(self, self.pool)
+
+
+class DefaultDescriptorPoolTest(DescriptorPoolTestBase, unittest.TestCase):
+
+  def setUp(self):
+    self.pool = descriptor_pool.Default()
+    self.factory_test1_fd = descriptor_pb2.FileDescriptorProto.FromString(
+        factory_test1_pb2.DESCRIPTOR.serialized_pb)
+    self.factory_test2_fd = descriptor_pb2.FileDescriptorProto.FromString(
+        factory_test2_pb2.DESCRIPTOR.serialized_pb)
+
+  def testFindMethods(self):
+    self.assertIs(
+        self.pool.FindFileByName('google/protobuf/unittest.proto'),
+        unittest_pb2.DESCRIPTOR)
+    self.assertIs(
+        self.pool.FindMessageTypeByName('protobuf_unittest.TestAllTypes'),
+        unittest_pb2.TestAllTypes.DESCRIPTOR)
+    self.assertIs(
+        self.pool.FindFieldByName(
+            'protobuf_unittest.TestAllTypes.optional_int32'),
+        unittest_pb2.TestAllTypes.DESCRIPTOR.fields_by_name['optional_int32'])
+    self.assertIs(
+        self.pool.FindEnumTypeByName('protobuf_unittest.ForeignEnum'),
+        unittest_pb2.ForeignEnum.DESCRIPTOR)
+    self.assertIs(
+        self.pool.FindExtensionByName(
+            'protobuf_unittest.optional_int32_extension'),
+        unittest_pb2.DESCRIPTOR.extensions_by_name['optional_int32_extension'])
+    self.assertIs(
+        self.pool.FindOneofByName('protobuf_unittest.TestAllTypes.oneof_field'),
+        unittest_pb2.TestAllTypes.DESCRIPTOR.oneofs_by_name['oneof_field'])
+    self.assertIs(
+        self.pool.FindServiceByName('protobuf_unittest.TestService'),
+        unittest_pb2.DESCRIPTOR.services_by_name['TestService'])
+
+
+class CreateDescriptorPoolTest(DescriptorPoolTestBase, unittest.TestCase):
+
+  def setUp(self):
+    self.pool = descriptor_pool.DescriptorPool()
+    self.factory_test1_fd = descriptor_pb2.FileDescriptorProto.FromString(
+        factory_test1_pb2.DESCRIPTOR.serialized_pb)
+    self.factory_test2_fd = descriptor_pb2.FileDescriptorProto.FromString(
+        factory_test2_pb2.DESCRIPTOR.serialized_pb)
+    self.pool.Add(self.factory_test1_fd)
+    self.pool.Add(self.factory_test2_fd)
+
+    self.pool.Add(descriptor_pb2.FileDescriptorProto.FromString(
+        unittest_import_public_pb2.DESCRIPTOR.serialized_pb))
+    self.pool.Add(descriptor_pb2.FileDescriptorProto.FromString(
+        unittest_import_pb2.DESCRIPTOR.serialized_pb))
+    self.pool.Add(descriptor_pb2.FileDescriptorProto.FromString(
+        unittest_pb2.DESCRIPTOR.serialized_pb))
+
+
+class SecondaryDescriptorFromDescriptorDB(DescriptorPoolTestBase,
+                                          unittest.TestCase):
+
+  def setUp(self):
+    self.factory_test1_fd = descriptor_pb2.FileDescriptorProto.FromString(
+        factory_test1_pb2.DESCRIPTOR.serialized_pb)
+    self.factory_test2_fd = descriptor_pb2.FileDescriptorProto.FromString(
+        factory_test2_pb2.DESCRIPTOR.serialized_pb)
+    db = descriptor_database.DescriptorDatabase()
+    db.Add(self.factory_test1_fd)
+    db.Add(self.factory_test2_fd)
+    db.Add(descriptor_pb2.FileDescriptorProto.FromString(
+        unittest_import_public_pb2.DESCRIPTOR.serialized_pb))
+    db.Add(descriptor_pb2.FileDescriptorProto.FromString(
+        unittest_import_pb2.DESCRIPTOR.serialized_pb))
+    db.Add(descriptor_pb2.FileDescriptorProto.FromString(
+        unittest_pb2.DESCRIPTOR.serialized_pb))
+    self.pool = descriptor_pool.DescriptorPool(descriptor_db=db)
+
 
 class ProtoFile(object):
 
@@ -568,6 +690,11 @@ class MessageField(object):
     test.assertEqual(msg_desc, field_desc.containing_type)
     test.assertEqual(field_type_desc, field_desc.message_type)
     test.assertEqual(file_desc, field_desc.file)
+    # TODO(jieluo): Fix python and cpp extension diff for message field
+    # default value.
+    if api_implementation.Type() == 'cpp':
+      test.assertRaises(
+          NotImplementedError, getattr, field_desc, 'default_value')
 
 
 class StringField(object):
@@ -739,6 +866,25 @@ class AddDescriptorTest(unittest.TestCase):
                      'some/file.proto')
     self.assertEqual(pool.FindMessageTypeByName('package.Message').name,
                      'Message')
+    # Test no package
+    file_proto = descriptor_pb2.FileDescriptorProto(
+        name='some/filename/container.proto')
+    message_proto = file_proto.message_type.add(
+        name='TopMessage')
+    message_proto.field.add(
+        name='bb',
+        number=1,
+        type=descriptor_pb2.FieldDescriptorProto.TYPE_INT32,
+        label=descriptor_pb2.FieldDescriptorProto.LABEL_OPTIONAL)
+    enum_proto = file_proto.enum_type.add(name='TopEnum')
+    enum_proto.value.add(name='FOREIGN_FOO', number=4)
+    file_proto.service.add(name='TopService')
+    pool = descriptor_pool.DescriptorPool()
+    pool.Add(file_proto)
+    self.assertEqual('TopMessage',
+                     pool.FindMessageTypeByName('TopMessage').name)
+    self.assertEqual('TopEnum', pool.FindEnumTypeByName('TopEnum').name)
+    self.assertEqual('TopService', pool.FindServiceByName('TopService').name)
 
   def testFileDescriptorOptionsWithCustomDescriptorPool(self):
     # Create a descriptor pool, and add a new FileDescriptorProto to it.
@@ -757,40 +903,18 @@ class AddDescriptorTest(unittest.TestCase):
     # The object returned by GetOptions() is cached.
     self.assertIs(options, file_descriptor.GetOptions())
 
-
-class DefaultPoolTest(unittest.TestCase):
-
-  def testFindMethods(self):
-    pool = descriptor_pool.Default()
-    self.assertIs(
-        pool.FindFileByName('google/protobuf/unittest.proto'),
-        unittest_pb2.DESCRIPTOR)
-    self.assertIs(
-        pool.FindMessageTypeByName('protobuf_unittest.TestAllTypes'),
-        unittest_pb2.TestAllTypes.DESCRIPTOR)
-    self.assertIs(
-        pool.FindFieldByName('protobuf_unittest.TestAllTypes.optional_int32'),
-        unittest_pb2.TestAllTypes.DESCRIPTOR.fields_by_name['optional_int32'])
-    self.assertIs(
-        pool.FindEnumTypeByName('protobuf_unittest.ForeignEnum'),
-        unittest_pb2.ForeignEnum.DESCRIPTOR)
-    if api_implementation.Type() != 'cpp':
-      self.skipTest('Only the C++ implementation correctly indexes all types')
-    self.assertIs(
-        pool.FindExtensionByName('protobuf_unittest.optional_int32_extension'),
-        unittest_pb2.DESCRIPTOR.extensions_by_name['optional_int32_extension'])
-    self.assertIs(
-        pool.FindOneofByName('protobuf_unittest.TestAllTypes.oneof_field'),
-        unittest_pb2.TestAllTypes.DESCRIPTOR.oneofs_by_name['oneof_field'])
-    self.assertIs(
-        pool.FindServiceByName('protobuf_unittest.TestService'),
-        unittest_pb2.DESCRIPTOR.services_by_name['TestService'])
-
-  def testAddFileDescriptor(self):
-    pool = descriptor_pool.Default()
-    file_desc = descriptor_pb2.FileDescriptorProto(name='some/file.proto')
-    pool.Add(file_desc)
-    pool.AddSerializedFile(file_desc.SerializeToString())
+  def testAddTypeError(self):
+    pool = descriptor_pool.DescriptorPool()
+    with self.assertRaises(TypeError):
+      pool.AddDescriptor(0)
+    with self.assertRaises(TypeError):
+      pool.AddEnumDescriptor(0)
+    with self.assertRaises(TypeError):
+      pool.AddServiceDescriptor(0)
+    with self.assertRaises(TypeError):
+      pool.AddExtensionDescriptor(0)
+    with self.assertRaises(TypeError):
+      pool.AddFileDescriptor(0)
 
 
 TEST1_FILE = ProtoFile(

python/google/protobuf/internal/descriptor_test.py

@@ -107,6 +107,12 @@ class DescriptorTest(unittest.TestCase):
         self.my_message.enum_types_by_name[
             'ForeignEnum'].values_by_number[4].name,
         self.my_message.EnumValueName('ForeignEnum', 4))
+    with self.assertRaises(KeyError):
+      self.my_message.EnumValueName('ForeignEnum', 999)
+    with self.assertRaises(KeyError):
+      self.my_message.EnumValueName('NoneEnum', 999)
+    with self.assertRaises(TypeError):
+      self.my_message.EnumValueName()
 
   def testEnumFixups(self):
     self.assertEqual(self.my_enum, self.my_enum.values[0].type)
@@ -134,15 +140,17 @@ class DescriptorTest(unittest.TestCase):
 
   def testSimpleCustomOptions(self):
     file_descriptor = unittest_custom_options_pb2.DESCRIPTOR
-    message_descriptor =\
-        unittest_custom_options_pb2.TestMessageWithCustomOptions.DESCRIPTOR
+    message_descriptor = (unittest_custom_options_pb2.
+                          TestMessageWithCustomOptions.DESCRIPTOR)
     field_descriptor = message_descriptor.fields_by_name['field1']
     oneof_descriptor = message_descriptor.oneofs_by_name['AnOneof']
     enum_descriptor = message_descriptor.enum_types_by_name['AnEnum']
-    enum_value_descriptor =\
-        message_descriptor.enum_values_by_name['ANENUM_VAL2']
-    service_descriptor =\
-        unittest_custom_options_pb2.TestServiceWithCustomOptions.DESCRIPTOR
+    enum_value_descriptor = (message_descriptor.
+                             enum_values_by_name['ANENUM_VAL2'])
+    other_enum_value_descriptor = (message_descriptor.
+                                   enum_values_by_name['ANENUM_VAL1'])
+    service_descriptor = (unittest_custom_options_pb2.
+                          TestServiceWithCustomOptions.DESCRIPTOR)
     method_descriptor = service_descriptor.FindMethodByName('Foo')
 
     file_options = file_descriptor.GetOptions()
@@ -178,6 +186,11 @@ class DescriptorTest(unittest.TestCase):
         unittest_custom_options_pb2.DummyMessageContainingEnum.DESCRIPTOR)
     self.assertTrue(file_descriptor.has_options)
     self.assertFalse(message_descriptor.has_options)
+    self.assertTrue(field_descriptor.has_options)
+    self.assertTrue(oneof_descriptor.has_options)
+    self.assertTrue(enum_descriptor.has_options)
+    self.assertTrue(enum_value_descriptor.has_options)
+    self.assertFalse(other_enum_value_descriptor.has_options)
 
   def testDifferentCustomOptionTypes(self):
     kint32min = -2**31
@@ -400,6 +413,12 @@ class DescriptorTest(unittest.TestCase):
     self.assertEqual(self.my_file.name, 'some/filename/some.proto')
     self.assertEqual(self.my_file.package, 'protobuf_unittest')
     self.assertEqual(self.my_file.pool, self.pool)
+    self.assertFalse(self.my_file.has_options)
+    self.assertEqual('proto2', self.my_file.syntax)
+    file_proto = descriptor_pb2.FileDescriptorProto()
+    self.my_file.CopyToProto(file_proto)
+    self.assertEqual(self.my_file.serialized_pb,
+                     file_proto.SerializeToString())
     # Generated modules also belong to the default pool.
     self.assertEqual(unittest_pb2.DESCRIPTOR.pool, descriptor_pool.Default())
 
@@ -407,13 +426,31 @@ class DescriptorTest(unittest.TestCase):
       api_implementation.Type() != 'cpp' or api_implementation.Version() != 2,
       'Immutability of descriptors is only enforced in v2 implementation')
   def testImmutableCppDescriptor(self):
+    file_descriptor = unittest_pb2.DESCRIPTOR
     message_descriptor = unittest_pb2.TestAllTypes.DESCRIPTOR
+    field_descriptor = message_descriptor.fields_by_name['optional_int32']
+    enum_descriptor = message_descriptor.enum_types_by_name['NestedEnum']
+    oneof_descriptor = message_descriptor.oneofs_by_name['oneof_field']
     with self.assertRaises(AttributeError):
       message_descriptor.fields_by_name = None
     with self.assertRaises(TypeError):
       message_descriptor.fields_by_name['Another'] = None
     with self.assertRaises(TypeError):
       message_descriptor.fields.append(None)
+    with self.assertRaises(AttributeError):
+      field_descriptor.containing_type = message_descriptor
+    with self.assertRaises(AttributeError):
+      file_descriptor.has_options = False
+    with self.assertRaises(AttributeError):
+      field_descriptor.has_options = False
+    with self.assertRaises(AttributeError):
+      oneof_descriptor.has_options = False
+    with self.assertRaises(AttributeError):
+      enum_descriptor.has_options = False
+    with self.assertRaises(AttributeError) as e:
+      message_descriptor.has_options = True
+    self.assertEqual('attribute is not writable: has_options',
+                     str(e.exception))
 
 
 class NewDescriptorTest(DescriptorTest):
@@ -442,6 +479,12 @@ class GeneratedDescriptorTest(unittest.TestCase):
     self.CheckDescriptorMapping(message_descriptor.fields_by_name)
     self.CheckDescriptorMapping(message_descriptor.fields_by_number)
     self.CheckDescriptorMapping(message_descriptor.fields_by_camelcase_name)
+    self.CheckDescriptorMapping(message_descriptor.enum_types_by_name)
+    self.CheckDescriptorMapping(message_descriptor.enum_values_by_name)
+    self.CheckDescriptorMapping(message_descriptor.oneofs_by_name)
+    self.CheckDescriptorMapping(message_descriptor.enum_types[0].values_by_name)
+    # Test extension range
+    self.assertEqual(message_descriptor.extension_ranges, [])
 
   def CheckFieldDescriptor(self, field_descriptor):
     # Basic properties
@@ -450,6 +493,7 @@ class GeneratedDescriptorTest(unittest.TestCase):
     self.assertEqual(field_descriptor.full_name,
                      'protobuf_unittest.TestAllTypes.optional_int32')
     self.assertEqual(field_descriptor.containing_type.name, 'TestAllTypes')
+    self.assertEqual(field_descriptor.file, unittest_pb2.DESCRIPTOR)
     # Test equality and hashability
     self.assertEqual(field_descriptor, field_descriptor)
     self.assertEqual(
@@ -461,32 +505,73 @@ class GeneratedDescriptorTest(unittest.TestCase):
         field_descriptor)
     self.assertIn(field_descriptor, [field_descriptor])
     self.assertIn(field_descriptor, {field_descriptor: None})
+    self.assertEqual(None, field_descriptor.extension_scope)
+    self.assertEqual(None, field_descriptor.enum_type)
+    if api_implementation.Type() == 'cpp':
+      # For test coverage only
+      self.assertEqual(field_descriptor.id, field_descriptor.id)
 
   def CheckDescriptorSequence(self, sequence):
     # Verifies that a property like 'messageDescriptor.fields' has all the
     # properties of an immutable abc.Sequence.
+    self.assertNotEqual(sequence,
+                        unittest_pb2.TestAllExtensions.DESCRIPTOR.fields)
+    self.assertNotEqual(sequence, [])
+    self.assertNotEqual(sequence, 1)
+    self.assertFalse(sequence == 1)  # Only for cpp test coverage
+    self.assertEqual(sequence, sequence)
+    expected_list = list(sequence)
+    self.assertEqual(expected_list, sequence)
     self.assertGreater(len(sequence), 0)  # Sized
-    self.assertEqual(len(sequence), len(list(sequence)))  # Iterable
+    self.assertEqual(len(sequence), len(expected_list))  # Iterable
+    self.assertEqual(sequence[len(sequence) -1], sequence[-1])
     item = sequence[0]
     self.assertEqual(item, sequence[0])
     self.assertIn(item, sequence)  # Container
     self.assertEqual(sequence.index(item), 0)
     self.assertEqual(sequence.count(item), 1)
+    other_item = unittest_pb2.NestedTestAllTypes.DESCRIPTOR.fields[0]
+    self.assertNotIn(other_item, sequence)
+    self.assertEqual(sequence.count(other_item), 0)
+    self.assertRaises(ValueError, sequence.index, other_item)
+    self.assertRaises(ValueError, sequence.index, [])
     reversed_iterator = reversed(sequence)
     self.assertEqual(list(reversed_iterator), list(sequence)[::-1])
     self.assertRaises(StopIteration, next, reversed_iterator)
+    expected_list[0] = 'change value'
+    self.assertNotEqual(expected_list, sequence)
+    # TODO(jieluo): Change __repr__ support for DescriptorSequence.
+    if api_implementation.Type() == 'python':
+      self.assertEqual(str(list(sequence)), str(sequence))
+    else:
+      self.assertEqual(str(sequence)[0], '<')
 
   def CheckDescriptorMapping(self, mapping):
     # Verifies that a property like 'messageDescriptor.fields' has all the
     # properties of an immutable abc.Mapping.
+    self.assertNotEqual(
+        mapping, unittest_pb2.TestAllExtensions.DESCRIPTOR.fields_by_name)
+    self.assertNotEqual(mapping, {})
+    self.assertNotEqual(mapping, 1)
+    self.assertFalse(mapping == 1)  # Only for cpp test coverage
+    excepted_dict = dict(mapping.items())
+    self.assertEqual(mapping, excepted_dict)
+    self.assertEqual(mapping, mapping)
     self.assertGreater(len(mapping), 0)  # Sized
-    self.assertEqual(len(mapping), len(list(mapping)))  # Iterable
+    self.assertEqual(len(mapping), len(excepted_dict))  # Iterable
     if sys.version_info >= (3,):
       key, item = next(iter(mapping.items()))
     else:
       key, item = mapping.items()[0]
     self.assertIn(key, mapping)  # Container
     self.assertEqual(mapping.get(key), item)
+    with self.assertRaises(TypeError):
+      mapping.get()
+    # TODO(jieluo): Fix python and cpp extension diff.
+    if api_implementation.Type() == 'python':
+      self.assertRaises(TypeError, mapping.get, [])
+    else:
+      self.assertEqual(None, mapping.get([]))
     # keys(), iterkeys() &co
     item = (next(iter(mapping.keys())), next(iter(mapping.values())))
     self.assertEqual(item, next(iter(mapping.items())))
@@ -497,6 +582,18 @@ class GeneratedDescriptorTest(unittest.TestCase):
       CheckItems(mapping.keys(), mapping.iterkeys())
       CheckItems(mapping.values(), mapping.itervalues())
       CheckItems(mapping.items(), mapping.iteritems())
+    excepted_dict[key] = 'change value'
+    self.assertNotEqual(mapping, excepted_dict)
+    del excepted_dict[key]
+    excepted_dict['new_key'] = 'new'
+    self.assertNotEqual(mapping, excepted_dict)
+    self.assertRaises(KeyError, mapping.__getitem__, 'key_error')
+    self.assertRaises(KeyError, mapping.__getitem__, len(mapping) + 1)
+    # TODO(jieluo): Add __repr__ support for DescriptorMapping.
+    if api_implementation.Type() == 'python':
+      self.assertEqual(len(str(dict(mapping.items()))), len(str(mapping)))
+    else:
+      self.assertEqual(str(mapping)[0], '<')
 
   def testDescriptor(self):
     message_descriptor = unittest_pb2.TestAllTypes.DESCRIPTOR
@@ -506,13 +603,26 @@ class GeneratedDescriptorTest(unittest.TestCase):
     field_descriptor = message_descriptor.fields_by_camelcase_name[
         'optionalInt32']
     self.CheckFieldDescriptor(field_descriptor)
+    enum_descriptor = unittest_pb2.DESCRIPTOR.enum_types_by_name[
+        'ForeignEnum']
+    self.assertEqual(None, enum_descriptor.containing_type)
+    # Test extension range
+    self.assertEqual(
+        unittest_pb2.TestAllExtensions.DESCRIPTOR.extension_ranges,
+        [(1, 536870912)])
+    self.assertEqual(
+        unittest_pb2.TestMultipleExtensionRanges.DESCRIPTOR.extension_ranges,
+        [(42, 43), (4143, 4244), (65536, 536870912)])
 
   def testCppDescriptorContainer(self):
-    # Check that the collection is still valid even if the parent disappeared.
-    enum = unittest_pb2.TestAllTypes.DESCRIPTOR.enum_types_by_name['NestedEnum']
-    values = enum.values
-    del enum
-    self.assertEqual('FOO', values[0].name)
+    containing_file = unittest_pb2.DESCRIPTOR
+    self.CheckDescriptorSequence(containing_file.dependencies)
+    self.CheckDescriptorMapping(containing_file.message_types_by_name)
+    self.CheckDescriptorMapping(containing_file.enum_types_by_name)
+    self.CheckDescriptorMapping(containing_file.services_by_name)
+    self.CheckDescriptorMapping(containing_file.extensions_by_name)
+    self.CheckDescriptorMapping(
+        unittest_pb2.TestNestedExtension.DESCRIPTOR.extensions_by_name)
 
   def testCppDescriptorContainer_Iterator(self):
     # Same test with the iterator
@@ -526,6 +636,18 @@ class GeneratedDescriptorTest(unittest.TestCase):
     self.assertEqual(service_descriptor.name, 'TestService')
     self.assertEqual(service_descriptor.methods[0].name, 'Foo')
     self.assertIs(service_descriptor.file, unittest_pb2.DESCRIPTOR)
+    self.assertEqual(service_descriptor.index, 0)
+    self.CheckDescriptorMapping(service_descriptor.methods_by_name)
+
+  def testOneofDescriptor(self):
+    message_descriptor = unittest_pb2.TestAllTypes.DESCRIPTOR
+    oneof_descriptor = message_descriptor.oneofs_by_name['oneof_field']
+    self.assertFalse(oneof_descriptor.has_options)
+    self.assertEqual(message_descriptor, oneof_descriptor.containing_type)
+    self.assertEqual('oneof_field', oneof_descriptor.name)
+    self.assertEqual('protobuf_unittest.TestAllTypes.oneof_field',
+                     oneof_descriptor.full_name)
+    self.assertEqual(0, oneof_descriptor.index)
 
 
 class DescriptorCopyToProtoTest(unittest.TestCase):
@@ -663,49 +785,64 @@ class DescriptorCopyToProtoTest(unittest.TestCase):
         descriptor_pb2.DescriptorProto,
         TEST_MESSAGE_WITH_SEVERAL_EXTENSIONS_ASCII)
 
-  # Disable this test so we can make changes to the proto file.
-  # TODO(xiaofeng): Enable this test after cl/55530659 is submitted.
-  #
-  # def testCopyToProto_FileDescriptor(self):
-  #   UNITTEST_IMPORT_FILE_DESCRIPTOR_ASCII = ("""
-  #     name: 'google/protobuf/unittest_import.proto'
-  #     package: 'protobuf_unittest_import'
-  #     dependency: 'google/protobuf/unittest_import_public.proto'
-  #     message_type: <
-  #       name: 'ImportMessage'
-  #       field: <
-  #         name: 'd'
-  #         number: 1
-  #         label: 1  # Optional
-  #         type: 5  # TYPE_INT32
-  #       >
-  #     >
-  #     """ +
-  #     """enum_type: <
-  #       name: 'ImportEnum'
-  #       value: <
-  #         name: 'IMPORT_FOO'
-  #         number: 7
-  #       >
-  #       value: <
-  #         name: 'IMPORT_BAR'
-  #         number: 8
-  #       >
-  #       value: <
-  #         name: 'IMPORT_BAZ'
-  #         number: 9
-  #       >
-  #     >
-  #     options: <
-  #       java_package: 'com.google.protobuf.test'
-  #       optimize_for: 1  # SPEED
-  #     >
-  #     public_dependency: 0
-  #  """)
-  #  self._InternalTestCopyToProto(
-  #      unittest_import_pb2.DESCRIPTOR,
-  #      descriptor_pb2.FileDescriptorProto,
-  #      UNITTEST_IMPORT_FILE_DESCRIPTOR_ASCII)
+  def testCopyToProto_FileDescriptor(self):
+    UNITTEST_IMPORT_FILE_DESCRIPTOR_ASCII = ("""
+      name: 'google/protobuf/unittest_import.proto'
+      package: 'protobuf_unittest_import'
+      dependency: 'google/protobuf/unittest_import_public.proto'
+      message_type: <
+        name: 'ImportMessage'
+        field: <
+          name: 'd'
+          number: 1
+          label: 1  # Optional
+          type: 5  # TYPE_INT32
+        >
+      >
+      """ +
+      """enum_type: <
+        name: 'ImportEnum'
+        value: <
+          name: 'IMPORT_FOO'
+          number: 7
+        >
+        value: <
+          name: 'IMPORT_BAR'
+          number: 8
+        >
+        value: <
+          name: 'IMPORT_BAZ'
+          number: 9
+        >
+      >
+      enum_type: <
+        name: 'ImportEnumForMap'
+        value: <
+          name: 'UNKNOWN'
+          number: 0
+        >
+        value: <
+          name: 'FOO'
+          number: 1
+        >
+        value: <
+          name: 'BAR'
+          number: 2
+        >
+      >
+      options: <
+        java_package: 'com.google.protobuf.test'
+        optimize_for: 1  # SPEED
+      """ +
+      """
+        cc_enable_arenas: true
+      >
+      public_dependency: 0
+    """)
+    self._InternalTestCopyToProto(
+        unittest_import_pb2.DESCRIPTOR,
+        descriptor_pb2.FileDescriptorProto,
+        UNITTEST_IMPORT_FILE_DESCRIPTOR_ASCII)
 
   def testCopyToProto_ServiceDescriptor(self):
     TEST_SERVICE_ASCII = """
@@ -721,12 +858,47 @@ class DescriptorCopyToProtoTest(unittest.TestCase):
         output_type: '.protobuf_unittest.BarResponse'
       >
       """
-    # TODO(rocking): enable this test after the proto descriptor change is
-    # checked in.
-    #self._InternalTestCopyToProto(
-    #    unittest_pb2.TestService.DESCRIPTOR,
-    #    descriptor_pb2.ServiceDescriptorProto,
-    #    TEST_SERVICE_ASCII)
+    self._InternalTestCopyToProto(
+        unittest_pb2.TestService.DESCRIPTOR,
+        descriptor_pb2.ServiceDescriptorProto,
+        TEST_SERVICE_ASCII)
+
+  @unittest.skipIf(
+      api_implementation.Type() == 'python',
+      'It is not implemented in python.')
+  # TODO(jieluo): Add support for pure python or remove in c extension.
+  def testCopyToProto_MethodDescriptor(self):
+    expected_ascii = """
+      name: 'Foo'
+      input_type: '.protobuf_unittest.FooRequest'
+      output_type: '.protobuf_unittest.FooResponse'
+    """
+    method_descriptor = unittest_pb2.TestService.DESCRIPTOR.FindMethodByName(
+        'Foo')
+    self._InternalTestCopyToProto(
+        method_descriptor,
+        descriptor_pb2.MethodDescriptorProto,
+        expected_ascii)
+
+  @unittest.skipIf(
+      api_implementation.Type() == 'python',
+      'Pure python does not raise error.')
+  # TODO(jieluo): Fix pure python to check with the proto type.
+  def testCopyToProto_TypeError(self):
+    file_proto = descriptor_pb2.FileDescriptorProto()
+    self.assertRaises(TypeError,
+                      unittest_pb2.TestEmptyMessage.DESCRIPTOR.CopyToProto,
+                      file_proto)
+    self.assertRaises(TypeError,
+                      unittest_pb2.ForeignEnum.DESCRIPTOR.CopyToProto,
+                      file_proto)
+    self.assertRaises(TypeError,
+                      unittest_pb2.TestService.DESCRIPTOR.CopyToProto,
+                      file_proto)
+    proto = descriptor_pb2.DescriptorProto()
+    self.assertRaises(TypeError,
+                      unittest_import_pb2.DESCRIPTOR.CopyToProto,
+                      proto)
 
 
 class MakeDescriptorTest(unittest.TestCase):
@@ -774,6 +946,9 @@ class MakeDescriptorTest(unittest.TestCase):
                      result.nested_types[0].enum_types[0])
     self.assertFalse(result.has_options)
     self.assertFalse(result.fields[0].has_options)
+    if api_implementation.Type() == 'cpp':
+      with self.assertRaises(AttributeError):
+        result.fields[0].has_options = False
 
   def testMakeDescriptorWithUnsignedIntField(self):
     file_descriptor_proto = descriptor_pb2.FileDescriptorProto()

python/google/protobuf/internal/encoder.py

@@ -819,7 +819,7 @@ def MapEncoder(field_descriptor):
   encode_message = MessageEncoder(field_descriptor.number, False, False)
 
   def EncodeField(write, value, deterministic):
-    value_keys = sorted(value.keys()) if deterministic else value.keys()
+    value_keys = sorted(value.keys()) if deterministic else value
     for key in value_keys:
       entry_msg = message_type._concrete_class(key=key, value=value[key])
       encode_message(write, entry_msg, deterministic)

python/google/protobuf/internal/json_format_test.py

@@ -159,6 +159,16 @@ class JsonFormatTest(JsonFormatBase):
     json_format.Parse(text, parsed_message)
     self.assertEqual(message, parsed_message)
 
+  def testUnknownEnumToJsonError(self):
+    message = json_format_proto3_pb2.TestMessage()
+    message.enum_value = 999
+    # TODO(jieluo): should accept numeric unknown enum for proto3.
+    with self.assertRaises(json_format.SerializeToJsonError) as e:
+      json_format.MessageToJson(message)
+    self.assertEqual(str(e.exception),
+                     'Enum field contains an integer value which can '
+                     'not mapped to an enum value.')
+
   def testExtensionToJsonAndBack(self):
     message = unittest_mset_pb2.TestMessageSetContainer()
     ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension
@@ -172,6 +182,10 @@ class JsonFormatTest(JsonFormatBase):
     json_format.Parse(message_text, parsed_message)
     self.assertEqual(message, parsed_message)
 
+  def testExtensionErrors(self):
+    self.CheckError('{"[extensionField]": {}}',
+                    'Message type proto3.TestMessage does not have extensions')
+
   def testExtensionToDictAndBack(self):
     message = unittest_mset_pb2.TestMessageSetContainer()
     ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension
@@ -294,7 +308,18 @@ class JsonFormatTest(JsonFormatBase):
     self.assertEqual(message.int32_value, 1)
 
   def testMapFields(self):
-    message = json_format_proto3_pb2.TestMap()
+    message = json_format_proto3_pb2.TestNestedMap()
+    self.assertEqual(
+        json.loads(json_format.MessageToJson(message, True)),
+        json.loads('{'
+                   '"boolMap": {},'
+                   '"int32Map": {},'
+                   '"int64Map": {},'
+                   '"uint32Map": {},'
+                   '"uint64Map": {},'
+                   '"stringMap": {},'
+                   '"mapMap": {}'
+                   '}'))
     message.bool_map[True] = 1
     message.bool_map[False] = 2
     message.int32_map[1] = 2
@@ -307,17 +332,19 @@ class JsonFormatTest(JsonFormatBase):
     message.uint64_map[2] = 3
     message.string_map['1'] = 2
     message.string_map['null'] = 3
+    message.map_map['1'].bool_map[True] = 3
     self.assertEqual(
-        json.loads(json_format.MessageToJson(message, True)),
+        json.loads(json_format.MessageToJson(message, False)),
         json.loads('{'
                    '"boolMap": {"false": 2, "true": 1},'
                    '"int32Map": {"1": 2, "2": 3},'
                    '"int64Map": {"1": 2, "2": 3},'
                    '"uint32Map": {"1": 2, "2": 3},'
                    '"uint64Map": {"1": 2, "2": 3},'
-                   '"stringMap": {"1": 2, "null": 3}'
+                   '"stringMap": {"1": 2, "null": 3},'
+                   '"mapMap": {"1": {"boolMap": {"true": 3}}}'
                    '}'))
-    parsed_message = json_format_proto3_pb2.TestMap()
+    parsed_message = json_format_proto3_pb2.TestNestedMap()
     self.CheckParseBack(message, parsed_message)
 
   def testOneofFields(self):
@@ -703,6 +730,9 @@ class JsonFormatTest(JsonFormatBase):
         json_format.Parse,
         '{"repeatedInt32Value":[1, null]}',
         parsed_message)
+    self.CheckError('{"repeatedMessageValue":[null]}',
+                    'Failed to parse repeatedMessageValue field: null is not'
+                    ' allowed to be used as an element in a repeated field.')
 
   def testNanFloat(self):
     message = json_format_proto3_pb2.TestMessage()
@@ -727,6 +757,11 @@ class JsonFormatTest(JsonFormatBase):
         '{"enumValue": "baz"}',
         'Failed to parse enumValue field: Invalid enum value baz '
         'for enum type proto3.EnumType.')
+    # TODO(jieluo): fix json format to accept numeric unknown enum for proto3.
+    self.CheckError(
+        '{"enumValue": 12345}',
+        'Failed to parse enumValue field: Invalid enum value 12345 '
+        'for enum type proto3.EnumType.')
 
   def testParseBadIdentifer(self):
     self.CheckError('{int32Value: 1}',
@@ -799,6 +834,11 @@ class JsonFormatTest(JsonFormatBase):
     self.CheckError('{"bytesValue": "AQI*"}',
                     'Failed to parse bytesValue field: Incorrect padding.')
 
+  def testInvalidRepeated(self):
+    self.CheckError('{"repeatedInt32Value": 12345}',
+                    (r'Failed to parse repeatedInt32Value field: repeated field'
+                     r' repeatedInt32Value must be in \[\] which is 12345.'))
+
   def testInvalidMap(self):
     message = json_format_proto3_pb2.TestMap()
     text = '{"int32Map": {"null": 2, "2": 3}}'
@@ -824,6 +864,12 @@ class JsonFormatTest(JsonFormatBase):
         json_format.ParseError,
         'Failed to load JSON: duplicate key a',
         json_format.Parse, text, message)
+    text = r'{"stringMap": 0}'
+    self.assertRaisesRegexp(
+        json_format.ParseError,
+        'Failed to parse stringMap field: Map field string_map must be '
+        'in a dict which is 0.',
+        json_format.Parse, text, message)
 
   def testInvalidTimestamp(self):
     message = json_format_proto3_pb2.TestTimestamp()
@@ -911,6 +957,12 @@ class JsonFormatTest(JsonFormatBase):
                      json_format.MessageToJson(message))
     self.assertEqual('{\n  "int32_value": 12345\n}',
                      json_format.MessageToJson(message, False, True))
+    # When including_default_value_fields is True.
+    message = json_format_proto3_pb2.TestTimestamp()
+    self.assertEqual('{\n  "repeatedValue": []\n}',
+                     json_format.MessageToJson(message, True, False))
+    self.assertEqual('{\n  "repeated_value": []\n}',
+                     json_format.MessageToJson(message, True, True))
 
     # Parsers accept both original proto field names and lowerCamelCase names.
     message = json_format_proto3_pb2.TestMessage()

python/google/protobuf/internal/message_factory_test.py

@@ -40,6 +40,7 @@ except ImportError:
   import unittest
 
 from google.protobuf import descriptor_pb2
+from google.protobuf.internal import api_implementation
 from google.protobuf.internal import factory_test1_pb2
 from google.protobuf.internal import factory_test2_pb2
 from google.protobuf import descriptor_database
@@ -130,6 +131,21 @@ class MessageFactoryTest(unittest.TestCase):
       msg1.Extensions[ext2] = 'test2'
       self.assertEqual('test1', msg1.Extensions[ext1])
       self.assertEqual('test2', msg1.Extensions[ext2])
+      self.assertEqual(None,
+                       msg1.Extensions._FindExtensionByNumber(12321))
+      if api_implementation.Type() == 'cpp':
+        # TODO(jieluo): Fix len to return the correct value.
+        # self.assertEqual(2, len(msg1.Extensions))
+        self.assertEqual(len(msg1.Extensions), len(msg1.Extensions))
+        self.assertRaises(TypeError,
+                          msg1.Extensions._FindExtensionByName, 0)
+        self.assertRaises(TypeError,
+                          msg1.Extensions._FindExtensionByNumber, '')
+      else:
+        self.assertEqual(None,
+                         msg1.Extensions._FindExtensionByName(0))
+        self.assertEqual(None,
+                         msg1.Extensions._FindExtensionByNumber(''))
 
   def testDuplicateExtensionNumber(self):
     pool = descriptor_pool.DescriptorPool()

python/google/protobuf/internal/message_test.py

@@ -61,6 +61,7 @@ try:
 except NameError:
   cmp = lambda x, y: (x > y) - (x < y)  # Python 3
 
+from google.protobuf import map_proto2_unittest_pb2
 from google.protobuf import map_unittest_pb2
 from google.protobuf import unittest_pb2
 from google.protobuf import unittest_proto3_arena_pb2
@@ -69,6 +70,7 @@ from google.protobuf import descriptor_pool
 from google.protobuf import message_factory
 from google.protobuf import text_format
 from google.protobuf.internal import api_implementation
+from google.protobuf.internal import encoder
 from google.protobuf.internal import packed_field_test_pb2
 from google.protobuf.internal import test_util
 from google.protobuf.internal import testing_refleaks
@@ -140,6 +142,18 @@ class MessageTest(BaseTestCase):
     golden_copy = copy.deepcopy(golden_message)
     self.assertEqual(golden_data, golden_copy.SerializeToString())
 
+  def testParseErrors(self, message_module):
+    msg = message_module.TestAllTypes()
+    self.assertRaises(TypeError, msg.FromString, 0)
+    self.assertRaises(Exception, msg.FromString, '0')
+    # TODO(jieluo): Fix cpp extension to check unexpected end-group tag.
+    # b/27494216
+    if api_implementation.Type() == 'python':
+      end_tag = encoder.TagBytes(1, 4)
+      with self.assertRaises(message.DecodeError) as context:
+        msg.FromString(end_tag)
+      self.assertEqual('Unexpected end-group tag.', str(context.exception))
+
   def testDeterminismParameters(self, message_module):
     # This message is always deterministically serialized, even if determinism
     # is disabled, so we can use it to verify that all the determinism
@@ -609,6 +623,13 @@ class MessageTest(BaseTestCase):
     self.assertIsInstance(m.repeated_nested_message,
                           collections.MutableSequence)
 
+  def testRepeatedFieldsNotHashable(self, message_module):
+    m = message_module.TestAllTypes()
+    with self.assertRaises(TypeError):
+      hash(m.repeated_int32)
+    with self.assertRaises(TypeError):
+      hash(m.repeated_nested_message)
+
   def testRepeatedFieldInsideNestedMessage(self, message_module):
     m = message_module.NestedTestAllTypes()
     m.payload.repeated_int32.extend([])
@@ -626,6 +647,7 @@ class MessageTest(BaseTestCase):
   def testOneofGetCaseNonexistingField(self, message_module):
     m = message_module.TestAllTypes()
     self.assertRaises(ValueError, m.WhichOneof, 'no_such_oneof_field')
+    self.assertRaises(Exception, m.WhichOneof, 0)
 
   def testOneofDefaultValues(self, message_module):
     m = message_module.TestAllTypes()
@@ -1001,6 +1023,8 @@ class MessageTest(BaseTestCase):
     m = message_module.TestAllTypes()
     with self.assertRaises(IndexError) as _:
       m.repeated_nested_message.pop()
+    with self.assertRaises(TypeError) as _:
+      m.repeated_nested_message.pop('0')
     for i in range(5):
       n = m.repeated_nested_message.add()
       n.bb = i
@@ -1009,6 +1033,39 @@ class MessageTest(BaseTestCase):
     self.assertEqual(2, m.repeated_nested_message.pop(1).bb)
     self.assertEqual([1, 3], [n.bb for n in m.repeated_nested_message])
 
+  def testRepeatedCompareWithSelf(self, message_module):
+    m = message_module.TestAllTypes()
+    for i in range(5):
+      m.repeated_int32.insert(i, i)
+      n = m.repeated_nested_message.add()
+      n.bb = i
+    self.assertSequenceEqual(m.repeated_int32, m.repeated_int32)
+    self.assertEqual(m.repeated_nested_message, m.repeated_nested_message)
+
+  def testReleasedNestedMessages(self, message_module):
+    """A case that lead to a segfault when a message detached from its parent
+    container has itself a child container.
+    """
+    m = message_module.NestedTestAllTypes()
+    m = m.repeated_child.add()
+    m = m.child
+    m = m.repeated_child.add()
+    self.assertEqual(m.payload.optional_int32, 0)
+
+  def testSetRepeatedComposite(self, message_module):
+    m = message_module.TestAllTypes()
+    with self.assertRaises(AttributeError):
+      m.repeated_int32 = []
+    m.repeated_int32.append(1)
+    if api_implementation.Type() == 'cpp':
+      # For test coverage: cpp has a different path if composite
+      # field is in cache
+      with self.assertRaises(TypeError):
+        m.repeated_int32 = []
+    else:
+      with self.assertRaises(AttributeError):
+        m.repeated_int32 = []
+
 
 # Class to test proto2-only features (required, extensions, etc.)
 class Proto2Test(BaseTestCase):
@@ -1061,18 +1118,46 @@ class Proto2Test(BaseTestCase):
     self.assertEqual(False, message.optional_bool)
     self.assertEqual(0, message.optional_nested_message.bb)
 
-  # TODO(tibell): The C++ implementations actually allows assignment
-  # of unknown enum values to *scalar* fields (but not repeated
-  # fields). Once checked enum fields becomes the default in the
-  # Python implementation, the C++ implementation should follow suit.
   def testAssignInvalidEnum(self):
-    """It should not be possible to assign an invalid enum number to an
-    enum field."""
+    """Assigning an invalid enum number is not allowed in proto2."""
     m = unittest_pb2.TestAllTypes()
 
+    # Proto2 can not assign unknown enum.
     with self.assertRaises(ValueError) as _:
       m.optional_nested_enum = 1234567
     self.assertRaises(ValueError, m.repeated_nested_enum.append, 1234567)
+    # Assignment is a different code path than append for the C++ impl.
+    m.repeated_nested_enum.append(2)
+    m.repeated_nested_enum[0] = 2
+    with self.assertRaises(ValueError):
+      m.repeated_nested_enum[0] = 123456
+
+    # Unknown enum value can be parsed but is ignored.
+    m2 = unittest_proto3_arena_pb2.TestAllTypes()
+    m2.optional_nested_enum = 1234567
+    m2.repeated_nested_enum.append(7654321)
+    serialized = m2.SerializeToString()
+
+    m3 = unittest_pb2.TestAllTypes()
+    m3.ParseFromString(serialized)
+    self.assertFalse(m3.HasField('optional_nested_enum'))
+    # 1 is the default value for optional_nested_enum.
+    self.assertEqual(1, m3.optional_nested_enum)
+    self.assertEqual(0, len(m3.repeated_nested_enum))
+    m2.Clear()
+    m2.ParseFromString(m3.SerializeToString())
+    self.assertEqual(1234567, m2.optional_nested_enum)
+    self.assertEqual(7654321, m2.repeated_nested_enum[0])
+
+  def testUnknownEnumMap(self):
+    m = map_proto2_unittest_pb2.TestEnumMap()
+    m.known_map_field[123] = 0
+    with self.assertRaises(ValueError):
+      m.unknown_map_field[1] = 123
+
+  def testExtensionsErrors(self):
+    msg = unittest_pb2.TestAllTypes()
+    self.assertRaises(AttributeError, getattr, msg, 'Extensions')
 
   def testGoldenExtensions(self):
     golden_data = test_util.GoldenFileData('golden_message')
@@ -1297,6 +1382,7 @@ class Proto3Test(BaseTestCase):
     """Assigning an unknown enum value is allowed and preserves the value."""
     m = unittest_proto3_arena_pb2.TestAllTypes()
 
+    # Proto3 can assign unknown enums.
     m.optional_nested_enum = 1234567
     self.assertEqual(1234567, m.optional_nested_enum)
     m.repeated_nested_enum.append(22334455)
@@ -1311,18 +1397,10 @@ class Proto3Test(BaseTestCase):
     self.assertEqual(1234567, m2.optional_nested_enum)
     self.assertEqual(7654321, m2.repeated_nested_enum[0])
 
-    # ParseFromString in Proto2 should accept unknown enums too.
-    m3 = unittest_pb2.TestAllTypes()
-    m3.ParseFromString(serialized)
-    m2.Clear()
-    m2.ParseFromString(m3.SerializeToString())
-    self.assertEqual(1234567, m2.optional_nested_enum)
-    self.assertEqual(7654321, m2.repeated_nested_enum[0])
-
   # Map isn't really a proto3-only feature. But there is no proto2 equivalent
   # of google/protobuf/map_unittest.proto right now, so it's not easy to
   # test both with the same test like we do for the other proto2/proto3 tests.
-  # (google/protobuf/map_protobuf_unittest.proto is very different in the set
+  # (google/protobuf/map_proto2_unittest.proto is very different in the set
   # of messages and fields it contains).
   def testScalarMapDefaults(self):
     msg = map_unittest_pb2.TestMap()
@@ -1383,12 +1461,21 @@ class Proto3Test(BaseTestCase):
 
     msg.map_int32_int32[5] = 15
     self.assertEqual(15, msg.map_int32_int32.get(5))
+    self.assertEqual(15, msg.map_int32_int32.get(5))
+    with self.assertRaises(TypeError):
+      msg.map_int32_int32.get('')
 
     self.assertIsNone(msg.map_int32_foreign_message.get(5))
     self.assertEqual(10, msg.map_int32_foreign_message.get(5, 10))
 
     submsg = msg.map_int32_foreign_message[5]
     self.assertIs(submsg, msg.map_int32_foreign_message.get(5))
+    # TODO(jieluo): Fix python and cpp extension diff.
+    if api_implementation.Type() == 'cpp':
+      with self.assertRaises(TypeError):
+        msg.map_int32_foreign_message.get('')
+    else:
+      self.assertEqual(None, msg.map_int32_foreign_message.get(''))
 
   def testScalarMap(self):
     msg = map_unittest_pb2.TestMap()
@@ -1400,8 +1487,13 @@ class Proto3Test(BaseTestCase):
     msg.map_int64_int64[-2**33] = -2**34
     msg.map_uint32_uint32[123] = 456
     msg.map_uint64_uint64[2**33] = 2**34
+    msg.map_int32_float[2] = 1.2
+    msg.map_int32_double[1] = 3.3
     msg.map_string_string['abc'] = '123'
+    msg.map_bool_bool[True] = True
     msg.map_int32_enum[888] = 2
+    # Unknown numeric enum is supported in proto3.
+    msg.map_int32_enum[123] = 456
 
     self.assertEqual([], msg.FindInitializationErrors())
 
@@ -1435,8 +1527,24 @@ class Proto3Test(BaseTestCase):
     self.assertEqual(-2**34, msg2.map_int64_int64[-2**33])
     self.assertEqual(456, msg2.map_uint32_uint32[123])
     self.assertEqual(2**34, msg2.map_uint64_uint64[2**33])
+    self.assertAlmostEqual(1.2, msg.map_int32_float[2])
+    self.assertEqual(3.3, msg.map_int32_double[1])
     self.assertEqual('123', msg2.map_string_string['abc'])
+    self.assertEqual(True, msg2.map_bool_bool[True])
     self.assertEqual(2, msg2.map_int32_enum[888])
+    self.assertEqual(456, msg2.map_int32_enum[123])
+    # TODO(jieluo): Add cpp extension support.
+    if api_implementation.Type() == 'python':
+      self.assertEqual('{-123: -456}',
+                       str(msg2.map_int32_int32))
+
+  def testMapEntryAlwaysSerialized(self):
+    msg = map_unittest_pb2.TestMap()
+    msg.map_int32_int32[0] = 0
+    msg.map_string_string[''] = ''
+    self.assertEqual(msg.ByteSize(), 12)
+    self.assertEqual(b'\n\x04\x08\x00\x10\x00r\x04\n\x00\x12\x00',
+                     msg.SerializeToString())
 
   def testStringUnicodeConversionInMap(self):
     msg = map_unittest_pb2.TestMap()
@@ -1489,6 +1597,11 @@ class Proto3Test(BaseTestCase):
     self.assertIn(123, msg2.map_int32_foreign_message)
     self.assertIn(-456, msg2.map_int32_foreign_message)
     self.assertEqual(2, len(msg2.map_int32_foreign_message))
+    # TODO(jieluo): Fix text format for message map.
+    # TODO(jieluo): Add cpp extension support.
+    if api_implementation.Type() == 'python':
+      self.assertEqual(15,
+                       len(str(msg2.map_int32_foreign_message)))
 
   def testNestedMessageMapItemDelete(self):
     msg = map_unittest_pb2.TestMap()
@@ -1572,6 +1685,12 @@ class Proto3Test(BaseTestCase):
 
     del msg2.map_int32_foreign_message[222]
     self.assertFalse(222 in msg2.map_int32_foreign_message)
+    if api_implementation.Type() == 'cpp':
+      with self.assertRaises(TypeError):
+        del msg2.map_int32_foreign_message['']
+    else:
+      with self.assertRaises(KeyError):
+        del msg2.map_int32_foreign_message['']
 
   def testMergeFromBadType(self):
     msg = map_unittest_pb2.TestMap()
@@ -1706,6 +1825,54 @@ class Proto3Test(BaseTestCase):
     matching_dict = {2: 4, 3: 6, 4: 8}
     self.assertMapIterEquals(msg.map_int32_int32.items(), matching_dict)
 
+  def testPython2Map(self):
+    if sys.version_info < (3,):
+      msg = map_unittest_pb2.TestMap()
+      msg.map_int32_int32[2] = 4
+      msg.map_int32_int32[3] = 6
+      msg.map_int32_int32[4] = 8
+      msg.map_int32_int32[5] = 10
+      map_int32 = msg.map_int32_int32
+      self.assertEqual(4, len(map_int32))
+      msg2 = map_unittest_pb2.TestMap()
+      msg2.ParseFromString(msg.SerializeToString())
+
+      def CheckItems(seq, iterator):
+        self.assertEqual(next(iterator), seq[0])
+        self.assertEqual(list(iterator), seq[1:])
+
+      CheckItems(map_int32.items(), map_int32.iteritems())
+      CheckItems(map_int32.keys(), map_int32.iterkeys())
+      CheckItems(map_int32.values(), map_int32.itervalues())
+
+      self.assertEqual(6, map_int32.get(3))
+      self.assertEqual(None, map_int32.get(999))
+      self.assertEqual(6, map_int32.pop(3))
+      self.assertEqual(0, map_int32.pop(3))
+      self.assertEqual(3, len(map_int32))
+      key, value = map_int32.popitem()
+      self.assertEqual(2 * key, value)
+      self.assertEqual(2, len(map_int32))
+      map_int32.clear()
+      self.assertEqual(0, len(map_int32))
+
+      with self.assertRaises(KeyError):
+        map_int32.popitem()
+
+      self.assertEqual(0, map_int32.setdefault(2))
+      self.assertEqual(1, len(map_int32))
+
+      map_int32.update(msg2.map_int32_int32)
+      self.assertEqual(4, len(map_int32))
+
+      with self.assertRaises(TypeError):
+        map_int32.update(msg2.map_int32_int32,
+                         msg2.map_int32_int32)
+      with self.assertRaises(TypeError):
+        map_int32.update(0)
+      with self.assertRaises(TypeError):
+        map_int32.update(value=12)
+
   def testMapItems(self):
     # Map items used to have strange behaviors when use c extension. Because
     # [] may reorder the map and invalidate any exsting iterators.
@@ -1836,6 +2003,9 @@ class Proto3Test(BaseTestCase):
     del msg.map_int32_int32[4]
     self.assertEqual(0, len(msg.map_int32_int32))
 
+    with self.assertRaises(KeyError):
+      del msg.map_int32_all_types[32]
+
   def testMapsAreMapping(self):
     msg = map_unittest_pb2.TestMap()
     self.assertIsInstance(msg.map_int32_int32, collections.Mapping)
@@ -1844,6 +2014,14 @@ class Proto3Test(BaseTestCase):
     self.assertIsInstance(msg.map_int32_foreign_message,
                           collections.MutableMapping)
 
+  def testMapsCompare(self):
+    msg = map_unittest_pb2.TestMap()
+    msg.map_int32_int32[-123] = -456
+    self.assertEqual(msg.map_int32_int32, msg.map_int32_int32)
+    self.assertEqual(msg.map_int32_foreign_message,
+                     msg.map_int32_foreign_message)
+    self.assertNotEqual(msg.map_int32_int32, 0)
+
   def testMapFindInitializationErrorsSmokeTest(self):
     msg = map_unittest_pb2.TestMap()
     msg.map_string_string['abc'] = '123'

python/google/protobuf/internal/python_message.py

@@ -1010,11 +1010,16 @@ def _AddByteSizeMethod(message_descriptor, cls):
       return self._cached_byte_size
 
     size = 0
-    for field_descriptor, field_value in self.ListFields():
-      size += field_descriptor._sizer(field_value)
-
-    for tag_bytes, value_bytes in self._unknown_fields:
-      size += len(tag_bytes) + len(value_bytes)
+    descriptor = self.DESCRIPTOR
+    if descriptor.GetOptions().map_entry:
+      # Fields of map entry should always be serialized.
+      size = descriptor.fields_by_name['key']._sizer(self.key)
+      size += descriptor.fields_by_name['value']._sizer(self.value)
+    else:
+      for field_descriptor, field_value in self.ListFields():
+        size += field_descriptor._sizer(field_value)
+      for tag_bytes, value_bytes in self._unknown_fields:
+        size += len(tag_bytes) + len(value_bytes)
 
     self._cached_byte_size = size
     self._cached_byte_size_dirty = False
@@ -1053,11 +1058,20 @@ def _AddSerializePartialToStringMethod(message_descriptor, cls):
           api_implementation.IsPythonDefaultSerializationDeterministic())
     else:
       deterministic = bool(deterministic)
-    for field_descriptor, field_value in self.ListFields():
-      field_descriptor._encoder(write_bytes, field_value, deterministic)
-    for tag_bytes, value_bytes in self._unknown_fields:
-      write_bytes(tag_bytes)
-      write_bytes(value_bytes)
+
+    descriptor = self.DESCRIPTOR
+    if descriptor.GetOptions().map_entry:
+      # Fields of map entry should always be serialized.
+      descriptor.fields_by_name['key']._encoder(
+          write_bytes, self.key, deterministic)
+      descriptor.fields_by_name['value']._encoder(
+          write_bytes, self.value, deterministic)
+    else:
+      for field_descriptor, field_value in self.ListFields():
+        field_descriptor._encoder(write_bytes, field_value, deterministic)
+      for tag_bytes, value_bytes in self._unknown_fields:
+        write_bytes(tag_bytes)
+        write_bytes(value_bytes)
   cls._InternalSerialize = InternalSerialize
 
 
@@ -1095,7 +1109,8 @@ def _AddMergeFromStringMethod(message_descriptor, cls):
         new_pos = local_SkipField(buffer, new_pos, end, tag_bytes)
         if new_pos == -1:
           return pos
-        if not is_proto3:
+        if (not is_proto3 or
+            api_implementation.GetPythonProto3PreserveUnknownsDefault()):
           if not unknown_field_list:
             unknown_field_list = self._unknown_fields = []
           unknown_field_list.append(

python/google/protobuf/internal/reflection_test.py

@@ -40,7 +40,6 @@ import gc
 import operator
 import six
 import struct
-import sys
 
 try:
   import unittest2 as unittest  #PY26
@@ -621,6 +620,14 @@ class ReflectionTest(BaseTestCase):
     self.assertRaises(TypeError, setattr, proto, 'optional_int32', 'foo')
     self.assertRaises(TypeError, setattr, proto, 'optional_string', 10)
     self.assertRaises(TypeError, setattr, proto, 'optional_bytes', 10)
+    self.assertRaises(TypeError, setattr, proto, 'optional_bool', 'foo')
+    self.assertRaises(TypeError, setattr, proto, 'optional_float', 'foo')
+    self.assertRaises(TypeError, setattr, proto, 'optional_double', 'foo')
+    # TODO(jieluo): Fix type checking difference for python and c extension
+    if api_implementation.Type() == 'python':
+      self.assertRaises(TypeError, setattr, proto, 'optional_bool', 1.1)
+    else:
+      proto.optional_bool = 1.1
 
   def assertIntegerTypes(self, integer_fn):
     """Verifies setting of scalar integers.
@@ -687,8 +694,10 @@ class ReflectionTest(BaseTestCase):
       self.assertEqual(expected_min, getattr(pb, field_name))
       setattr(pb, field_name, expected_max)
       self.assertEqual(expected_max, getattr(pb, field_name))
-      self.assertRaises(Exception, setattr, pb, field_name, expected_min - 1)
-      self.assertRaises(Exception, setattr, pb, field_name, expected_max + 1)
+      self.assertRaises((ValueError, TypeError), setattr, pb, field_name,
+                        expected_min - 1)
+      self.assertRaises((ValueError, TypeError), setattr, pb, field_name,
+                        expected_max + 1)
 
     TestMinAndMaxIntegers('optional_int32', -(1 << 31), (1 << 31) - 1)
     TestMinAndMaxIntegers('optional_uint32', 0, 0xffffffff)
@@ -697,7 +706,7 @@ class ReflectionTest(BaseTestCase):
     # A bit of white-box testing since -1 is an int and not a long in C++ and
     # so goes down a different path.
     pb = unittest_pb2.TestAllTypes()
-    with self.assertRaises(Exception):
+    with self.assertRaises((ValueError, TypeError)):
       pb.optional_uint64 = integer_fn(-(1 << 63))
 
     pb = unittest_pb2.TestAllTypes()
@@ -721,6 +730,12 @@ class ReflectionTest(BaseTestCase):
     proto.repeated_int32[0] = 23
     self.assertRaises(IndexError, proto.repeated_int32.__setitem__, 500, 23)
     self.assertRaises(TypeError, proto.repeated_int32.__setitem__, 0, 'abc')
+    self.assertRaises(TypeError, proto.repeated_int32.__setitem__, 0, [])
+    self.assertRaises(TypeError, proto.repeated_int32.__setitem__,
+                      'index', 23)
+
+    proto.repeated_string.append('2')
+    self.assertRaises(TypeError, proto.repeated_string.__setitem__, 0, 10)
 
     # Repeated enums tests.
     #proto.repeated_nested_enum.append(0)
@@ -1008,6 +1023,14 @@ class ReflectionTest(BaseTestCase):
     self.assertEqual(4, len(proto.repeated_nested_message))
     self.assertEqual(n1, proto.repeated_nested_message[2])
     self.assertEqual(n2, proto.repeated_nested_message[3])
+    self.assertRaises(TypeError,
+                      proto.repeated_nested_message.extend, n1)
+    self.assertRaises(TypeError,
+                      proto.repeated_nested_message.extend, [0])
+    wrong_message_type = unittest_pb2.TestAllTypes()
+    self.assertRaises(TypeError,
+                      proto.repeated_nested_message.extend,
+                      [wrong_message_type])
 
     # Test clearing.
     proto.ClearField('repeated_nested_message')
@@ -1019,6 +1042,8 @@ class ReflectionTest(BaseTestCase):
     self.assertEqual(1, len(proto.repeated_nested_message))
     self.assertEqual(23, proto.repeated_nested_message[0].bb)
     self.assertRaises(TypeError, proto.repeated_nested_message.add, 23)
+    with self.assertRaises(Exception):
+      proto.repeated_nested_message[0] = 23
 
   def testRepeatedCompositeRemove(self):
     proto = unittest_pb2.TestAllTypes()
@@ -1643,8 +1668,11 @@ class ReflectionTest(BaseTestCase):
     proto.SerializeToString()
     proto.SerializePartialToString()
 
-  def assertNotInitialized(self, proto):
+  def assertNotInitialized(self, proto, error_size=None):
+    errors = []
     self.assertFalse(proto.IsInitialized())
+    self.assertFalse(proto.IsInitialized(errors))
+    self.assertEqual(error_size, len(errors))
     self.assertRaises(message.EncodeError, proto.SerializeToString)
     # "Partial" serialization doesn't care if message is uninitialized.
     proto.SerializePartialToString()
@@ -1658,7 +1686,7 @@ class ReflectionTest(BaseTestCase):
 
     # The case of uninitialized required fields.
     proto = unittest_pb2.TestRequired()
-    self.assertNotInitialized(proto)
+    self.assertNotInitialized(proto, 3)
     proto.a = proto.b = proto.c = 2
     self.assertInitialized(proto)
 
@@ -1666,14 +1694,14 @@ class ReflectionTest(BaseTestCase):
     proto = unittest_pb2.TestRequiredForeign()
     self.assertInitialized(proto)
     proto.optional_message.a = 1
-    self.assertNotInitialized(proto)
+    self.assertNotInitialized(proto, 2)
     proto.optional_message.b = 0
     proto.optional_message.c = 0
     self.assertInitialized(proto)
 
     # Uninitialized repeated submessage.
     message1 = proto.repeated_message.add()
-    self.assertNotInitialized(proto)
+    self.assertNotInitialized(proto, 3)
     message1.a = message1.b = message1.c = 0
     self.assertInitialized(proto)
 
@@ -1682,11 +1710,11 @@ class ReflectionTest(BaseTestCase):
     extension = unittest_pb2.TestRequired.multi
     message1 = proto.Extensions[extension].add()
     message2 = proto.Extensions[extension].add()
-    self.assertNotInitialized(proto)
+    self.assertNotInitialized(proto, 6)
     message1.a = 1
     message1.b = 1
     message1.c = 1
-    self.assertNotInitialized(proto)
+    self.assertNotInitialized(proto, 3)
     message2.a = 2
     message2.b = 2
     message2.c = 2
@@ -1696,7 +1724,7 @@ class ReflectionTest(BaseTestCase):
     proto = unittest_pb2.TestAllExtensions()
     extension = unittest_pb2.TestRequired.single
     proto.Extensions[extension].a = 1
-    self.assertNotInitialized(proto)
+    self.assertNotInitialized(proto, 2)
     proto.Extensions[extension].b = 2
     proto.Extensions[extension].c = 3
     self.assertInitialized(proto)
@@ -2155,6 +2183,8 @@ class ByteSizeTest(BaseTestCase):
     foreign_message_1 = self.proto.repeated_nested_message.add()
     foreign_message_1.bb = 9
     self.assertEqual(2 + 1 + 2 + 1 + 1 + 1, self.Size())
+    repeated_nested_message = copy.deepcopy(
+        self.proto.repeated_nested_message)
 
     # 2 bytes tag plus 1 byte length plus 1 byte bb tag 1 byte int.
     del self.proto.repeated_nested_message[0]
@@ -2175,6 +2205,16 @@ class ByteSizeTest(BaseTestCase):
     del self.proto.repeated_nested_message[0]
     self.assertEqual(0, self.Size())
 
+    self.assertEqual(2, len(repeated_nested_message))
+    del repeated_nested_message[0:1]
+    # TODO(jieluo): Fix cpp extension bug when delete repeated message.
+    if api_implementation.Type() == 'python':
+      self.assertEqual(1, len(repeated_nested_message))
+    del repeated_nested_message[-1]
+    # TODO(jieluo): Fix cpp extension bug when delete repeated message.
+    if api_implementation.Type() == 'python':
+      self.assertEqual(0, len(repeated_nested_message))
+
   def testRepeatedGroups(self):
     # 2-byte START_GROUP plus 2-byte END_GROUP.
     group_0 = self.proto.repeatedgroup.add()
@@ -2191,6 +2231,10 @@ class ByteSizeTest(BaseTestCase):
     proto.Extensions[extension] = 23
     # 1 byte for tag, 1 byte for value.
     self.assertEqual(2, proto.ByteSize())
+    field = unittest_pb2.TestAllTypes.DESCRIPTOR.fields_by_name[
+        'optional_int32']
+    with self.assertRaises(KeyError):
+      proto.Extensions[field] = 23
 
   def testCacheInvalidationForNonrepeatedScalar(self):
     # Test non-extension.

python/google/protobuf/internal/service_reflection_test.py

@@ -82,6 +82,10 @@ class FooUnitTest(unittest.TestCase):
     service_descriptor = unittest_pb2.TestService.GetDescriptor()
     srvc.CallMethod(service_descriptor.methods[1], rpc_controller,
                     unittest_pb2.BarRequest(), MyCallback)
+    self.assertTrue(srvc.GetRequestClass(service_descriptor.methods[1]) is
+                    unittest_pb2.BarRequest)
+    self.assertTrue(srvc.GetResponseClass(service_descriptor.methods[1]) is
+                    unittest_pb2.BarResponse)
     self.assertEqual('Method Bar not implemented.',
                      rpc_controller.failure_message)
     self.assertEqual(None, self.callback_response)

python/google/protobuf/internal/test_util.py

@@ -44,9 +44,9 @@ from google.protobuf import unittest_import_pb2
 from google.protobuf import unittest_pb2
 
 try:
-    long        # Python 2
+  long        # Python 2
 except NameError:
-    long = int  # Python 3
+  long = int  # Python 3
 
 
 # Tests whether the given TestAllTypes message is proto2 or not.
@@ -133,22 +133,37 @@ def SetAllNonLazyFields(message):
   message.repeated_string_piece.append(u'224')
   message.repeated_cord.append(u'225')
 
-  # Add a second one of each field.
-  message.repeated_int32.append(301)
-  message.repeated_int64.append(302)
-  message.repeated_uint32.append(303)
-  message.repeated_uint64.append(304)
-  message.repeated_sint32.append(305)
-  message.repeated_sint64.append(306)
-  message.repeated_fixed32.append(307)
-  message.repeated_fixed64.append(308)
-  message.repeated_sfixed32.append(309)
-  message.repeated_sfixed64.append(310)
-  message.repeated_float.append(311)
-  message.repeated_double.append(312)
-  message.repeated_bool.append(False)
-  message.repeated_string.append(u'315')
-  message.repeated_bytes.append(b'316')
+  # Add a second one of each field and set value by index.
+  message.repeated_int32.append(0)
+  message.repeated_int64.append(0)
+  message.repeated_uint32.append(0)
+  message.repeated_uint64.append(0)
+  message.repeated_sint32.append(0)
+  message.repeated_sint64.append(0)
+  message.repeated_fixed32.append(0)
+  message.repeated_fixed64.append(0)
+  message.repeated_sfixed32.append(0)
+  message.repeated_sfixed64.append(0)
+  message.repeated_float.append(0)
+  message.repeated_double.append(0)
+  message.repeated_bool.append(True)
+  message.repeated_string.append(u'0')
+  message.repeated_bytes.append(b'0')
+  message.repeated_int32[1] = 301
+  message.repeated_int64[1] = 302
+  message.repeated_uint32[1] = 303
+  message.repeated_uint64[1] = 304
+  message.repeated_sint32[1] = 305
+  message.repeated_sint64[1] = 306
+  message.repeated_fixed32[1] = 307
+  message.repeated_fixed64[1] = 308
+  message.repeated_sfixed32[1] = 309
+  message.repeated_sfixed64[1] = 310
+  message.repeated_float[1] = 311
+  message.repeated_double[1] = 312
+  message.repeated_bool[1] = False
+  message.repeated_string[1] = u'315'
+  message.repeated_bytes[1] = b'316'
 
   if IsProto2(message):
     message.repeatedgroup.add().a = 317
@@ -157,7 +172,8 @@ def SetAllNonLazyFields(message):
   message.repeated_import_message.add().d = 320
   message.repeated_lazy_message.add().bb = 327
 
-  message.repeated_nested_enum.append(unittest_pb2.TestAllTypes.BAZ)
+  message.repeated_nested_enum.append(unittest_pb2.TestAllTypes.BAR)
+  message.repeated_nested_enum[1] = unittest_pb2.TestAllTypes.BAZ
   message.repeated_foreign_enum.append(unittest_pb2.FOREIGN_BAZ)
   if IsProto2(message):
     message.repeated_import_enum.append(unittest_import_pb2.IMPORT_BAZ)

python/google/protobuf/internal/text_format_test.py

@@ -35,6 +35,7 @@
 __author__ = 'kenton@google.com (Kenton Varda)'
 
 
+import math
 import re
 import six
 import string
@@ -53,8 +54,8 @@ from google.protobuf import unittest_pb2
 from google.protobuf import unittest_proto3_arena_pb2
 from google.protobuf.internal import api_implementation
 from google.protobuf.internal import any_test_pb2 as test_extend_any
-from google.protobuf.internal import test_util
 from google.protobuf.internal import message_set_extensions_pb2
+from google.protobuf.internal import test_util
 from google.protobuf import descriptor_pool
 from google.protobuf import text_format
 
@@ -371,7 +372,10 @@ class TextFormatTest(TextFormatBase):
   def testParseInvalidUtf8(self, message_module):
     message = message_module.TestAllTypes()
     text = 'repeated_string: "\\xc3\\xc3"'
-    self.assertRaises(text_format.ParseError, text_format.Parse, text, message)
+    with self.assertRaises(text_format.ParseError) as e:
+      text_format.Parse(text, message)
+    self.assertEqual(e.exception.GetLine(), 1)
+    self.assertEqual(e.exception.GetColumn(), 28)
 
   def testParseSingleWord(self, message_module):
     message = message_module.TestAllTypes()
@@ -784,13 +788,14 @@ class Proto2Tests(TextFormatBase):
             '    bin: "\xe0"'
             '    [nested_unknown_ext]: {\n'
             '      i: 23\n'
+            '      x: x\n'
             '      test: "test_string"\n'
             '      floaty_float: -0.315\n'
             '      num: -inf\n'
             '      multiline_str: "abc"\n'
             '          "def"\n'
             '          "xyz."\n'
-            '      [nested_unknown_ext]: <\n'
+            '      [nested_unknown_ext.ext]: <\n'
             '        i: 23\n'
             '        i: 24\n'
             '        pointfloat: .3\n'
@@ -896,6 +901,14 @@ class Proto2Tests(TextFormatBase):
     self.assertEqual(23, message.message_set.Extensions[ext1].i)
     self.assertEqual('foo', message.message_set.Extensions[ext2].str)
 
+  def testParseBadIdentifier(self):
+    message = unittest_pb2.TestAllTypes()
+    text = ('optional_nested_message { "bb": 1 }')
+    with self.assertRaises(text_format.ParseError) as e:
+      text_format.Parse(text, message)
+    self.assertEqual(str(e.exception),
+                     '1:27 : Expected identifier or number, got "bb".')
+
   def testParseBadExtension(self):
     message = unittest_pb2.TestAllExtensions()
     text = '[unknown_extension]: 8\n'
@@ -1095,6 +1108,19 @@ class Proto3Tests(unittest.TestCase):
         ' < data: "string" > '
         '>')
 
+  def testUnknownEnums(self):
+    message = unittest_proto3_arena_pb2.TestAllTypes()
+    message2 = unittest_proto3_arena_pb2.TestAllTypes()
+    message.optional_nested_enum = 999
+    text_string = text_format.MessageToString(message)
+    # TODO(jieluo): proto3 should support numeric unknown enum.
+    with self.assertRaises(text_format.ParseError) as e:
+      text_format.Parse(text_string, message2)
+      self.assertEqual(999, message2.optional_nested_enum)
+    self.assertEqual(str(e.exception),
+                     '1:23 : Enum type "proto3_arena_unittest.TestAllTypes.'
+                     'NestedEnum" has no value with number 999.')
+
   def testMergeExpandedAny(self):
     message = any_test_pb2.TestAny()
     text = ('any_value {\n'
@@ -1180,6 +1206,15 @@ class Proto3Tests(unittest.TestCase):
     message.any_value.Unpack(packed_message)
     self.assertEqual('string', packed_message.data)
 
+  def testMergeMissingAnyEndToken(self):
+    message = any_test_pb2.TestAny()
+    text = ('any_value {\n'
+            '  [type.googleapis.com/protobuf_unittest.OneString] {\n'
+            '    data: "string"\n')
+    with self.assertRaises(text_format.ParseError) as e:
+      text_format.Merge(text, message)
+    self.assertEqual(str(e.exception), '3:11 : Expected "}".')
+
 
 class TokenizerTest(unittest.TestCase):
 
@@ -1191,7 +1226,7 @@ class TokenizerTest(unittest.TestCase):
             'ID9: 22 ID10: -111111111111111111 ID11: -22\n'
             'ID12: 2222222222222222222 ID13: 1.23456f ID14: 1.2e+2f '
             'false_bool:  0 true_BOOL:t \n true_bool1:  1 false_BOOL1:f '
-            'False_bool: False True_bool: True')
+            'False_bool: False True_bool: True X:iNf Y:-inF Z:nAN')
     tokenizer = text_format.Tokenizer(text.splitlines())
     methods = [(tokenizer.ConsumeIdentifier, 'identifier1'), ':',
                (tokenizer.ConsumeString, 'string1'),
@@ -1239,7 +1274,13 @@ class TokenizerTest(unittest.TestCase):
                (tokenizer.ConsumeIdentifier, 'False_bool'), ':',
                (tokenizer.ConsumeBool, False),
                (tokenizer.ConsumeIdentifier, 'True_bool'), ':',
-               (tokenizer.ConsumeBool, True)]
+               (tokenizer.ConsumeBool, True),
+               (tokenizer.ConsumeIdentifier, 'X'), ':',
+               (tokenizer.ConsumeFloat, float('inf')),
+               (tokenizer.ConsumeIdentifier, 'Y'), ':',
+               (tokenizer.ConsumeFloat, float('-inf')),
+               (tokenizer.ConsumeIdentifier, 'Z'), ':',
+               (tokenizer.ConsumeFloat, float('nan'))]
 
     i = 0
     while not tokenizer.AtEnd():
@@ -1248,6 +1289,8 @@ class TokenizerTest(unittest.TestCase):
         token = tokenizer.token
         self.assertEqual(token, m)
         tokenizer.NextToken()
+      elif isinstance(m[1], float) and math.isnan(m[1]):
+        self.assertTrue(math.isnan(m[0]()))
       else:
         self.assertEqual(m[1], m[0]())
       i += 1
@@ -1266,10 +1309,15 @@ class TokenizerTest(unittest.TestCase):
     self.assertEqual(int64_max + 1, tokenizer.ConsumeInteger())
     self.assertTrue(tokenizer.AtEnd())
 
-    text = '-0 0'
+    text = '-0 0 0 1.2'
     tokenizer = text_format.Tokenizer(text.splitlines())
     self.assertEqual(0, tokenizer.ConsumeInteger())
     self.assertEqual(0, tokenizer.ConsumeInteger())
+    self.assertEqual(True, tokenizer.TryConsumeInteger())
+    self.assertEqual(False, tokenizer.TryConsumeInteger())
+    with self.assertRaises(text_format.ParseError):
+      tokenizer.ConsumeInteger()
+    self.assertEqual(1.2, tokenizer.ConsumeFloat())
     self.assertTrue(tokenizer.AtEnd())
 
   def testConsumeIntegers(self):

python/google/protobuf/internal/unknown_fields_test.py

@@ -54,10 +54,13 @@ from google.protobuf.internal import type_checkers
 BaseTestCase = testing_refleaks.BaseTestCase
 
 
-def SkipIfCppImplementation(func):
+# CheckUnknownField() cannot be used by the C++ implementation because
+# some protect members are called. It is not a behavior difference
+# for python and C++ implementation.
+def SkipCheckUnknownFieldIfCppImplementation(func):
   return unittest.skipIf(
       api_implementation.Type() == 'cpp' and api_implementation.Version() == 2,
-      'C++ implementation does not expose unknown fields to Python')(func)
+      'Addtional test for pure python involved protect members')(func)
 
 
 class UnknownFieldsTest(BaseTestCase):
@@ -77,11 +80,24 @@ class UnknownFieldsTest(BaseTestCase):
     # stdout.
     self.assertTrue(data == self.all_fields_data)
 
-  def testSerializeProto3(self):
-    # Verify that proto3 doesn't preserve unknown fields.
+  def expectSerializeProto3(self, preserve):
     message = unittest_proto3_arena_pb2.TestEmptyMessage()
     message.ParseFromString(self.all_fields_data)
-    self.assertEqual(0, len(message.SerializeToString()))
+    if preserve:
+      self.assertEqual(self.all_fields_data, message.SerializeToString())
+    else:
+      self.assertEqual(0, len(message.SerializeToString()))
+
+  def testSerializeProto3(self):
+    # Verify that proto3 unknown fields behavior.
+    default_preserve = (api_implementation
+                        .GetPythonProto3PreserveUnknownsDefault())
+    self.assertEqual(False, default_preserve)
+    self.expectSerializeProto3(default_preserve)
+    api_implementation.SetPythonProto3PreserveUnknownsDefault(
+        not default_preserve)
+    self.expectSerializeProto3(not default_preserve)
+    api_implementation.SetPythonProto3PreserveUnknownsDefault(default_preserve)
 
   def testByteSize(self):
     self.assertEqual(self.all_fields.ByteSize(), self.empty_message.ByteSize())
@@ -154,12 +170,13 @@ class UnknownFieldsAccessorsTest(BaseTestCase):
     self.empty_message = unittest_pb2.TestEmptyMessage()
     self.empty_message.ParseFromString(self.all_fields_data)
 
-  # GetUnknownField() checks a detail of the Python implementation, which stores
-  # unknown fields as serialized strings. It cannot be used by the C++
-  # implementation: it's enough to check that the message is correctly
-  # serialized.
+  # CheckUnknownField() is an additional Pure Python check which checks
+  # a detail of unknown fields. It cannot be used by the C++
+  # implementation because some protect members are called.
+  # The test is added for historical reasons. It is not necessary as
+  # serialized string is checked.
 
-  def GetUnknownField(self, name):
+  def CheckUnknownField(self, name, expected_value):
     field_descriptor = self.descriptor.fields_by_name[name]
     wire_type = type_checkers.FIELD_TYPE_TO_WIRE_TYPE[field_descriptor.type]
     field_tag = encoder.TagBytes(field_descriptor.number, wire_type)
@@ -168,42 +185,35 @@ class UnknownFieldsAccessorsTest(BaseTestCase):
       if tag_bytes == field_tag:
         decoder = unittest_pb2.TestAllTypes._decoders_by_tag[tag_bytes][0]
         decoder(value, 0, len(value), self.all_fields, result_dict)
-    return result_dict[field_descriptor]
-
-  @SkipIfCppImplementation
-  def testEnum(self):
-    value = self.GetUnknownField('optional_nested_enum')
-    self.assertEqual(self.all_fields.optional_nested_enum, value)
-
-  @SkipIfCppImplementation
-  def testRepeatedEnum(self):
-    value = self.GetUnknownField('repeated_nested_enum')
-    self.assertEqual(self.all_fields.repeated_nested_enum, value)
-
-  @SkipIfCppImplementation
-  def testVarint(self):
-    value = self.GetUnknownField('optional_int32')
-    self.assertEqual(self.all_fields.optional_int32, value)
-
-  @SkipIfCppImplementation
-  def testFixed32(self):
-    value = self.GetUnknownField('optional_fixed32')
-    self.assertEqual(self.all_fields.optional_fixed32, value)
-
-  @SkipIfCppImplementation
-  def testFixed64(self):
-    value = self.GetUnknownField('optional_fixed64')
-    self.assertEqual(self.all_fields.optional_fixed64, value)
-
-  @SkipIfCppImplementation
-  def testLengthDelimited(self):
-    value = self.GetUnknownField('optional_string')
-    self.assertEqual(self.all_fields.optional_string, value)
-
-  @SkipIfCppImplementation
-  def testGroup(self):
-    value = self.GetUnknownField('optionalgroup')
-    self.assertEqual(self.all_fields.optionalgroup, value)
+    self.assertEqual(expected_value, result_dict[field_descriptor])
+
+  @SkipCheckUnknownFieldIfCppImplementation
+  def testCheckUnknownFieldValue(self):
+    # Test enum.
+    self.CheckUnknownField('optional_nested_enum',
+                           self.all_fields.optional_nested_enum)
+    # Test repeated enum.
+    self.CheckUnknownField('repeated_nested_enum',
+                           self.all_fields.repeated_nested_enum)
+
+    # Test varint.
+    self.CheckUnknownField('optional_int32',
+                           self.all_fields.optional_int32)
+    # Test fixed32.
+    self.CheckUnknownField('optional_fixed32',
+                           self.all_fields.optional_fixed32)
+
+    # Test fixed64.
+    self.CheckUnknownField('optional_fixed64',
+                           self.all_fields.optional_fixed64)
+
+    # Test lengthd elimited.
+    self.CheckUnknownField('optional_string',
+                           self.all_fields.optional_string)
+
+    # Test group.
+    self.CheckUnknownField('optionalgroup',
+                           self.all_fields.optionalgroup)
 
   def testCopyFrom(self):
     message = unittest_pb2.TestEmptyMessage()
@@ -263,12 +273,13 @@ class UnknownEnumValuesTest(BaseTestCase):
     self.missing_message = missing_enum_values_pb2.TestMissingEnumValues()
     self.missing_message.ParseFromString(self.message_data)
 
-  # GetUnknownField() checks a detail of the Python implementation, which stores
-  # unknown fields as serialized strings. It cannot be used by the C++
-  # implementation: it's enough to check that the message is correctly
-  # serialized.
+  # CheckUnknownField() is an additional Pure Python check which checks
+  # a detail of unknown fields. It cannot be used by the C++
+  # implementation because some protect members are called.
+  # The test is added for historical reasons. It is not necessary as
+  # serialized string is checked.
 
-  def GetUnknownField(self, name):
+  def CheckUnknownField(self, name, expected_value):
     field_descriptor = self.descriptor.fields_by_name[name]
     wire_type = type_checkers.FIELD_TYPE_TO_WIRE_TYPE[field_descriptor.type]
     field_tag = encoder.TagBytes(field_descriptor.number, wire_type)
@@ -278,7 +289,7 @@ class UnknownEnumValuesTest(BaseTestCase):
         decoder = missing_enum_values_pb2.TestEnumValues._decoders_by_tag[
             tag_bytes][0]
         decoder(value, 0, len(value), self.message, result_dict)
-    return result_dict[field_descriptor]
+    self.assertEqual(expected_value, result_dict[field_descriptor])
 
   def testUnknownParseMismatchEnumValue(self):
     just_string = missing_enum_values_pb2.JustString()
@@ -294,38 +305,27 @@ class UnknownEnumValuesTest(BaseTestCase):
     self.assertEqual(missing.optional_nested_enum, 0)
 
   def testUnknownEnumValue(self):
-    if api_implementation.Type() == 'cpp':
-      # The CPP implementation of protos (wrongly) allows unknown enum values
-      # for proto2.
-      self.assertTrue(self.missing_message.HasField('optional_nested_enum'))
-      self.assertEqual(self.message.optional_nested_enum,
-                       self.missing_message.optional_nested_enum)
-    else:
-      # On the other hand, the Python implementation considers unknown values
-      # as unknown fields. This is the correct behavior.
-      self.assertFalse(self.missing_message.HasField('optional_nested_enum'))
-      value = self.GetUnknownField('optional_nested_enum')
-      self.assertEqual(self.message.optional_nested_enum, value)
-    self.missing_message.ClearField('optional_nested_enum')
     self.assertFalse(self.missing_message.HasField('optional_nested_enum'))
+    self.assertEqual(self.missing_message.optional_nested_enum, 2)
+    # Clear does not do anything.
+    serialized = self.missing_message.SerializeToString()
+    self.missing_message.ClearField('optional_nested_enum')
+    self.assertEqual(self.missing_message.SerializeToString(), serialized)
 
   def testUnknownRepeatedEnumValue(self):
-    if api_implementation.Type() == 'cpp':
-      # For repeated enums, both implementations agree.
-      self.assertEqual([], self.missing_message.repeated_nested_enum)
-    else:
-      self.assertEqual([], self.missing_message.repeated_nested_enum)
-      value = self.GetUnknownField('repeated_nested_enum')
-      self.assertEqual(self.message.repeated_nested_enum, value)
+    self.assertEqual([], self.missing_message.repeated_nested_enum)
 
   def testUnknownPackedEnumValue(self):
-    if api_implementation.Type() == 'cpp':
-      # For repeated enums, both implementations agree.
-      self.assertEqual([], self.missing_message.packed_nested_enum)
-    else:
-      self.assertEqual([], self.missing_message.packed_nested_enum)
-      value = self.GetUnknownField('packed_nested_enum')
-      self.assertEqual(self.message.packed_nested_enum, value)
+    self.assertEqual([], self.missing_message.packed_nested_enum)
+
+  @SkipCheckUnknownFieldIfCppImplementation
+  def testCheckUnknownFieldValueForEnum(self):
+    self.CheckUnknownField('optional_nested_enum',
+                           self.message.optional_nested_enum)
+    self.CheckUnknownField('repeated_nested_enum',
+                           self.message.repeated_nested_enum)
+    self.CheckUnknownField('packed_nested_enum',
+                           self.message.packed_nested_enum)
 
   def testRoundTrip(self):
     new_message = missing_enum_values_pb2.TestEnumValues()

python/google/protobuf/internal/well_known_types.py

@@ -473,7 +473,7 @@ def _IsValidPath(message_descriptor, path):
   parts = path.split('.')
   last = parts.pop()
   for name in parts:
-    field = message_descriptor.fields_by_name[name]
+    field = message_descriptor.fields_by_name.get(name)
     if (field is None or
         field.label == FieldDescriptor.LABEL_REPEATED or
         field.type != FieldDescriptor.TYPE_MESSAGE):
@@ -698,6 +698,12 @@ def _SetStructValue(struct_value, value):
     struct_value.string_value = value
   elif isinstance(value, _INT_OR_FLOAT):
     struct_value.number_value = value
+  elif isinstance(value, dict):
+    struct_value.struct_value.Clear()
+    struct_value.struct_value.update(value)
+  elif isinstance(value, list):
+    struct_value.list_value.Clear()
+    struct_value.list_value.extend(value)
   else:
     raise ValueError('Unexpected type')
 
@@ -733,13 +739,21 @@ class Struct(object):
 
   def get_or_create_list(self, key):
     """Returns a list for this key, creating if it didn't exist already."""
+    if not self.fields[key].HasField('list_value'):
+      # Clear will mark list_value modified which will indeed create a list.
+      self.fields[key].list_value.Clear()
     return self.fields[key].list_value
 
   def get_or_create_struct(self, key):
     """Returns a struct for this key, creating if it didn't exist already."""
+    if not self.fields[key].HasField('struct_value'):
+      # Clear will mark struct_value modified which will indeed create a struct.
+      self.fields[key].struct_value.Clear()
     return self.fields[key].struct_value
 
-  # TODO(haberman): allow constructing/merging from dict.
+  def update(self, dictionary):  # pylint: disable=invalid-name
+    for key, value in dictionary.items():
+      _SetStructValue(self.fields[key], value)
 
 
 class ListValue(object):
@@ -768,11 +782,17 @@ class ListValue(object):
 
   def add_struct(self):
     """Appends and returns a struct value as the next value in the list."""
-    return self.values.add().struct_value
+    struct_value = self.values.add().struct_value
+    # Clear will mark struct_value modified which will indeed create a struct.
+    struct_value.Clear()
+    return struct_value
 
   def add_list(self):
     """Appends and returns a list value as the next value in the list."""
-    return self.values.add().list_value
+    list_value = self.values.add().list_value
+    # Clear will mark list_value modified which will indeed create a list.
+    list_value.Clear()
+    return list_value
 
 
 WKTBASES = {

python/google/protobuf/internal/well_known_types_test.py

@@ -105,6 +105,10 @@ class TimeUtilTest(TimeUtilTestBase):
     self.assertEqual(8 * 3600, message.seconds)
     self.assertEqual(0, message.nanos)
 
+    # It is not easy to check with current time. For test coverage only.
+    message.GetCurrentTime()
+    self.assertNotEqual(8 * 3600, message.seconds)
+
   def testDurationSerializeAndParse(self):
     message = duration_pb2.Duration()
     # Generated output should contain 3, 6, or 9 fractional digits.
@@ -267,6 +271,17 @@ class TimeUtilTest(TimeUtilTestBase):
 
   def testInvalidTimestamp(self):
     message = timestamp_pb2.Timestamp()
+    self.assertRaisesRegexp(
+        well_known_types.ParseError,
+        'Failed to parse timestamp: missing valid timezone offset.',
+        message.FromJsonString,
+        '')
+    self.assertRaisesRegexp(
+        well_known_types.ParseError,
+        'Failed to parse timestamp: invalid trailing data '
+        '1970-01-01T00:00:01Ztrail.',
+        message.FromJsonString,
+        '1970-01-01T00:00:01Ztrail')
     self.assertRaisesRegexp(
         ValueError,
         'time data \'10000-01-01T00:00:00\' does not match'
@@ -322,6 +337,13 @@ class TimeUtilTest(TimeUtilTestBase):
         r'Duration is not valid\: Seconds -315576000001 must be in range'
         r' \[-315576000000\, 315576000000\].',
         message.ToJsonString)
+    message.seconds = 0
+    message.nanos = 999999999 + 1
+    self.assertRaisesRegexp(
+        well_known_types.Error,
+        r'Duration is not valid\: Nanos 1000000000 must be in range'
+        r' \[-999999999\, 999999999\].',
+        message.ToJsonString)
 
 
 class FieldMaskTest(unittest.TestCase):
@@ -363,10 +385,37 @@ class FieldMaskTest(unittest.TestCase):
     self.assertTrue(mask.IsValidForDescriptor(msg_descriptor))
     for field in msg_descriptor.fields:
       self.assertTrue(field.name in mask.paths)
+
+  def testIsValidForDescriptor(self):
+    msg_descriptor = unittest_pb2.TestAllTypes.DESCRIPTOR
+    # Empty mask
+    mask = field_mask_pb2.FieldMask()
+    self.assertTrue(mask.IsValidForDescriptor(msg_descriptor))
+    # All fields from descriptor
+    mask.AllFieldsFromDescriptor(msg_descriptor)
+    self.assertTrue(mask.IsValidForDescriptor(msg_descriptor))
+    # Child under optional message
     mask.paths.append('optional_nested_message.bb')
     self.assertTrue(mask.IsValidForDescriptor(msg_descriptor))
+    # Repeated field is only allowed in the last position of path
     mask.paths.append('repeated_nested_message.bb')
     self.assertFalse(mask.IsValidForDescriptor(msg_descriptor))
+    # Invalid top level field
+    mask = field_mask_pb2.FieldMask()
+    mask.paths.append('xxx')
+    self.assertFalse(mask.IsValidForDescriptor(msg_descriptor))
+    # Invalid field in root
+    mask = field_mask_pb2.FieldMask()
+    mask.paths.append('xxx.zzz')
+    self.assertFalse(mask.IsValidForDescriptor(msg_descriptor))
+    # Invalid field in internal node
+    mask = field_mask_pb2.FieldMask()
+    mask.paths.append('optional_nested_message.xxx.zzz')
+    self.assertFalse(mask.IsValidForDescriptor(msg_descriptor))
+    # Invalid field in leaf
+    mask = field_mask_pb2.FieldMask()
+    mask.paths.append('optional_nested_message.xxx')
+    self.assertFalse(mask.IsValidForDescriptor(msg_descriptor))
 
   def testCanonicalFrom(self):
     mask = field_mask_pb2.FieldMask()
@@ -422,6 +471,9 @@ class FieldMaskTest(unittest.TestCase):
     mask2.FromJsonString('foo.bar,bar')
     out_mask.Union(mask1, mask2)
     self.assertEqual('bar,foo.bar,quz', out_mask.ToJsonString())
+    src = unittest_pb2.TestAllTypes()
+    with self.assertRaises(ValueError):
+      out_mask.Union(src, mask2)
 
   def testIntersect(self):
     mask1 = field_mask_pb2.FieldMask()
@@ -546,6 +598,19 @@ class FieldMaskTest(unittest.TestCase):
     self.assertEqual(1, len(nested_dst.payload.repeated_int32))
     self.assertEqual(1234, nested_dst.payload.repeated_int32[0])
 
+  def testMergeErrors(self):
+    src = unittest_pb2.TestAllTypes()
+    dst = unittest_pb2.TestAllTypes()
+    mask = field_mask_pb2.FieldMask()
+    test_util.SetAllFields(src)
+    mask.FromJsonString('optionalInt32.field')
+    with self.assertRaises(ValueError) as e:
+      mask.MergeMessage(src, dst)
+    self.assertEqual('Error: Field optional_int32 in message '
+                     'protobuf_unittest.TestAllTypes is not a singular '
+                     'message field and cannot have sub-fields.',
+                     str(e.exception))
+
   def testSnakeCaseToCamelCase(self):
     self.assertEqual('fooBar',
                      well_known_types._SnakeCaseToCamelCase('foo_bar'))
@@ -611,6 +676,8 @@ class StructTest(unittest.TestCase):
     struct_list = struct.get_or_create_list('key5')
     struct_list.extend([6, 'seven', True, False, None])
     struct_list.add_struct()['subkey2'] = 9
+    struct['key6'] = {'subkey': {}}
+    struct['key7'] = [2, False]
 
     self.assertTrue(isinstance(struct, well_known_types.Struct))
     self.assertEqual(5, struct['key1'])
@@ -621,9 +688,10 @@ class StructTest(unittest.TestCase):
     inner_struct['subkey2'] = 9
     self.assertEqual([6, 'seven', True, False, None, inner_struct],
                      list(struct['key5'].items()))
+    self.assertEqual({}, dict(struct['key6']['subkey'].fields))
+    self.assertEqual([2, False], list(struct['key7'].items()))
 
     serialized = struct.SerializeToString()
-
     struct2 = struct_pb2.Struct()
     struct2.ParseFromString(serialized)
 
@@ -651,6 +719,17 @@ class StructTest(unittest.TestCase):
     struct_list.add_list().extend([1, 'two', True, False, None])
     self.assertEqual([1, 'two', True, False, None],
                      list(struct_list[6].items()))
+    struct_list.extend([{'nested_struct': 30}, ['nested_list', 99], {}, []])
+    self.assertEqual(11, len(struct_list.values))
+    self.assertEqual(30, struct_list[7]['nested_struct'])
+    self.assertEqual('nested_list', struct_list[8][0])
+    self.assertEqual(99, struct_list[8][1])
+    self.assertEqual({}, dict(struct_list[9].fields))
+    self.assertEqual([], list(struct_list[10].items()))
+    struct_list[0] = {'replace': 'set'}
+    struct_list[1] = ['replace', 'set']
+    self.assertEqual('set', struct_list[0]['replace'])
+    self.assertEqual(['replace', 'set'], list(struct_list[1].items()))
 
     text_serialized = str(struct)
     struct3 = struct_pb2.Struct()
@@ -660,6 +739,67 @@ class StructTest(unittest.TestCase):
     struct.get_or_create_struct('key3')['replace'] = 12
     self.assertEqual(12, struct['key3']['replace'])
 
+    # Tests empty list.
+    struct.get_or_create_list('empty_list')
+    empty_list = struct['empty_list']
+    self.assertEqual([], list(empty_list.items()))
+    list2 = struct_pb2.ListValue()
+    list2.add_list()
+    empty_list = list2[0]
+    self.assertEqual([], list(empty_list.items()))
+
+    # Tests empty struct.
+    struct.get_or_create_struct('empty_struct')
+    empty_struct = struct['empty_struct']
+    self.assertEqual({}, dict(empty_struct.fields))
+    list2.add_struct()
+    empty_struct = list2[1]
+    self.assertEqual({}, dict(empty_struct.fields))
+
+  def testMergeFrom(self):
+    struct = struct_pb2.Struct()
+    struct_class = struct.__class__
+
+    dictionary = {
+        'key1': 5,
+        'key2': 'abc',
+        'key3': True,
+        'key4': {'subkey': 11.0},
+        'key5': [6, 'seven', True, False, None, {'subkey2': 9}],
+        'key6': [['nested_list', True]],
+        'empty_struct': {},
+        'empty_list': []
+    }
+    struct.update(dictionary)
+    self.assertEqual(5, struct['key1'])
+    self.assertEqual('abc', struct['key2'])
+    self.assertIs(True, struct['key3'])
+    self.assertEqual(11, struct['key4']['subkey'])
+    inner_struct = struct_class()
+    inner_struct['subkey2'] = 9
+    self.assertEqual([6, 'seven', True, False, None, inner_struct],
+                     list(struct['key5'].items()))
+    self.assertEqual(2, len(struct['key6'][0].values))
+    self.assertEqual('nested_list', struct['key6'][0][0])
+    self.assertEqual(True, struct['key6'][0][1])
+    empty_list = struct['empty_list']
+    self.assertEqual([], list(empty_list.items()))
+    empty_struct = struct['empty_struct']
+    self.assertEqual({}, dict(empty_struct.fields))
+
+    # According to documentation: "When parsing from the wire or when merging,
+    # if there are duplicate map keys the last key seen is used".
+    duplicate = {
+        'key4': {'replace': 20},
+        'key5': [[False, 5]]
+    }
+    struct.update(duplicate)
+    self.assertEqual(1, len(struct['key4'].fields))
+    self.assertEqual(20, struct['key4']['replace'])
+    self.assertEqual(1, len(struct['key5'].values))
+    self.assertEqual(False, struct['key5'][0][0])
+    self.assertEqual(5, struct['key5'][0][1])
+
 
 class AnyTest(unittest.TestCase):
 

python/google/protobuf/pyext/map_container.cc

@@ -720,14 +720,17 @@ int MapReflectionFriend::MessageMapSetItem(PyObject* _self, PyObject* key,
                                        map_key, &value);
     ScopedPyObjectPtr key(PyLong_FromVoidPtr(value.MutableMessageValue()));
 
-    // PyDict_DelItem will have key error if the key is not in the map. We do
-    // not want to call PyErr_Clear() which may clear other errors. Thus
-    // PyDict_Contains() check is called before delete.
-    int contains = PyDict_Contains(self->message_dict, key.get());
-    if (contains < 0) {
-      return -1;
-    }
-    if (contains) {
+    PyObject* cmsg_value = PyDict_GetItem(self->message_dict, key.get());
+    if (cmsg_value) {
+      // Need to keep CMessage stay alive if it is still referenced after
+      // deletion. Makes a new message and swaps values into CMessage
+      // instead of just removing.
+      CMessage* cmsg =  reinterpret_cast<CMessage*>(cmsg_value);
+      Message* msg = cmsg->message;
+      cmsg->owner.reset(msg->New());
+      cmsg->message = cmsg->owner.get();
+      cmsg->parent = NULL;
+      msg->GetReflection()->Swap(msg, cmsg->message);
       if (PyDict_DelItem(self->message_dict, key.get()) < 0) {
         return -1;
       }

python/google/protobuf/pyext/message.cc

@@ -605,19 +605,21 @@ void OutOfRangeError(PyObject* arg) {
 
 template<class RangeType, class ValueType>
 bool VerifyIntegerCastAndRange(PyObject* arg, ValueType value) {
-  if GOOGLE_PREDICT_FALSE(value == -1 && PyErr_Occurred()) {
-    if (PyErr_ExceptionMatches(PyExc_OverflowError)) {
-      // Replace it with the same ValueError as pure python protos instead of
-      // the default one.
-      PyErr_Clear();
+  if
+    GOOGLE_PREDICT_FALSE(value == -1 && PyErr_Occurred()) {
+      if (PyErr_ExceptionMatches(PyExc_OverflowError)) {
+        // Replace it with the same ValueError as pure python protos instead of
+        // the default one.
+        PyErr_Clear();
+        OutOfRangeError(arg);
+      }  // Otherwise propagate existing error.
+      return false;
+    }
+  if
+    GOOGLE_PREDICT_FALSE(!IsValidNumericCast<RangeType>(value)) {
       OutOfRangeError(arg);
-    }  // Otherwise propagate existing error.
-    return false;
-  }
-  if GOOGLE_PREDICT_FALSE(!IsValidNumericCast<RangeType>(value)) {
-    OutOfRangeError(arg);
-    return false;
-  }
+      return false;
+    }
   return true;
 }
 
@@ -626,25 +628,29 @@ bool CheckAndGetInteger(PyObject* arg, T* value) {
   // The fast path.
 #if PY_MAJOR_VERSION < 3
   // For the typical case, offer a fast path.
-  if GOOGLE_PREDICT_TRUE(PyInt_Check(arg)) {
-    long int_result =  PyInt_AsLong(arg);
-    if GOOGLE_PREDICT_TRUE(IsValidNumericCast<T>(int_result)) {
-      *value = static_cast<T>(int_result);
-      return true;
-    } else {
-      OutOfRangeError(arg);
-      return false;
+  if
+    GOOGLE_PREDICT_TRUE(PyInt_Check(arg)) {
+      long int_result = PyInt_AsLong(arg);
+      if
+        GOOGLE_PREDICT_TRUE(IsValidNumericCast<T>(int_result)) {
+          *value = static_cast<T>(int_result);
+          return true;
+        }
+      else {
+        OutOfRangeError(arg);
+        return false;
+      }
     }
-  }
 #endif
   // This effectively defines an integer as "an object that can be cast as
   // an integer and can be used as an ordinal number".
   // This definition includes everything that implements numbers.Integral
   // and shouldn't cast the net too wide.
-  if GOOGLE_PREDICT_FALSE(!PyIndex_Check(arg)) {
-    FormatTypeError(arg, "int, long");
-    return false;
-  }
+  if
+    GOOGLE_PREDICT_FALSE(!PyIndex_Check(arg)) {
+      FormatTypeError(arg, "int, long");
+      return false;
+    }
 
   // Now we have an integral number so we can safely use PyLong_ functions.
   // We need to treat the signed and unsigned cases differently in case arg is
@@ -658,10 +664,11 @@ bool CheckAndGetInteger(PyObject* arg, T* value) {
       // Unlike PyLong_AsLongLong, PyLong_AsUnsignedLongLong is very
       // picky about the exact type.
       PyObject* casted = PyNumber_Long(arg);
-      if GOOGLE_PREDICT_FALSE(casted == NULL) {
-        // Propagate existing error.
-        return false;
-      }
+      if
+        GOOGLE_PREDICT_FALSE(casted == NULL) {
+          // Propagate existing error.
+          return false;
+        }
       ulong_result = PyLong_AsUnsignedLongLong(casted);
       Py_DECREF(casted);
     }
@@ -683,10 +690,11 @@ bool CheckAndGetInteger(PyObject* arg, T* value) {
       // Valid subclasses of numbers.Integral should have a __long__() method
       // so fall back to that.
       PyObject* casted = PyNumber_Long(arg);
-      if GOOGLE_PREDICT_FALSE(casted == NULL) {
-        // Propagate existing error.
-        return false;
-      }
+      if
+        GOOGLE_PREDICT_FALSE(casted == NULL) {
+          // Propagate existing error.
+          return false;
+        }
       long_result = PyLong_AsLongLong(casted);
       Py_DECREF(casted);
     }
@@ -709,10 +717,11 @@ template bool CheckAndGetInteger<uint64>(PyObject*, uint64*);
 
 bool CheckAndGetDouble(PyObject* arg, double* value) {
   *value = PyFloat_AsDouble(arg);
-  if GOOGLE_PREDICT_FALSE(*value == -1 && PyErr_Occurred()) {
-    FormatTypeError(arg, "int, long, float");
-    return false;
-  }
+  if
+    GOOGLE_PREDICT_FALSE(*value == -1 && PyErr_Occurred()) {
+      FormatTypeError(arg, "int, long, float");
+      return false;
+    }
   return true;
 }
 
@@ -1553,20 +1562,7 @@ PyObject* HasField(CMessage* self, PyObject* arg) {
   if (message->GetReflection()->HasField(*message, field_descriptor)) {
     Py_RETURN_TRUE;
   }
-  if (!message->GetReflection()->SupportsUnknownEnumValues() &&
-      field_descriptor->cpp_type() == FieldDescriptor::CPPTYPE_ENUM) {
-    // Special case: Python HasField() differs in semantics from C++
-    // slightly: we return HasField('enum_field') == true if there is
-    // an unknown enum value present. To implement this we have to
-    // look in the UnknownFieldSet.
-    const UnknownFieldSet& unknown_field_set =
-        message->GetReflection()->GetUnknownFields(*message);
-    for (int i = 0; i < unknown_field_set.field_count(); ++i) {
-      if (unknown_field_set.field(i).number() == field_descriptor->number()) {
-        Py_RETURN_TRUE;
-      }
-    }
-  }
+
   Py_RETURN_FALSE;
 }
 
@@ -1745,12 +1741,6 @@ PyObject* ClearFieldByDescriptor(
   AssureWritable(self);
   Message* message = self->message;
   message->GetReflection()->ClearField(message, field_descriptor);
-  if (field_descriptor->cpp_type() == FieldDescriptor::CPPTYPE_ENUM &&
-      !message->GetReflection()->SupportsUnknownEnumValues()) {
-    UnknownFieldSet* unknown_field_set =
-        message->GetReflection()->MutableUnknownFields(message);
-    unknown_field_set->DeleteByNumber(field_descriptor->number());
-  }
   Py_RETURN_NONE;
 }
 
@@ -2345,27 +2335,9 @@ PyObject* InternalGetScalar(const Message* message,
       break;
     }
     case FieldDescriptor::CPPTYPE_ENUM: {
-      if (!message->GetReflection()->SupportsUnknownEnumValues() &&
-          !message->GetReflection()->HasField(*message, field_descriptor)) {
-        // Look for the value in the unknown fields.
-        const UnknownFieldSet& unknown_field_set =
-            message->GetReflection()->GetUnknownFields(*message);
-        for (int i = 0; i < unknown_field_set.field_count(); ++i) {
-          if (unknown_field_set.field(i).number() ==
-              field_descriptor->number() &&
-              unknown_field_set.field(i).type() ==
-              google::protobuf::UnknownField::TYPE_VARINT) {
-            result = PyInt_FromLong(unknown_field_set.field(i).varint());
-            break;
-          }
-        }
-      }
-
-      if (result == NULL) {
-        const EnumValueDescriptor* enum_value =
-            message->GetReflection()->GetEnum(*message, field_descriptor);
-        result = PyInt_FromLong(enum_value->number());
-      }
+      const EnumValueDescriptor* enum_value =
+          message->GetReflection()->GetEnum(*message, field_descriptor);
+      result = PyInt_FromLong(enum_value->number());
       break;
     }
     default:
@@ -3087,5 +3059,4 @@ bool InitProto2MessageModule(PyObject *m) {
 
 }  // namespace python
 }  // namespace protobuf
-
 }  // namespace google

python/google/protobuf/pyext/message_module.cc

@@ -28,8 +28,33 @@
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+#include <Python.h>
+
 #include <google/protobuf/pyext/message.h>
 
+#include <google/protobuf/message_lite.h>
+
+static PyObject* GetPythonProto3PreserveUnknownsDefault(
+    PyObject* /*m*/, PyObject* /*args*/) {
+  if (google::protobuf::internal::GetProto3PreserveUnknownsDefault()) {
+    Py_RETURN_TRUE;
+  } else {
+    Py_RETURN_FALSE;
+  }
+}
+
+static PyObject* SetPythonProto3PreserveUnknownsDefault(
+    PyObject* /*m*/, PyObject* arg) {
+  if (!arg || !PyBool_Check(arg)) {
+    PyErr_SetString(
+        PyExc_TypeError,
+        "Argument to SetPythonProto3PreserveUnknownsDefault must be boolean");
+    return NULL;
+  }
+  google::protobuf::internal::SetProto3PreserveUnknownsDefault(PyObject_IsTrue(arg));
+  Py_RETURN_NONE;
+}
+
 static const char module_docstring[] =
 "python-proto2 is a module that can be used to enhance proto2 Python API\n"
 "performance.\n"
@@ -41,6 +66,14 @@ static PyMethodDef ModuleMethods[] = {
   {"SetAllowOversizeProtos",
     (PyCFunction)google::protobuf::python::cmessage::SetAllowOversizeProtos,
     METH_O, "Enable/disable oversize proto parsing."},
+  // DO NOT USE: For migration and testing only.
+  {"GetPythonProto3PreserveUnknownsDefault",
+    (PyCFunction)GetPythonProto3PreserveUnknownsDefault,
+    METH_NOARGS, "Get Proto3 preserve unknowns default."},
+  // DO NOT USE: For migration and testing only.
+  {"SetPythonProto3PreserveUnknownsDefault",
+    (PyCFunction)SetPythonProto3PreserveUnknownsDefault,
+    METH_O, "Enable/disable proto3 unknowns preservation."},
   { NULL, NULL}
 };
 

python/google/protobuf/pyext/python.proto

@@ -58,11 +58,11 @@ message ForeignMessage {
   repeated int32 d = 2;
 }
 
-message TestAllExtensions {
+message TestAllExtensions {  // extension begin
   extensions 1 to max;
-}
+}  // extension end
 
-extend TestAllExtensions {
+extend TestAllExtensions {  // extension begin
   optional TestAllTypes.NestedMessage optional_nested_message_extension = 1;
   repeated TestAllTypes.NestedMessage repeated_nested_message_extension = 2;
-}
+}  // extension end

python/google/protobuf/pyext/repeated_scalar_container.cc

@@ -261,22 +261,6 @@ static PyObject* Item(RepeatedScalarContainer* self, Py_ssize_t index) {
       result = ToStringObject(field_descriptor, value);
       break;
     }
-    case FieldDescriptor::CPPTYPE_MESSAGE: {
-      PyObject* py_cmsg = PyObject_CallObject(reinterpret_cast<PyObject*>(
-          &CMessage_Type), NULL);
-      if (py_cmsg == NULL) {
-        return NULL;
-      }
-      CMessage* cmsg = reinterpret_cast<CMessage*>(py_cmsg);
-      const Message& msg = reflection->GetRepeatedMessage(
-          *message, field_descriptor, index);
-      cmsg->owner = self->owner;
-      cmsg->parent = self->parent;
-      cmsg->message = const_cast<Message*>(&msg);
-      cmsg->read_only = false;
-      result = reinterpret_cast<PyObject*>(py_cmsg);
-      break;
-    }
     default:
       PyErr_Format(
           PyExc_SystemError,

python/setup.py

@@ -78,6 +78,7 @@ def generate_proto(source, require = True):
 
 def GenerateUnittestProtos():
   generate_proto("../src/google/protobuf/any_test.proto", False)
+  generate_proto("../src/google/protobuf/map_proto2_unittest.proto", False)
   generate_proto("../src/google/protobuf/map_unittest.proto", False)
   generate_proto("../src/google/protobuf/test_messages_proto3.proto", False)
   generate_proto("../src/google/protobuf/test_messages_proto2.proto", False)

src/Makefile.am

@@ -356,7 +356,10 @@ libprotoc_la_SOURCES =                                         \
   google/protobuf/compiler/cpp/cpp_message.h                   \
   google/protobuf/compiler/cpp/cpp_message_field.cc            \
   google/protobuf/compiler/cpp/cpp_message_field.h             \
+  google/protobuf/compiler/cpp/cpp_message_layout_helper.h     \
   google/protobuf/compiler/cpp/cpp_options.h                   \
+  google/protobuf/compiler/cpp/cpp_padding_optimizer.cc        \
+  google/protobuf/compiler/cpp/cpp_padding_optimizer.h         \
   google/protobuf/compiler/cpp/cpp_primitive_field.cc          \
   google/protobuf/compiler/cpp/cpp_primitive_field.h           \
   google/protobuf/compiler/cpp/cpp_service.cc                  \

src/google/protobuf/any.pb.cc

@@ -1,7 +1,6 @@
 // Generated by the protocol buffer compiler.  DO NOT EDIT!
 // source: google/protobuf/any.proto
 
-#define INTERNAL_SUPPRESS_PROTOBUF_FIELD_DEPRECATION
 #include <google/protobuf/any.pb.h>
 
 #include <algorithm>
@@ -16,57 +15,52 @@
 #include <google/protobuf/reflection_ops.h>
 #include <google/protobuf/wire_format.h>
 // @@protoc_insertion_point(includes)
-
 namespace google {
 namespace protobuf {
 class AnyDefaultTypeInternal {
-public:
- ::google::protobuf::internal::ExplicitlyConstructed<Any>
-     _instance;
+ public:
+  ::google::protobuf::internal::ExplicitlyConstructed<Any>
+      _instance;
 } _Any_default_instance_;
-
+}  // namespace protobuf
+}  // namespace google
 namespace protobuf_google_2fprotobuf_2fany_2eproto {
+void InitDefaultsAnyImpl() {
+  GOOGLE_PROTOBUF_VERIFY_VERSION;
 
+  ::google::protobuf::internal::InitProtobufDefaults();
+  {
+    void* ptr = &::google::protobuf::_Any_default_instance_;
+    new (ptr) ::google::protobuf::Any();
+    ::google::protobuf::internal::OnShutdownDestroyMessage(ptr);
+  }
+  ::google::protobuf::Any::InitAsDefaultInstance();
+}
 
-namespace {
+void InitDefaultsAny() {
+  static GOOGLE_PROTOBUF_DECLARE_ONCE(once);
+  ::google::protobuf::GoogleOnceInit(&once, &InitDefaultsAnyImpl);
+}
 
 ::google::protobuf::Metadata file_level_metadata[1];
 
-}  // namespace
-
-PROTOBUF_CONSTEXPR_VAR ::google::protobuf::internal::ParseTableField
-    const TableStruct::entries[] GOOGLE_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = {
-  {0, 0, 0, ::google::protobuf::internal::kInvalidMask, 0, 0},
-};
-
-PROTOBUF_CONSTEXPR_VAR ::google::protobuf::internal::AuxillaryParseTableField
-    const TableStruct::aux[] GOOGLE_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = {
-  ::google::protobuf::internal::AuxillaryParseTableField(),
-};
-PROTOBUF_CONSTEXPR_VAR ::google::protobuf::internal::ParseTable const
-    TableStruct::schema[] GOOGLE_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = {
-  { NULL, NULL, 0, -1, -1, -1, -1, NULL, false },
-};
-
-const ::google::protobuf::uint32 TableStruct::offsets[] GOOGLE_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = {
+const ::google::protobuf::uint32 TableStruct::offsets[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = {
   ~0u,  // no _has_bits_
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Any, _internal_metadata_),
+  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Any, _internal_metadata_),
   ~0u,  // no _extensions_
   ~0u,  // no _oneof_case_
   ~0u,  // no _weak_field_map_
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Any, type_url_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Any, value_),
+  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Any, type_url_),
+  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Any, value_),
 };
-static const ::google::protobuf::internal::MigrationSchema schemas[] GOOGLE_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = {
-  { 0, -1, sizeof(Any)},
+static const ::google::protobuf::internal::MigrationSchema schemas[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = {
+  { 0, -1, sizeof(::google::protobuf::Any)},
 };
 
 static ::google::protobuf::Message const * const file_default_instances[] = {
-  reinterpret_cast<const ::google::protobuf::Message*>(&_Any_default_instance_),
+  reinterpret_cast<const ::google::protobuf::Message*>(&::google::protobuf::_Any_default_instance_),
 };
 
-namespace {
-
 void protobuf_AssignDescriptors() {
   AddDescriptors();
   ::google::protobuf::MessageFactory* factory = NULL;
@@ -80,29 +74,15 @@ void protobuf_AssignDescriptorsOnce() {
   ::google::protobuf::GoogleOnceInit(&once, &protobuf_AssignDescriptors);
 }
 
-void protobuf_RegisterTypes(const ::std::string&) GOOGLE_ATTRIBUTE_COLD;
+void protobuf_RegisterTypes(const ::std::string&) GOOGLE_PROTOBUF_ATTRIBUTE_COLD;
 void protobuf_RegisterTypes(const ::std::string&) {
   protobuf_AssignDescriptorsOnce();
   ::google::protobuf::internal::RegisterAllTypes(file_level_metadata, 1);
 }
 
-}  // namespace
-void TableStruct::InitDefaultsImpl() {
-  GOOGLE_PROTOBUF_VERIFY_VERSION;
-
-  ::google::protobuf::internal::InitProtobufDefaults();
-  _Any_default_instance_._instance.DefaultConstruct();
-  ::google::protobuf::internal::OnShutdownDestroyMessage(
-      &_Any_default_instance_);}
-
-void InitDefaults() {
-  static GOOGLE_PROTOBUF_DECLARE_ONCE(once);
-  ::google::protobuf::GoogleOnceInit(&once, &TableStruct::InitDefaultsImpl);
-}
-namespace {
 void AddDescriptorsImpl() {
   InitDefaults();
-  static const char descriptor[] GOOGLE_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = {
+  static const char descriptor[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = {
       "\n\031google/protobuf/any.proto\022\017google.prot"
       "obuf\"&\n\003Any\022\020\n\010type_url\030\001 \001(\t\022\r\n\005value\030\002"
       " \001(\014Bo\n\023com.google.protobufB\010AnyProtoP\001Z"
@@ -115,7 +95,6 @@ void AddDescriptorsImpl() {
   ::google::protobuf::MessageFactory::InternalRegisterGeneratedFile(
     "google/protobuf/any.proto", &protobuf_RegisterTypes);
 }
-} // anonymous namespace
 
 void AddDescriptors() {
   static GOOGLE_PROTOBUF_DECLARE_ONCE(once);
@@ -127,12 +106,14 @@ struct StaticDescriptorInitializer {
     AddDescriptors();
   }
 } static_descriptor_initializer;
-
 }  // namespace protobuf_google_2fprotobuf_2fany_2eproto
-
+namespace google {
+namespace protobuf {
 
 // ===================================================================
 
+void Any::InitAsDefaultInstance() {
+}
 void Any::PackFrom(const ::google::protobuf::Message& message) {
   _any_metadata_.PackFrom(message);
 }
@@ -154,7 +135,7 @@ const int Any::kValueFieldNumber;
 Any::Any()
   : ::google::protobuf::Message(), _internal_metadata_(NULL), _any_metadata_(&type_url_, &value_) {
   if (GOOGLE_PREDICT_TRUE(this != internal_default_instance())) {
-    protobuf_google_2fprotobuf_2fany_2eproto::InitDefaults();
+    ::protobuf_google_2fprotobuf_2fany_2eproto::InitDefaultsAny();
   }
   SharedCtor();
   // @@protoc_insertion_point(constructor:google.protobuf.Any)
@@ -198,12 +179,12 @@ void Any::SetCachedSize(int size) const {
   GOOGLE_SAFE_CONCURRENT_WRITES_END();
 }
 const ::google::protobuf::Descriptor* Any::descriptor() {
-  protobuf_google_2fprotobuf_2fany_2eproto::protobuf_AssignDescriptorsOnce();
-  return protobuf_google_2fprotobuf_2fany_2eproto::file_level_metadata[kIndexInFileMessages].descriptor;
+  ::protobuf_google_2fprotobuf_2fany_2eproto::protobuf_AssignDescriptorsOnce();
+  return ::protobuf_google_2fprotobuf_2fany_2eproto::file_level_metadata[kIndexInFileMessages].descriptor;
 }
 
 const Any& Any::default_instance() {
-  protobuf_google_2fprotobuf_2fany_2eproto::InitDefaults();
+  ::protobuf_google_2fprotobuf_2fany_2eproto::InitDefaultsAny();
   return *internal_default_instance();
 }
 
@@ -440,122 +421,11 @@ void Any::InternalSwap(Any* other) {
 
 ::google::protobuf::Metadata Any::GetMetadata() const {
   protobuf_google_2fprotobuf_2fany_2eproto::protobuf_AssignDescriptorsOnce();
-  return protobuf_google_2fprotobuf_2fany_2eproto::file_level_metadata[kIndexInFileMessages];
+  return ::protobuf_google_2fprotobuf_2fany_2eproto::file_level_metadata[kIndexInFileMessages];
 }
 
-#if PROTOBUF_INLINE_NOT_IN_HEADERS
-// Any
-
-// string type_url = 1;
-void Any::clear_type_url() {
-  type_url_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-}
-const ::std::string& Any::type_url() const {
-  // @@protoc_insertion_point(field_get:google.protobuf.Any.type_url)
-  return type_url_.GetNoArena();
-}
-void Any::set_type_url(const ::std::string& value) {
-  
-  type_url_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value);
-  // @@protoc_insertion_point(field_set:google.protobuf.Any.type_url)
-}
-#if LANG_CXX11
-void Any::set_type_url(::std::string&& value) {
-  
-  type_url_.SetNoArena(
-    &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value));
-  // @@protoc_insertion_point(field_set_rvalue:google.protobuf.Any.type_url)
-}
-#endif
-void Any::set_type_url(const char* value) {
-  GOOGLE_DCHECK(value != NULL);
-  
-  type_url_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value));
-  // @@protoc_insertion_point(field_set_char:google.protobuf.Any.type_url)
-}
-void Any::set_type_url(const char* value, size_t size) {
-  
-  type_url_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(),
-      ::std::string(reinterpret_cast<const char*>(value), size));
-  // @@protoc_insertion_point(field_set_pointer:google.protobuf.Any.type_url)
-}
-::std::string* Any::mutable_type_url() {
-  
-  // @@protoc_insertion_point(field_mutable:google.protobuf.Any.type_url)
-  return type_url_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-}
-::std::string* Any::release_type_url() {
-  // @@protoc_insertion_point(field_release:google.protobuf.Any.type_url)
-  
-  return type_url_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-}
-void Any::set_allocated_type_url(::std::string* type_url) {
-  if (type_url != NULL) {
-    
-  } else {
-    
-  }
-  type_url_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), type_url);
-  // @@protoc_insertion_point(field_set_allocated:google.protobuf.Any.type_url)
-}
-
-// bytes value = 2;
-void Any::clear_value() {
-  value_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-}
-const ::std::string& Any::value() const {
-  // @@protoc_insertion_point(field_get:google.protobuf.Any.value)
-  return value_.GetNoArena();
-}
-void Any::set_value(const ::std::string& value) {
-  
-  value_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value);
-  // @@protoc_insertion_point(field_set:google.protobuf.Any.value)
-}
-#if LANG_CXX11
-void Any::set_value(::std::string&& value) {
-  
-  value_.SetNoArena(
-    &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value));
-  // @@protoc_insertion_point(field_set_rvalue:google.protobuf.Any.value)
-}
-#endif
-void Any::set_value(const char* value) {
-  GOOGLE_DCHECK(value != NULL);
-  
-  value_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value));
-  // @@protoc_insertion_point(field_set_char:google.protobuf.Any.value)
-}
-void Any::set_value(const void* value, size_t size) {
-  
-  value_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(),
-      ::std::string(reinterpret_cast<const char*>(value), size));
-  // @@protoc_insertion_point(field_set_pointer:google.protobuf.Any.value)
-}
-::std::string* Any::mutable_value() {
-  
-  // @@protoc_insertion_point(field_mutable:google.protobuf.Any.value)
-  return value_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-}
-::std::string* Any::release_value() {
-  // @@protoc_insertion_point(field_release:google.protobuf.Any.value)
-  
-  return value_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-}
-void Any::set_allocated_value(::std::string* value) {
-  if (value != NULL) {
-    
-  } else {
-    
-  }
-  value_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value);
-  // @@protoc_insertion_point(field_set_allocated:google.protobuf.Any.value)
-}
-
-#endif  // PROTOBUF_INLINE_NOT_IN_HEADERS
 
 // @@protoc_insertion_point(namespace_scope)
-
 }  // namespace protobuf
 }  // namespace google
 

src/google/protobuf/any.pb.h

@@ -31,31 +31,33 @@
 #include <google/protobuf/unknown_field_set.h>
 #include <google/protobuf/any.h>
 // @@protoc_insertion_point(includes)
-namespace google {
-namespace protobuf {
-class Any;
-class AnyDefaultTypeInternal;
-LIBPROTOBUF_EXPORT extern AnyDefaultTypeInternal _Any_default_instance_;
-}  // namespace protobuf
-}  // namespace google
-
-namespace google {
-namespace protobuf {
 
 namespace protobuf_google_2fprotobuf_2fany_2eproto {
-// Internal implementation detail -- do not call these.
+// Internal implementation detail -- do not use these members.
 struct LIBPROTOBUF_EXPORT TableStruct {
   static const ::google::protobuf::internal::ParseTableField entries[];
   static const ::google::protobuf::internal::AuxillaryParseTableField aux[];
-  static const ::google::protobuf::internal::ParseTable schema[];
-  static const ::google::protobuf::uint32 offsets[];
+  static const ::google::protobuf::internal::ParseTable schema[1];
   static const ::google::protobuf::internal::FieldMetadata field_metadata[];
   static const ::google::protobuf::internal::SerializationTable serialization_table[];
-  static void InitDefaultsImpl();
+  static const ::google::protobuf::uint32 offsets[];
 };
 void LIBPROTOBUF_EXPORT AddDescriptors();
-void LIBPROTOBUF_EXPORT InitDefaults();
+void LIBPROTOBUF_EXPORT InitDefaultsAnyImpl();
+void LIBPROTOBUF_EXPORT InitDefaultsAny();
+inline void LIBPROTOBUF_EXPORT InitDefaults() {
+  InitDefaultsAny();
+}
 }  // namespace protobuf_google_2fprotobuf_2fany_2eproto
+namespace google {
+namespace protobuf {
+class Any;
+class AnyDefaultTypeInternal;
+LIBPROTOBUF_EXPORT extern AnyDefaultTypeInternal _Any_default_instance_;
+}  // namespace protobuf
+}  // namespace google
+namespace google {
+namespace protobuf {
 
 // ===================================================================
 
@@ -70,9 +72,25 @@ class LIBPROTOBUF_EXPORT Any : public ::google::protobuf::Message /* @@protoc_in
     CopyFrom(from);
     return *this;
   }
+  #if LANG_CXX11
+  Any(Any&& from) noexcept
+    : Any() {
+    *this = ::std::move(from);
+  }
+
+  inline Any& operator=(Any&& from) noexcept {
+    if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) {
+      if (this != &from) InternalSwap(&from);
+    } else {
+      CopyFrom(from);
+    }
+    return *this;
+  }
+  #endif
   static const ::google::protobuf::Descriptor* descriptor();
   static const Any& default_instance();
 
+  static void InitAsDefaultInstance();  // FOR INTERNAL USE ONLY
   static inline const Any* internal_default_instance() {
     return reinterpret_cast<const Any*>(
                &_Any_default_instance_);
@@ -171,14 +189,14 @@ class LIBPROTOBUF_EXPORT Any : public ::google::protobuf::Message /* @@protoc_in
   ::google::protobuf::internal::ArenaStringPtr value_;
   mutable int _cached_size_;
   ::google::protobuf::internal::AnyMetadata _any_metadata_;
-  friend struct protobuf_google_2fprotobuf_2fany_2eproto::TableStruct;
+  friend struct ::protobuf_google_2fprotobuf_2fany_2eproto::TableStruct;
+  friend void ::protobuf_google_2fprotobuf_2fany_2eproto::InitDefaultsAnyImpl();
 };
 // ===================================================================
 
 
 // ===================================================================
 
-#if !PROTOBUF_INLINE_NOT_IN_HEADERS
 #ifdef __GNUC__
   #pragma GCC diagnostic push
   #pragma GCC diagnostic ignored "-Wstrict-aliasing"
@@ -294,11 +312,9 @@ inline void Any::set_allocated_value(::std::string* value) {
 #ifdef __GNUC__
   #pragma GCC diagnostic pop
 #endif  // __GNUC__
-#endif  // !PROTOBUF_INLINE_NOT_IN_HEADERS
 
 // @@protoc_insertion_point(namespace_scope)
 
-
 }  // namespace protobuf
 }  // namespace google
 

src/google/protobuf/api.pb.cc

@@ -1,7 +1,6 @@
 // Generated by the protocol buffer compiler.  DO NOT EDIT!
 // source: google/protobuf/api.proto
 
-#define INTERNAL_SUPPRESS_PROTOBUF_FIELD_DEPRECATION
 #include <google/protobuf/api.pb.h>
 
 #include <algorithm>
@@ -16,97 +15,129 @@
 #include <google/protobuf/reflection_ops.h>
 #include <google/protobuf/wire_format.h>
 // @@protoc_insertion_point(includes)
-
 namespace google {
 namespace protobuf {
 class ApiDefaultTypeInternal {
-public:
- ::google::protobuf::internal::ExplicitlyConstructed<Api>
-     _instance;
+ public:
+  ::google::protobuf::internal::ExplicitlyConstructed<Api>
+      _instance;
 } _Api_default_instance_;
 class MethodDefaultTypeInternal {
-public:
- ::google::protobuf::internal::ExplicitlyConstructed<Method>
-     _instance;
+ public:
+  ::google::protobuf::internal::ExplicitlyConstructed<Method>
+      _instance;
 } _Method_default_instance_;
 class MixinDefaultTypeInternal {
-public:
- ::google::protobuf::internal::ExplicitlyConstructed<Mixin>
-     _instance;
+ public:
+  ::google::protobuf::internal::ExplicitlyConstructed<Mixin>
+      _instance;
 } _Mixin_default_instance_;
-
+}  // namespace protobuf
+}  // namespace google
 namespace protobuf_google_2fprotobuf_2fapi_2eproto {
+void InitDefaultsApiImpl() {
+  GOOGLE_PROTOBUF_VERIFY_VERSION;
 
+  ::google::protobuf::internal::InitProtobufDefaults();
+  protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaultsMethod();
+  protobuf_google_2fprotobuf_2ftype_2eproto::InitDefaultsOption();
+  protobuf_google_2fprotobuf_2fsource_5fcontext_2eproto::InitDefaultsSourceContext();
+  protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaultsMixin();
+  {
+    void* ptr = &::google::protobuf::_Api_default_instance_;
+    new (ptr) ::google::protobuf::Api();
+    ::google::protobuf::internal::OnShutdownDestroyMessage(ptr);
+  }
+  ::google::protobuf::Api::InitAsDefaultInstance();
+}
 
-namespace {
+void InitDefaultsApi() {
+  static GOOGLE_PROTOBUF_DECLARE_ONCE(once);
+  ::google::protobuf::GoogleOnceInit(&once, &InitDefaultsApiImpl);
+}
 
-::google::protobuf::Metadata file_level_metadata[3];
+void InitDefaultsMethodImpl() {
+  GOOGLE_PROTOBUF_VERIFY_VERSION;
 
-}  // namespace
+  ::google::protobuf::internal::InitProtobufDefaults();
+  protobuf_google_2fprotobuf_2ftype_2eproto::InitDefaultsOption();
+  {
+    void* ptr = &::google::protobuf::_Method_default_instance_;
+    new (ptr) ::google::protobuf::Method();
+    ::google::protobuf::internal::OnShutdownDestroyMessage(ptr);
+  }
+  ::google::protobuf::Method::InitAsDefaultInstance();
+}
 
-PROTOBUF_CONSTEXPR_VAR ::google::protobuf::internal::ParseTableField
-    const TableStruct::entries[] GOOGLE_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = {
-  {0, 0, 0, ::google::protobuf::internal::kInvalidMask, 0, 0},
-};
+void InitDefaultsMethod() {
+  static GOOGLE_PROTOBUF_DECLARE_ONCE(once);
+  ::google::protobuf::GoogleOnceInit(&once, &InitDefaultsMethodImpl);
+}
 
-PROTOBUF_CONSTEXPR_VAR ::google::protobuf::internal::AuxillaryParseTableField
-    const TableStruct::aux[] GOOGLE_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = {
-  ::google::protobuf::internal::AuxillaryParseTableField(),
-};
-PROTOBUF_CONSTEXPR_VAR ::google::protobuf::internal::ParseTable const
-    TableStruct::schema[] GOOGLE_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = {
-  { NULL, NULL, 0, -1, -1, -1, -1, NULL, false },
-  { NULL, NULL, 0, -1, -1, -1, -1, NULL, false },
-  { NULL, NULL, 0, -1, -1, -1, -1, NULL, false },
-};
+void InitDefaultsMixinImpl() {
+  GOOGLE_PROTOBUF_VERIFY_VERSION;
 
-const ::google::protobuf::uint32 TableStruct::offsets[] GOOGLE_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = {
+  ::google::protobuf::internal::InitProtobufDefaults();
+  {
+    void* ptr = &::google::protobuf::_Mixin_default_instance_;
+    new (ptr) ::google::protobuf::Mixin();
+    ::google::protobuf::internal::OnShutdownDestroyMessage(ptr);
+  }
+  ::google::protobuf::Mixin::InitAsDefaultInstance();
+}
+
+void InitDefaultsMixin() {
+  static GOOGLE_PROTOBUF_DECLARE_ONCE(once);
+  ::google::protobuf::GoogleOnceInit(&once, &InitDefaultsMixinImpl);
+}
+
+::google::protobuf::Metadata file_level_metadata[3];
+
+const ::google::protobuf::uint32 TableStruct::offsets[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = {
   ~0u,  // no _has_bits_
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Api, _internal_metadata_),
+  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Api, _internal_metadata_),
   ~0u,  // no _extensions_
   ~0u,  // no _oneof_case_
   ~0u,  // no _weak_field_map_
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Api, name_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Api, methods_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Api, options_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Api, version_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Api, source_context_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Api, mixins_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Api, syntax_),
+  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Api, name_),
+  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Api, methods_),
+  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Api, options_),
+  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Api, version_),
+  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Api, source_context_),
+  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Api, mixins_),
+  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Api, syntax_),
   ~0u,  // no _has_bits_
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Method, _internal_metadata_),
+  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Method, _internal_metadata_),
   ~0u,  // no _extensions_
   ~0u,  // no _oneof_case_
   ~0u,  // no _weak_field_map_
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Method, name_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Method, request_type_url_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Method, request_streaming_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Method, response_type_url_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Method, response_streaming_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Method, options_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Method, syntax_),
+  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Method, name_),
+  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Method, request_type_url_),
+  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Method, request_streaming_),
+  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Method, response_type_url_),
+  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Method, response_streaming_),
+  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Method, options_),
+  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Method, syntax_),
   ~0u,  // no _has_bits_
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Mixin, _internal_metadata_),
+  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Mixin, _internal_metadata_),
   ~0u,  // no _extensions_
   ~0u,  // no _oneof_case_
   ~0u,  // no _weak_field_map_
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Mixin, name_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Mixin, root_),
+  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Mixin, name_),
+  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Mixin, root_),
 };
-static const ::google::protobuf::internal::MigrationSchema schemas[] GOOGLE_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = {
-  { 0, -1, sizeof(Api)},
-  { 12, -1, sizeof(Method)},
-  { 24, -1, sizeof(Mixin)},
+static const ::google::protobuf::internal::MigrationSchema schemas[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = {
+  { 0, -1, sizeof(::google::protobuf::Api)},
+  { 12, -1, sizeof(::google::protobuf::Method)},
+  { 24, -1, sizeof(::google::protobuf::Mixin)},
 };
 
 static ::google::protobuf::Message const * const file_default_instances[] = {
-  reinterpret_cast<const ::google::protobuf::Message*>(&_Api_default_instance_),
-  reinterpret_cast<const ::google::protobuf::Message*>(&_Method_default_instance_),
-  reinterpret_cast<const ::google::protobuf::Message*>(&_Mixin_default_instance_),
+  reinterpret_cast<const ::google::protobuf::Message*>(&::google::protobuf::_Api_default_instance_),
+  reinterpret_cast<const ::google::protobuf::Message*>(&::google::protobuf::_Method_default_instance_),
+  reinterpret_cast<const ::google::protobuf::Message*>(&::google::protobuf::_Mixin_default_instance_),
 };
 
-namespace {
-
 void protobuf_AssignDescriptors() {
   AddDescriptors();
   ::google::protobuf::MessageFactory* factory = NULL;
@@ -120,37 +151,15 @@ void protobuf_AssignDescriptorsOnce() {
   ::google::protobuf::GoogleOnceInit(&once, &protobuf_AssignDescriptors);
 }
 
-void protobuf_RegisterTypes(const ::std::string&) GOOGLE_ATTRIBUTE_COLD;
+void protobuf_RegisterTypes(const ::std::string&) GOOGLE_PROTOBUF_ATTRIBUTE_COLD;
 void protobuf_RegisterTypes(const ::std::string&) {
   protobuf_AssignDescriptorsOnce();
   ::google::protobuf::internal::RegisterAllTypes(file_level_metadata, 3);
 }
 
-}  // namespace
-void TableStruct::InitDefaultsImpl() {
-  GOOGLE_PROTOBUF_VERIFY_VERSION;
-
-  ::google::protobuf::internal::InitProtobufDefaults();
-  ::google::protobuf::protobuf_google_2fprotobuf_2fsource_5fcontext_2eproto::InitDefaults();
-  ::google::protobuf::protobuf_google_2fprotobuf_2ftype_2eproto::InitDefaults();
-  _Api_default_instance_._instance.DefaultConstruct();
-  ::google::protobuf::internal::OnShutdownDestroyMessage(
-      &_Api_default_instance_);_Method_default_instance_._instance.DefaultConstruct();
-  ::google::protobuf::internal::OnShutdownDestroyMessage(
-      &_Method_default_instance_);_Mixin_default_instance_._instance.DefaultConstruct();
-  ::google::protobuf::internal::OnShutdownDestroyMessage(
-      &_Mixin_default_instance_);_Api_default_instance_._instance.get_mutable()->source_context_ = const_cast< ::google::protobuf::SourceContext*>(
-      ::google::protobuf::SourceContext::internal_default_instance());
-}
-
-void InitDefaults() {
-  static GOOGLE_PROTOBUF_DECLARE_ONCE(once);
-  ::google::protobuf::GoogleOnceInit(&once, &TableStruct::InitDefaultsImpl);
-}
-namespace {
 void AddDescriptorsImpl() {
   InitDefaults();
-  static const char descriptor[] GOOGLE_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = {
+  static const char descriptor[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = {
       "\n\031google/protobuf/api.proto\022\017google.prot"
       "obuf\032$google/protobuf/source_context.pro"
       "to\032\032google/protobuf/type.proto\"\201\002\n\003Api\022\014"
@@ -175,10 +184,9 @@ void AddDescriptorsImpl() {
       descriptor, 750);
   ::google::protobuf::MessageFactory::InternalRegisterGeneratedFile(
     "google/protobuf/api.proto", &protobuf_RegisterTypes);
-  ::google::protobuf::protobuf_google_2fprotobuf_2fsource_5fcontext_2eproto::AddDescriptors();
-  ::google::protobuf::protobuf_google_2fprotobuf_2ftype_2eproto::AddDescriptors();
+  ::protobuf_google_2fprotobuf_2fsource_5fcontext_2eproto::AddDescriptors();
+  ::protobuf_google_2fprotobuf_2ftype_2eproto::AddDescriptors();
 }
-} // anonymous namespace
 
 void AddDescriptors() {
   static GOOGLE_PROTOBUF_DECLARE_ONCE(once);
@@ -190,12 +198,16 @@ struct StaticDescriptorInitializer {
     AddDescriptors();
   }
 } static_descriptor_initializer;
-
 }  // namespace protobuf_google_2fprotobuf_2fapi_2eproto
-
+namespace google {
+namespace protobuf {
 
 // ===================================================================
 
+void Api::InitAsDefaultInstance() {
+  ::google::protobuf::_Api_default_instance_._instance.get_mutable()->source_context_ = const_cast< ::google::protobuf::SourceContext*>(
+      ::google::protobuf::SourceContext::internal_default_instance());
+}
 #if !defined(_MSC_VER) || _MSC_VER >= 1900
 const int Api::kNameFieldNumber;
 const int Api::kMethodsFieldNumber;
@@ -209,7 +221,7 @@ const int Api::kSyntaxFieldNumber;
 Api::Api()
   : ::google::protobuf::Message(), _internal_metadata_(NULL) {
   if (GOOGLE_PREDICT_TRUE(this != internal_default_instance())) {
-    protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaults();
+    ::protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaultsApi();
   }
   SharedCtor();
   // @@protoc_insertion_point(constructor:google.protobuf.Api)
@@ -265,12 +277,12 @@ void Api::SetCachedSize(int size) const {
   GOOGLE_SAFE_CONCURRENT_WRITES_END();
 }
 const ::google::protobuf::Descriptor* Api::descriptor() {
-  protobuf_google_2fprotobuf_2fapi_2eproto::protobuf_AssignDescriptorsOnce();
-  return protobuf_google_2fprotobuf_2fapi_2eproto::file_level_metadata[kIndexInFileMessages].descriptor;
+  ::protobuf_google_2fprotobuf_2fapi_2eproto::protobuf_AssignDescriptorsOnce();
+  return ::protobuf_google_2fprotobuf_2fapi_2eproto::file_level_metadata[kIndexInFileMessages].descriptor;
 }
 
 const Api& Api::default_instance() {
-  protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaults();
+  ::protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaultsApi();
   return *internal_default_instance();
 }
 
@@ -720,266 +732,14 @@ void Api::InternalSwap(Api* other) {
 
 ::google::protobuf::Metadata Api::GetMetadata() const {
   protobuf_google_2fprotobuf_2fapi_2eproto::protobuf_AssignDescriptorsOnce();
-  return protobuf_google_2fprotobuf_2fapi_2eproto::file_level_metadata[kIndexInFileMessages];
+  return ::protobuf_google_2fprotobuf_2fapi_2eproto::file_level_metadata[kIndexInFileMessages];
 }
 
-#if PROTOBUF_INLINE_NOT_IN_HEADERS
-// Api
-
-// string name = 1;
-void Api::clear_name() {
-  name_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-}
-const ::std::string& Api::name() const {
-  // @@protoc_insertion_point(field_get:google.protobuf.Api.name)
-  return name_.GetNoArena();
-}
-void Api::set_name(const ::std::string& value) {
-  
-  name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value);
-  // @@protoc_insertion_point(field_set:google.protobuf.Api.name)
-}
-#if LANG_CXX11
-void Api::set_name(::std::string&& value) {
-  
-  name_.SetNoArena(
-    &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value));
-  // @@protoc_insertion_point(field_set_rvalue:google.protobuf.Api.name)
-}
-#endif
-void Api::set_name(const char* value) {
-  GOOGLE_DCHECK(value != NULL);
-  
-  name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value));
-  // @@protoc_insertion_point(field_set_char:google.protobuf.Api.name)
-}
-void Api::set_name(const char* value, size_t size) {
-  
-  name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(),
-      ::std::string(reinterpret_cast<const char*>(value), size));
-  // @@protoc_insertion_point(field_set_pointer:google.protobuf.Api.name)
-}
-::std::string* Api::mutable_name() {
-  
-  // @@protoc_insertion_point(field_mutable:google.protobuf.Api.name)
-  return name_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-}
-::std::string* Api::release_name() {
-  // @@protoc_insertion_point(field_release:google.protobuf.Api.name)
-  
-  return name_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-}
-void Api::set_allocated_name(::std::string* name) {
-  if (name != NULL) {
-    
-  } else {
-    
-  }
-  name_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), name);
-  // @@protoc_insertion_point(field_set_allocated:google.protobuf.Api.name)
-}
-
-// repeated .google.protobuf.Method methods = 2;
-int Api::methods_size() const {
-  return methods_.size();
-}
-void Api::clear_methods() {
-  methods_.Clear();
-}
-const ::google::protobuf::Method& Api::methods(int index) const {
-  // @@protoc_insertion_point(field_get:google.protobuf.Api.methods)
-  return methods_.Get(index);
-}
-::google::protobuf::Method* Api::mutable_methods(int index) {
-  // @@protoc_insertion_point(field_mutable:google.protobuf.Api.methods)
-  return methods_.Mutable(index);
-}
-::google::protobuf::Method* Api::add_methods() {
-  // @@protoc_insertion_point(field_add:google.protobuf.Api.methods)
-  return methods_.Add();
-}
-::google::protobuf::RepeatedPtrField< ::google::protobuf::Method >*
-Api::mutable_methods() {
-  // @@protoc_insertion_point(field_mutable_list:google.protobuf.Api.methods)
-  return &methods_;
-}
-const ::google::protobuf::RepeatedPtrField< ::google::protobuf::Method >&
-Api::methods() const {
-  // @@protoc_insertion_point(field_list:google.protobuf.Api.methods)
-  return methods_;
-}
-
-// repeated .google.protobuf.Option options = 3;
-int Api::options_size() const {
-  return options_.size();
-}
-void Api::clear_options() {
-  options_.Clear();
-}
-const ::google::protobuf::Option& Api::options(int index) const {
-  // @@protoc_insertion_point(field_get:google.protobuf.Api.options)
-  return options_.Get(index);
-}
-::google::protobuf::Option* Api::mutable_options(int index) {
-  // @@protoc_insertion_point(field_mutable:google.protobuf.Api.options)
-  return options_.Mutable(index);
-}
-::google::protobuf::Option* Api::add_options() {
-  // @@protoc_insertion_point(field_add:google.protobuf.Api.options)
-  return options_.Add();
-}
-::google::protobuf::RepeatedPtrField< ::google::protobuf::Option >*
-Api::mutable_options() {
-  // @@protoc_insertion_point(field_mutable_list:google.protobuf.Api.options)
-  return &options_;
-}
-const ::google::protobuf::RepeatedPtrField< ::google::protobuf::Option >&
-Api::options() const {
-  // @@protoc_insertion_point(field_list:google.protobuf.Api.options)
-  return options_;
-}
-
-// string version = 4;
-void Api::clear_version() {
-  version_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-}
-const ::std::string& Api::version() const {
-  // @@protoc_insertion_point(field_get:google.protobuf.Api.version)
-  return version_.GetNoArena();
-}
-void Api::set_version(const ::std::string& value) {
-  
-  version_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value);
-  // @@protoc_insertion_point(field_set:google.protobuf.Api.version)
-}
-#if LANG_CXX11
-void Api::set_version(::std::string&& value) {
-  
-  version_.SetNoArena(
-    &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value));
-  // @@protoc_insertion_point(field_set_rvalue:google.protobuf.Api.version)
-}
-#endif
-void Api::set_version(const char* value) {
-  GOOGLE_DCHECK(value != NULL);
-  
-  version_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value));
-  // @@protoc_insertion_point(field_set_char:google.protobuf.Api.version)
-}
-void Api::set_version(const char* value, size_t size) {
-  
-  version_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(),
-      ::std::string(reinterpret_cast<const char*>(value), size));
-  // @@protoc_insertion_point(field_set_pointer:google.protobuf.Api.version)
-}
-::std::string* Api::mutable_version() {
-  
-  // @@protoc_insertion_point(field_mutable:google.protobuf.Api.version)
-  return version_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-}
-::std::string* Api::release_version() {
-  // @@protoc_insertion_point(field_release:google.protobuf.Api.version)
-  
-  return version_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-}
-void Api::set_allocated_version(::std::string* version) {
-  if (version != NULL) {
-    
-  } else {
-    
-  }
-  version_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), version);
-  // @@protoc_insertion_point(field_set_allocated:google.protobuf.Api.version)
-}
-
-// .google.protobuf.SourceContext source_context = 5;
-bool Api::has_source_context() const {
-  return this != internal_default_instance() && source_context_ != NULL;
-}
-void Api::clear_source_context() {
-  if (GetArenaNoVirtual() == NULL && source_context_ != NULL) delete source_context_;
-  source_context_ = NULL;
-}
-const ::google::protobuf::SourceContext& Api::source_context() const {
-  const ::google::protobuf::SourceContext* p = source_context_;
-  // @@protoc_insertion_point(field_get:google.protobuf.Api.source_context)
-  return p != NULL ? *p : *reinterpret_cast<const ::google::protobuf::SourceContext*>(
-      &::google::protobuf::_SourceContext_default_instance_);
-}
-::google::protobuf::SourceContext* Api::mutable_source_context() {
-  
-  if (source_context_ == NULL) {
-    source_context_ = new ::google::protobuf::SourceContext;
-  }
-  // @@protoc_insertion_point(field_mutable:google.protobuf.Api.source_context)
-  return source_context_;
-}
-::google::protobuf::SourceContext* Api::release_source_context() {
-  // @@protoc_insertion_point(field_release:google.protobuf.Api.source_context)
-  
-  ::google::protobuf::SourceContext* temp = source_context_;
-  source_context_ = NULL;
-  return temp;
-}
-void Api::set_allocated_source_context(::google::protobuf::SourceContext* source_context) {
-  delete source_context_;
-  source_context_ = source_context;
-  if (source_context) {
-    
-  } else {
-    
-  }
-  // @@protoc_insertion_point(field_set_allocated:google.protobuf.Api.source_context)
-}
-
-// repeated .google.protobuf.Mixin mixins = 6;
-int Api::mixins_size() const {
-  return mixins_.size();
-}
-void Api::clear_mixins() {
-  mixins_.Clear();
-}
-const ::google::protobuf::Mixin& Api::mixins(int index) const {
-  // @@protoc_insertion_point(field_get:google.protobuf.Api.mixins)
-  return mixins_.Get(index);
-}
-::google::protobuf::Mixin* Api::mutable_mixins(int index) {
-  // @@protoc_insertion_point(field_mutable:google.protobuf.Api.mixins)
-  return mixins_.Mutable(index);
-}
-::google::protobuf::Mixin* Api::add_mixins() {
-  // @@protoc_insertion_point(field_add:google.protobuf.Api.mixins)
-  return mixins_.Add();
-}
-::google::protobuf::RepeatedPtrField< ::google::protobuf::Mixin >*
-Api::mutable_mixins() {
-  // @@protoc_insertion_point(field_mutable_list:google.protobuf.Api.mixins)
-  return &mixins_;
-}
-const ::google::protobuf::RepeatedPtrField< ::google::protobuf::Mixin >&
-Api::mixins() const {
-  // @@protoc_insertion_point(field_list:google.protobuf.Api.mixins)
-  return mixins_;
-}
-
-// .google.protobuf.Syntax syntax = 7;
-void Api::clear_syntax() {
-  syntax_ = 0;
-}
-::google::protobuf::Syntax Api::syntax() const {
-  // @@protoc_insertion_point(field_get:google.protobuf.Api.syntax)
-  return static_cast< ::google::protobuf::Syntax >(syntax_);
-}
-void Api::set_syntax(::google::protobuf::Syntax value) {
-  
-  syntax_ = value;
-  // @@protoc_insertion_point(field_set:google.protobuf.Api.syntax)
-}
-
-#endif  // PROTOBUF_INLINE_NOT_IN_HEADERS
 
 // ===================================================================
 
+void Method::InitAsDefaultInstance() {
+}
 #if !defined(_MSC_VER) || _MSC_VER >= 1900
 const int Method::kNameFieldNumber;
 const int Method::kRequestTypeUrlFieldNumber;
@@ -993,7 +753,7 @@ const int Method::kSyntaxFieldNumber;
 Method::Method()
   : ::google::protobuf::Message(), _internal_metadata_(NULL) {
   if (GOOGLE_PREDICT_TRUE(this != internal_default_instance())) {
-    protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaults();
+    ::protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaultsMethod();
   }
   SharedCtor();
   // @@protoc_insertion_point(constructor:google.protobuf.Method)
@@ -1049,12 +809,12 @@ void Method::SetCachedSize(int size) const {
   GOOGLE_SAFE_CONCURRENT_WRITES_END();
 }
 const ::google::protobuf::Descriptor* Method::descriptor() {
-  protobuf_google_2fprotobuf_2fapi_2eproto::protobuf_AssignDescriptorsOnce();
-  return protobuf_google_2fprotobuf_2fapi_2eproto::file_level_metadata[kIndexInFileMessages].descriptor;
+  ::protobuf_google_2fprotobuf_2fapi_2eproto::protobuf_AssignDescriptorsOnce();
+  return ::protobuf_google_2fprotobuf_2fapi_2eproto::file_level_metadata[kIndexInFileMessages].descriptor;
 }
 
 const Method& Method::default_instance() {
-  protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaults();
+  ::protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaultsMethod();
   return *internal_default_instance();
 }
 
@@ -1500,247 +1260,14 @@ void Method::InternalSwap(Method* other) {
 
 ::google::protobuf::Metadata Method::GetMetadata() const {
   protobuf_google_2fprotobuf_2fapi_2eproto::protobuf_AssignDescriptorsOnce();
-  return protobuf_google_2fprotobuf_2fapi_2eproto::file_level_metadata[kIndexInFileMessages];
-}
-
-#if PROTOBUF_INLINE_NOT_IN_HEADERS
-// Method
-
-// string name = 1;
-void Method::clear_name() {
-  name_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-}
-const ::std::string& Method::name() const {
-  // @@protoc_insertion_point(field_get:google.protobuf.Method.name)
-  return name_.GetNoArena();
-}
-void Method::set_name(const ::std::string& value) {
-  
-  name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value);
-  // @@protoc_insertion_point(field_set:google.protobuf.Method.name)
-}
-#if LANG_CXX11
-void Method::set_name(::std::string&& value) {
-  
-  name_.SetNoArena(
-    &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value));
-  // @@protoc_insertion_point(field_set_rvalue:google.protobuf.Method.name)
-}
-#endif
-void Method::set_name(const char* value) {
-  GOOGLE_DCHECK(value != NULL);
-  
-  name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value));
-  // @@protoc_insertion_point(field_set_char:google.protobuf.Method.name)
-}
-void Method::set_name(const char* value, size_t size) {
-  
-  name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(),
-      ::std::string(reinterpret_cast<const char*>(value), size));
-  // @@protoc_insertion_point(field_set_pointer:google.protobuf.Method.name)
-}
-::std::string* Method::mutable_name() {
-  
-  // @@protoc_insertion_point(field_mutable:google.protobuf.Method.name)
-  return name_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-}
-::std::string* Method::release_name() {
-  // @@protoc_insertion_point(field_release:google.protobuf.Method.name)
-  
-  return name_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-}
-void Method::set_allocated_name(::std::string* name) {
-  if (name != NULL) {
-    
-  } else {
-    
-  }
-  name_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), name);
-  // @@protoc_insertion_point(field_set_allocated:google.protobuf.Method.name)
-}
-
-// string request_type_url = 2;
-void Method::clear_request_type_url() {
-  request_type_url_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-}
-const ::std::string& Method::request_type_url() const {
-  // @@protoc_insertion_point(field_get:google.protobuf.Method.request_type_url)
-  return request_type_url_.GetNoArena();
-}
-void Method::set_request_type_url(const ::std::string& value) {
-  
-  request_type_url_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value);
-  // @@protoc_insertion_point(field_set:google.protobuf.Method.request_type_url)
-}
-#if LANG_CXX11
-void Method::set_request_type_url(::std::string&& value) {
-  
-  request_type_url_.SetNoArena(
-    &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value));
-  // @@protoc_insertion_point(field_set_rvalue:google.protobuf.Method.request_type_url)
-}
-#endif
-void Method::set_request_type_url(const char* value) {
-  GOOGLE_DCHECK(value != NULL);
-  
-  request_type_url_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value));
-  // @@protoc_insertion_point(field_set_char:google.protobuf.Method.request_type_url)
-}
-void Method::set_request_type_url(const char* value, size_t size) {
-  
-  request_type_url_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(),
-      ::std::string(reinterpret_cast<const char*>(value), size));
-  // @@protoc_insertion_point(field_set_pointer:google.protobuf.Method.request_type_url)
-}
-::std::string* Method::mutable_request_type_url() {
-  
-  // @@protoc_insertion_point(field_mutable:google.protobuf.Method.request_type_url)
-  return request_type_url_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-}
-::std::string* Method::release_request_type_url() {
-  // @@protoc_insertion_point(field_release:google.protobuf.Method.request_type_url)
-  
-  return request_type_url_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-}
-void Method::set_allocated_request_type_url(::std::string* request_type_url) {
-  if (request_type_url != NULL) {
-    
-  } else {
-    
-  }
-  request_type_url_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), request_type_url);
-  // @@protoc_insertion_point(field_set_allocated:google.protobuf.Method.request_type_url)
-}
-
-// bool request_streaming = 3;
-void Method::clear_request_streaming() {
-  request_streaming_ = false;
-}
-bool Method::request_streaming() const {
-  // @@protoc_insertion_point(field_get:google.protobuf.Method.request_streaming)
-  return request_streaming_;
-}
-void Method::set_request_streaming(bool value) {
-  
-  request_streaming_ = value;
-  // @@protoc_insertion_point(field_set:google.protobuf.Method.request_streaming)
+  return ::protobuf_google_2fprotobuf_2fapi_2eproto::file_level_metadata[kIndexInFileMessages];
 }
 
-// string response_type_url = 4;
-void Method::clear_response_type_url() {
-  response_type_url_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-}
-const ::std::string& Method::response_type_url() const {
-  // @@protoc_insertion_point(field_get:google.protobuf.Method.response_type_url)
-  return response_type_url_.GetNoArena();
-}
-void Method::set_response_type_url(const ::std::string& value) {
-  
-  response_type_url_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value);
-  // @@protoc_insertion_point(field_set:google.protobuf.Method.response_type_url)
-}
-#if LANG_CXX11
-void Method::set_response_type_url(::std::string&& value) {
-  
-  response_type_url_.SetNoArena(
-    &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value));
-  // @@protoc_insertion_point(field_set_rvalue:google.protobuf.Method.response_type_url)
-}
-#endif
-void Method::set_response_type_url(const char* value) {
-  GOOGLE_DCHECK(value != NULL);
-  
-  response_type_url_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value));
-  // @@protoc_insertion_point(field_set_char:google.protobuf.Method.response_type_url)
-}
-void Method::set_response_type_url(const char* value, size_t size) {
-  
-  response_type_url_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(),
-      ::std::string(reinterpret_cast<const char*>(value), size));
-  // @@protoc_insertion_point(field_set_pointer:google.protobuf.Method.response_type_url)
-}
-::std::string* Method::mutable_response_type_url() {
-  
-  // @@protoc_insertion_point(field_mutable:google.protobuf.Method.response_type_url)
-  return response_type_url_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-}
-::std::string* Method::release_response_type_url() {
-  // @@protoc_insertion_point(field_release:google.protobuf.Method.response_type_url)
-  
-  return response_type_url_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-}
-void Method::set_allocated_response_type_url(::std::string* response_type_url) {
-  if (response_type_url != NULL) {
-    
-  } else {
-    
-  }
-  response_type_url_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), response_type_url);
-  // @@protoc_insertion_point(field_set_allocated:google.protobuf.Method.response_type_url)
-}
-
-// bool response_streaming = 5;
-void Method::clear_response_streaming() {
-  response_streaming_ = false;
-}
-bool Method::response_streaming() const {
-  // @@protoc_insertion_point(field_get:google.protobuf.Method.response_streaming)
-  return response_streaming_;
-}
-void Method::set_response_streaming(bool value) {
-  
-  response_streaming_ = value;
-  // @@protoc_insertion_point(field_set:google.protobuf.Method.response_streaming)
-}
-
-// repeated .google.protobuf.Option options = 6;
-int Method::options_size() const {
-  return options_.size();
-}
-void Method::clear_options() {
-  options_.Clear();
-}
-const ::google::protobuf::Option& Method::options(int index) const {
-  // @@protoc_insertion_point(field_get:google.protobuf.Method.options)
-  return options_.Get(index);
-}
-::google::protobuf::Option* Method::mutable_options(int index) {
-  // @@protoc_insertion_point(field_mutable:google.protobuf.Method.options)
-  return options_.Mutable(index);
-}
-::google::protobuf::Option* Method::add_options() {
-  // @@protoc_insertion_point(field_add:google.protobuf.Method.options)
-  return options_.Add();
-}
-::google::protobuf::RepeatedPtrField< ::google::protobuf::Option >*
-Method::mutable_options() {
-  // @@protoc_insertion_point(field_mutable_list:google.protobuf.Method.options)
-  return &options_;
-}
-const ::google::protobuf::RepeatedPtrField< ::google::protobuf::Option >&
-Method::options() const {
-  // @@protoc_insertion_point(field_list:google.protobuf.Method.options)
-  return options_;
-}
-
-// .google.protobuf.Syntax syntax = 7;
-void Method::clear_syntax() {
-  syntax_ = 0;
-}
-::google::protobuf::Syntax Method::syntax() const {
-  // @@protoc_insertion_point(field_get:google.protobuf.Method.syntax)
-  return static_cast< ::google::protobuf::Syntax >(syntax_);
-}
-void Method::set_syntax(::google::protobuf::Syntax value) {
-  
-  syntax_ = value;
-  // @@protoc_insertion_point(field_set:google.protobuf.Method.syntax)
-}
-
-#endif  // PROTOBUF_INLINE_NOT_IN_HEADERS
 
 // ===================================================================
 
+void Mixin::InitAsDefaultInstance() {
+}
 #if !defined(_MSC_VER) || _MSC_VER >= 1900
 const int Mixin::kNameFieldNumber;
 const int Mixin::kRootFieldNumber;
@@ -1749,7 +1276,7 @@ const int Mixin::kRootFieldNumber;
 Mixin::Mixin()
   : ::google::protobuf::Message(), _internal_metadata_(NULL) {
   if (GOOGLE_PREDICT_TRUE(this != internal_default_instance())) {
-    protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaults();
+    ::protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaultsMixin();
   }
   SharedCtor();
   // @@protoc_insertion_point(constructor:google.protobuf.Mixin)
@@ -1792,12 +1319,12 @@ void Mixin::SetCachedSize(int size) const {
   GOOGLE_SAFE_CONCURRENT_WRITES_END();
 }
 const ::google::protobuf::Descriptor* Mixin::descriptor() {
-  protobuf_google_2fprotobuf_2fapi_2eproto::protobuf_AssignDescriptorsOnce();
-  return protobuf_google_2fprotobuf_2fapi_2eproto::file_level_metadata[kIndexInFileMessages].descriptor;
+  ::protobuf_google_2fprotobuf_2fapi_2eproto::protobuf_AssignDescriptorsOnce();
+  return ::protobuf_google_2fprotobuf_2fapi_2eproto::file_level_metadata[kIndexInFileMessages].descriptor;
 }
 
 const Mixin& Mixin::default_instance() {
-  protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaults();
+  ::protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaultsMixin();
   return *internal_default_instance();
 }
 
@@ -2046,122 +1573,11 @@ void Mixin::InternalSwap(Mixin* other) {
 
 ::google::protobuf::Metadata Mixin::GetMetadata() const {
   protobuf_google_2fprotobuf_2fapi_2eproto::protobuf_AssignDescriptorsOnce();
-  return protobuf_google_2fprotobuf_2fapi_2eproto::file_level_metadata[kIndexInFileMessages];
-}
-
-#if PROTOBUF_INLINE_NOT_IN_HEADERS
-// Mixin
-
-// string name = 1;
-void Mixin::clear_name() {
-  name_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-}
-const ::std::string& Mixin::name() const {
-  // @@protoc_insertion_point(field_get:google.protobuf.Mixin.name)
-  return name_.GetNoArena();
-}
-void Mixin::set_name(const ::std::string& value) {
-  
-  name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value);
-  // @@protoc_insertion_point(field_set:google.protobuf.Mixin.name)
-}
-#if LANG_CXX11
-void Mixin::set_name(::std::string&& value) {
-  
-  name_.SetNoArena(
-    &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value));
-  // @@protoc_insertion_point(field_set_rvalue:google.protobuf.Mixin.name)
-}
-#endif
-void Mixin::set_name(const char* value) {
-  GOOGLE_DCHECK(value != NULL);
-  
-  name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value));
-  // @@protoc_insertion_point(field_set_char:google.protobuf.Mixin.name)
-}
-void Mixin::set_name(const char* value, size_t size) {
-  
-  name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(),
-      ::std::string(reinterpret_cast<const char*>(value), size));
-  // @@protoc_insertion_point(field_set_pointer:google.protobuf.Mixin.name)
-}
-::std::string* Mixin::mutable_name() {
-  
-  // @@protoc_insertion_point(field_mutable:google.protobuf.Mixin.name)
-  return name_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-}
-::std::string* Mixin::release_name() {
-  // @@protoc_insertion_point(field_release:google.protobuf.Mixin.name)
-  
-  return name_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-}
-void Mixin::set_allocated_name(::std::string* name) {
-  if (name != NULL) {
-    
-  } else {
-    
-  }
-  name_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), name);
-  // @@protoc_insertion_point(field_set_allocated:google.protobuf.Mixin.name)
-}
-
-// string root = 2;
-void Mixin::clear_root() {
-  root_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-}
-const ::std::string& Mixin::root() const {
-  // @@protoc_insertion_point(field_get:google.protobuf.Mixin.root)
-  return root_.GetNoArena();
-}
-void Mixin::set_root(const ::std::string& value) {
-  
-  root_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value);
-  // @@protoc_insertion_point(field_set:google.protobuf.Mixin.root)
-}
-#if LANG_CXX11
-void Mixin::set_root(::std::string&& value) {
-  
-  root_.SetNoArena(
-    &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value));
-  // @@protoc_insertion_point(field_set_rvalue:google.protobuf.Mixin.root)
-}
-#endif
-void Mixin::set_root(const char* value) {
-  GOOGLE_DCHECK(value != NULL);
-  
-  root_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value));
-  // @@protoc_insertion_point(field_set_char:google.protobuf.Mixin.root)
-}
-void Mixin::set_root(const char* value, size_t size) {
-  
-  root_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(),
-      ::std::string(reinterpret_cast<const char*>(value), size));
-  // @@protoc_insertion_point(field_set_pointer:google.protobuf.Mixin.root)
-}
-::std::string* Mixin::mutable_root() {
-  
-  // @@protoc_insertion_point(field_mutable:google.protobuf.Mixin.root)
-  return root_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-}
-::std::string* Mixin::release_root() {
-  // @@protoc_insertion_point(field_release:google.protobuf.Mixin.root)
-  
-  return root_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-}
-void Mixin::set_allocated_root(::std::string* root) {
-  if (root != NULL) {
-    
-  } else {
-    
-  }
-  root_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), root);
-  // @@protoc_insertion_point(field_set_allocated:google.protobuf.Mixin.root)
+  return ::protobuf_google_2fprotobuf_2fapi_2eproto::file_level_metadata[kIndexInFileMessages];
 }
 
-#endif  // PROTOBUF_INLINE_NOT_IN_HEADERS
 
 // @@protoc_insertion_point(namespace_scope)
-
 }  // namespace protobuf
 }  // namespace google
 

src/google/protobuf/api.pb.h

@@ -32,6 +32,30 @@
 #include <google/protobuf/source_context.pb.h>
 #include <google/protobuf/type.pb.h>
 // @@protoc_insertion_point(includes)
+
+namespace protobuf_google_2fprotobuf_2fapi_2eproto {
+// Internal implementation detail -- do not use these members.
+struct LIBPROTOBUF_EXPORT TableStruct {
+  static const ::google::protobuf::internal::ParseTableField entries[];
+  static const ::google::protobuf::internal::AuxillaryParseTableField aux[];
+  static const ::google::protobuf::internal::ParseTable schema[3];
+  static const ::google::protobuf::internal::FieldMetadata field_metadata[];
+  static const ::google::protobuf::internal::SerializationTable serialization_table[];
+  static const ::google::protobuf::uint32 offsets[];
+};
+void LIBPROTOBUF_EXPORT AddDescriptors();
+void LIBPROTOBUF_EXPORT InitDefaultsApiImpl();
+void LIBPROTOBUF_EXPORT InitDefaultsApi();
+void LIBPROTOBUF_EXPORT InitDefaultsMethodImpl();
+void LIBPROTOBUF_EXPORT InitDefaultsMethod();
+void LIBPROTOBUF_EXPORT InitDefaultsMixinImpl();
+void LIBPROTOBUF_EXPORT InitDefaultsMixin();
+inline void LIBPROTOBUF_EXPORT InitDefaults() {
+  InitDefaultsApi();
+  InitDefaultsMethod();
+  InitDefaultsMixin();
+}
+}  // namespace protobuf_google_2fprotobuf_2fapi_2eproto
 namespace google {
 namespace protobuf {
 class Api;
@@ -45,25 +69,9 @@ class MixinDefaultTypeInternal;
 LIBPROTOBUF_EXPORT extern MixinDefaultTypeInternal _Mixin_default_instance_;
 }  // namespace protobuf
 }  // namespace google
-
 namespace google {
 namespace protobuf {
 
-namespace protobuf_google_2fprotobuf_2fapi_2eproto {
-// Internal implementation detail -- do not call these.
-struct LIBPROTOBUF_EXPORT TableStruct {
-  static const ::google::protobuf::internal::ParseTableField entries[];
-  static const ::google::protobuf::internal::AuxillaryParseTableField aux[];
-  static const ::google::protobuf::internal::ParseTable schema[];
-  static const ::google::protobuf::uint32 offsets[];
-  static const ::google::protobuf::internal::FieldMetadata field_metadata[];
-  static const ::google::protobuf::internal::SerializationTable serialization_table[];
-  static void InitDefaultsImpl();
-};
-void LIBPROTOBUF_EXPORT AddDescriptors();
-void LIBPROTOBUF_EXPORT InitDefaults();
-}  // namespace protobuf_google_2fprotobuf_2fapi_2eproto
-
 // ===================================================================
 
 class LIBPROTOBUF_EXPORT Api : public ::google::protobuf::Message /* @@protoc_insertion_point(class_definition:google.protobuf.Api) */ {
@@ -95,6 +103,7 @@ class LIBPROTOBUF_EXPORT Api : public ::google::protobuf::Message /* @@protoc_in
   static const ::google::protobuf::Descriptor* descriptor();
   static const Api& default_instance();
 
+  static void InitAsDefaultInstance();  // FOR INTERNAL USE ONLY
   static inline const Api* internal_default_instance() {
     return reinterpret_cast<const Api*>(
                &_Api_default_instance_);
@@ -238,7 +247,8 @@ class LIBPROTOBUF_EXPORT Api : public ::google::protobuf::Message /* @@protoc_in
   ::google::protobuf::SourceContext* source_context_;
   int syntax_;
   mutable int _cached_size_;
-  friend struct protobuf_google_2fprotobuf_2fapi_2eproto::TableStruct;
+  friend struct ::protobuf_google_2fprotobuf_2fapi_2eproto::TableStruct;
+  friend void ::protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaultsApiImpl();
 };
 // -------------------------------------------------------------------
 
@@ -271,6 +281,7 @@ class LIBPROTOBUF_EXPORT Method : public ::google::protobuf::Message /* @@protoc
   static const ::google::protobuf::Descriptor* descriptor();
   static const Method& default_instance();
 
+  static void InitAsDefaultInstance();  // FOR INTERNAL USE ONLY
   static inline const Method* internal_default_instance() {
     return reinterpret_cast<const Method*>(
                &_Method_default_instance_);
@@ -407,7 +418,8 @@ class LIBPROTOBUF_EXPORT Method : public ::google::protobuf::Message /* @@protoc
   bool response_streaming_;
   int syntax_;
   mutable int _cached_size_;
-  friend struct protobuf_google_2fprotobuf_2fapi_2eproto::TableStruct;
+  friend struct ::protobuf_google_2fprotobuf_2fapi_2eproto::TableStruct;
+  friend void ::protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaultsMethodImpl();
 };
 // -------------------------------------------------------------------
 
@@ -440,6 +452,7 @@ class LIBPROTOBUF_EXPORT Mixin : public ::google::protobuf::Message /* @@protoc_
   static const ::google::protobuf::Descriptor* descriptor();
   static const Mixin& default_instance();
 
+  static void InitAsDefaultInstance();  // FOR INTERNAL USE ONLY
   static inline const Mixin* internal_default_instance() {
     return reinterpret_cast<const Mixin*>(
                &_Mixin_default_instance_);
@@ -527,14 +540,14 @@ class LIBPROTOBUF_EXPORT Mixin : public ::google::protobuf::Message /* @@protoc_
   ::google::protobuf::internal::ArenaStringPtr name_;
   ::google::protobuf::internal::ArenaStringPtr root_;
   mutable int _cached_size_;
-  friend struct protobuf_google_2fprotobuf_2fapi_2eproto::TableStruct;
+  friend struct ::protobuf_google_2fprotobuf_2fapi_2eproto::TableStruct;
+  friend void ::protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaultsMixinImpl();
 };
 // ===================================================================
 
 
 // ===================================================================
 
-#if !PROTOBUF_INLINE_NOT_IN_HEADERS
 #ifdef __GNUC__
   #pragma GCC diagnostic push
   #pragma GCC diagnostic ignored "-Wstrict-aliasing"
@@ -1139,7 +1152,6 @@ inline void Mixin::set_allocated_root(::std::string* root) {
 #ifdef __GNUC__
   #pragma GCC diagnostic pop
 #endif  // __GNUC__
-#endif  // !PROTOBUF_INLINE_NOT_IN_HEADERS
 // -------------------------------------------------------------------
 
 // -------------------------------------------------------------------
@@ -1147,7 +1159,6 @@ inline void Mixin::set_allocated_root(::std::string* root) {
 
 // @@protoc_insertion_point(namespace_scope)
 
-
 }  // namespace protobuf
 }  // namespace google
 

src/google/protobuf/arena.cc

@@ -38,6 +38,8 @@
 #include <sanitizer/asan_interface.h>
 #endif  // ADDRESS_SANITIZER
 
+#include <google/protobuf/stubs/port.h>
+
 namespace google {
 static const size_t kMinCleanupListElements = 8;
 static const size_t kMaxCleanupListElements = 64;  // 1kB on 64-bit.
@@ -162,7 +164,8 @@ ArenaImpl::Block* ArenaImpl::ExpandCleanupList(Block* b) {
   return b;
 }
 
-inline GOOGLE_ATTRIBUTE_ALWAYS_INLINE void ArenaImpl::AddCleanupInBlock(
+inline GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
+void ArenaImpl::AddCleanupInBlock(
     Block* b, void* elem, void (*cleanup)(void*)) {
   if (b->cleanup == NULL || b->cleanup->len == b->cleanup->size) {
     b = ExpandCleanupList(b);
@@ -194,7 +197,8 @@ void* ArenaImpl::AllocateAlignedAndAddCleanup(size_t n,
   return mem;
 }
 
-inline GOOGLE_ATTRIBUTE_ALWAYS_INLINE ArenaImpl::Block* ArenaImpl::GetBlock(size_t n) {
+inline GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
+ArenaImpl::Block* ArenaImpl::GetBlock(size_t n) {
   Block* my_block = NULL;
 
   // If this thread already owns a block in this arena then try to use that.
@@ -221,8 +225,8 @@ inline GOOGLE_ATTRIBUTE_ALWAYS_INLINE ArenaImpl::Block* ArenaImpl::GetBlock(size
   return GetBlockSlow(tc, my_block, n);
 }
 
-inline GOOGLE_ATTRIBUTE_ALWAYS_INLINE void* ArenaImpl::AllocFromBlock(Block* b,
-                                                               size_t n) {
+inline GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
+void* ArenaImpl::AllocFromBlock(Block* b, size_t n) {
   GOOGLE_DCHECK_EQ(internal::AlignUpTo8(b->pos), b->pos);  // Must be already aligned.
   GOOGLE_DCHECK_EQ(internal::AlignUpTo8(n), n);  // Must be already aligned.
   GOOGLE_DCHECK_GE(b->avail(), n);
@@ -346,6 +350,19 @@ ArenaImpl::Block* ArenaImpl::FindBlock(void* me) {
 
 }  // namespace internal
 
+void Arena::CallDestructorHooks() {
+  uint64 space_allocated = SpaceAllocated();
+  // Call the reset hook
+  if (on_arena_reset_ != NULL) {
+    on_arena_reset_(this, hooks_cookie_, space_allocated);
+  }
+
+  // Call the destruction hook
+  if (on_arena_destruction_ != NULL) {
+    on_arena_destruction_(this, hooks_cookie_, space_allocated);
+  }
+}
+
 void Arena::OnArenaAllocation(const std::type_info* allocated_type,
                               size_t n) const {
   if (on_arena_allocation_ != NULL) {

src/google/protobuf/arena.h

@@ -52,6 +52,7 @@ using type_info = ::type_info;
 #endif
 
 #include <google/protobuf/arena_impl.h>
+#include <google/protobuf/stubs/port.h>
 
 namespace google {
 namespace protobuf {
@@ -227,15 +228,8 @@ class LIBPROTOBUF_EXPORT Arena {
   Arena() : impl_(ArenaOptions()) { Init(ArenaOptions()); }
 
   ~Arena() {
-    uint64 space_allocated = SpaceAllocated();
-    // Call the reset hook
-    if (on_arena_reset_ != NULL) {
-      on_arena_reset_(this, hooks_cookie_, space_allocated);
-    }
-
-    // Call the destruction hook
-    if (on_arena_destruction_ != NULL) {
-      on_arena_destruction_(this, hooks_cookie_, space_allocated);
+    if (on_arena_reset_ != NULL || on_arena_destruction_ != NULL) {
+      CallDestructorHooks();
     }
   }
 
@@ -261,7 +255,21 @@ class LIBPROTOBUF_EXPORT Arena {
   //
   // This function also accepts any type T that satisfies the arena message
   // allocation protocol, documented above.
-  template <typename T> GOOGLE_ATTRIBUTE_ALWAYS_INLINE
+#if LANG_CXX11
+  template <typename T, typename... Args>
+  GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE static T* CreateMessage(
+      ::google::protobuf::Arena* arena, Args&&... args) {
+    static_assert(
+        InternalHelper<T>::is_arena_constructable::value,
+        "CreateMessage can only construct types that are ArenaConstructable");
+    if (arena == NULL) {
+      return new T(NULL, std::forward<Args>(args)...);
+    } else {
+      return arena->CreateMessageInternal<T>(std::forward<Args>(args)...);
+    }
+  }
+#endif
+  template <typename T> GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
   static T* CreateMessage(::google::protobuf::Arena* arena) {
 #if LANG_CXX11
     static_assert(
@@ -278,7 +286,7 @@ class LIBPROTOBUF_EXPORT Arena {
   // One-argument form of CreateMessage. This is useful for constructing objects
   // that implement the arena message construction protocol described above but
   // take additional constructor arguments.
-  template <typename T, typename Arg> GOOGLE_ATTRIBUTE_ALWAYS_INLINE
+  template <typename T, typename Arg> GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
   static T* CreateMessage(::google::protobuf::Arena* arena, const Arg& arg) {
 #if LANG_CXX11
     static_assert(
@@ -295,7 +303,8 @@ class LIBPROTOBUF_EXPORT Arena {
   // Two-argument form of CreateMessage. This is useful for constructing objects
   // that implement the arena message construction protocol described above but
   // take additional constructor arguments.
-  template <typename T, typename Arg1, typename Arg2> GOOGLE_ATTRIBUTE_ALWAYS_INLINE
+  template <typename T, typename Arg1, typename Arg2>
+  GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
   static T* CreateMessage(::google::protobuf::Arena* arena,
                           const Arg1& arg1,
                           const Arg2& arg2) {
@@ -327,7 +336,8 @@ class LIBPROTOBUF_EXPORT Arena {
   // if the object were allocated on the heap (except that the underlying memory
   // is obtained from the arena).
 #if LANG_CXX11
-  template <typename T, typename... Args> GOOGLE_ATTRIBUTE_ALWAYS_INLINE
+  template <typename T, typename... Args>
+  GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
   static T* Create(::google::protobuf::Arena* arena, Args&&... args) {
     if (arena == NULL) {
       return new T(std::forward<Args>(args)...);
@@ -337,7 +347,7 @@ class LIBPROTOBUF_EXPORT Arena {
     }
   }
 #endif
-  template <typename T> GOOGLE_ATTRIBUTE_ALWAYS_INLINE
+  template <typename T> GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
   static T* Create(::google::protobuf::Arena* arena) {
     if (arena == NULL) {
       return new T();
@@ -347,7 +357,7 @@ class LIBPROTOBUF_EXPORT Arena {
   }
 
   // Version of the above with one constructor argument for the created object.
-  template <typename T, typename Arg> GOOGLE_ATTRIBUTE_ALWAYS_INLINE
+  template <typename T, typename Arg> GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
   static T* Create(::google::protobuf::Arena* arena, const Arg& arg) {
     if (arena == NULL) {
       return new T(arg);
@@ -358,7 +368,8 @@ class LIBPROTOBUF_EXPORT Arena {
   }
 
   // Version of the above with two constructor arguments for the created object.
-  template <typename T, typename Arg1, typename Arg2> GOOGLE_ATTRIBUTE_ALWAYS_INLINE
+  template <typename T, typename Arg1, typename Arg2>
+  GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
   static T* Create(::google::protobuf::Arena* arena, const Arg1& arg1, const Arg2& arg2) {
     if (arena == NULL) {
       return new T(arg1, arg2);
@@ -371,9 +382,11 @@ class LIBPROTOBUF_EXPORT Arena {
   // Version of the above with three constructor arguments for the created
   // object.
   template <typename T, typename Arg1, typename Arg2, typename Arg3>
-  GOOGLE_ATTRIBUTE_ALWAYS_INLINE static T* Create(::google::protobuf::Arena* arena,
-                                           const Arg1& arg1, const Arg2& arg2,
-                                           const Arg3& arg3) {
+  GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
+  static T* Create(::google::protobuf::Arena* arena,
+                   const Arg1& arg1,
+                   const Arg2& arg2,
+                   const Arg3& arg3) {
     if (arena == NULL) {
       return new T(arg1, arg2, arg3);
     } else {
@@ -386,9 +399,10 @@ class LIBPROTOBUF_EXPORT Arena {
   // object.
   template <typename T, typename Arg1, typename Arg2, typename Arg3,
             typename Arg4>
-  GOOGLE_ATTRIBUTE_ALWAYS_INLINE static T* Create(::google::protobuf::Arena* arena,
-                                           const Arg1& arg1, const Arg2& arg2,
-                                           const Arg3& arg3, const Arg4& arg4) {
+  GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
+  static T* Create(::google::protobuf::Arena* arena,
+                   const Arg1& arg1, const Arg2& arg2,
+                   const Arg3& arg3, const Arg4& arg4) {
     if (arena == NULL) {
       return new T(arg1, arg2, arg3, arg4);
     } else {
@@ -401,10 +415,11 @@ class LIBPROTOBUF_EXPORT Arena {
   // object.
   template <typename T, typename Arg1, typename Arg2, typename Arg3,
             typename Arg4, typename Arg5>
-  GOOGLE_ATTRIBUTE_ALWAYS_INLINE static T* Create(::google::protobuf::Arena* arena,
-                                           const Arg1& arg1, const Arg2& arg2,
-                                           const Arg3& arg3, const Arg4& arg4,
-                                           const Arg5& arg5) {
+  GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
+  static T* Create(::google::protobuf::Arena* arena,
+                   const Arg1& arg1, const Arg2& arg2,
+                   const Arg3& arg3, const Arg4& arg4,
+                   const Arg5& arg5) {
     if (arena == NULL) {
       return new T(arg1, arg2, arg3, arg4, arg5);
     } else {
@@ -417,10 +432,11 @@ class LIBPROTOBUF_EXPORT Arena {
   // object.
   template <typename T, typename Arg1, typename Arg2, typename Arg3,
             typename Arg4, typename Arg5, typename Arg6>
-  GOOGLE_ATTRIBUTE_ALWAYS_INLINE static T* Create(::google::protobuf::Arena* arena,
-                                           const Arg1& arg1, const Arg2& arg2,
-                                           const Arg3& arg3, const Arg4& arg4,
-                                           const Arg5& arg5, const Arg6& arg6) {
+  GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
+  static T* Create(::google::protobuf::Arena* arena,
+                   const Arg1& arg1, const Arg2& arg2,
+                   const Arg3& arg3, const Arg4& arg4,
+                   const Arg5& arg5, const Arg6& arg6) {
     if (arena == NULL) {
       return new T(arg1, arg2, arg3, arg4, arg5, arg6);
     } else {
@@ -433,11 +449,12 @@ class LIBPROTOBUF_EXPORT Arena {
   // object.
   template <typename T, typename Arg1, typename Arg2, typename Arg3,
             typename Arg4, typename Arg5, typename Arg6, typename Arg7>
-  GOOGLE_ATTRIBUTE_ALWAYS_INLINE static T* Create(::google::protobuf::Arena* arena,
-                                           const Arg1& arg1, const Arg2& arg2,
-                                           const Arg3& arg3, const Arg4& arg4,
-                                           const Arg5& arg5, const Arg6& arg6,
-                                           const Arg7& arg7) {
+  GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
+  static T* Create(::google::protobuf::Arena* arena,
+                   const Arg1& arg1, const Arg2& arg2,
+                   const Arg3& arg3, const Arg4& arg4,
+                   const Arg5& arg5, const Arg6& arg6,
+                   const Arg7& arg7) {
     if (arena == NULL) {
       return new T(arg1, arg2, arg3, arg4, arg5, arg6, arg7);
     } else {
@@ -451,11 +468,12 @@ class LIBPROTOBUF_EXPORT Arena {
   template <typename T, typename Arg1, typename Arg2, typename Arg3,
             typename Arg4, typename Arg5, typename Arg6, typename Arg7,
             typename Arg8>
-  GOOGLE_ATTRIBUTE_ALWAYS_INLINE static T* Create(::google::protobuf::Arena* arena,
-                                           const Arg1& arg1, const Arg2& arg2,
-                                           const Arg3& arg3, const Arg4& arg4,
-                                           const Arg5& arg5, const Arg6& arg6,
-                                           const Arg7& arg7, const Arg8& arg8) {
+  GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
+  static T* Create(::google::protobuf::Arena* arena,
+                   const Arg1& arg1, const Arg2& arg2,
+                   const Arg3& arg3, const Arg4& arg4,
+                   const Arg5& arg5, const Arg6& arg6,
+                   const Arg7& arg7, const Arg8& arg8) {
     if (arena == NULL) {
       return new T(arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8);
     } else {
@@ -471,7 +489,7 @@ class LIBPROTOBUF_EXPORT Arena {
   // To ensure safe uses, this function checks at compile time
   // (when compiled as C++11) that T is trivially default-constructible and
   // trivially destructible.
-  template <typename T> GOOGLE_ATTRIBUTE_ALWAYS_INLINE
+  template <typename T> GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
   static T* CreateArray(::google::protobuf::Arena* arena, size_t num_elements) {
     GOOGLE_CHECK_LE(num_elements,
              std::numeric_limits<size_t>::max() / sizeof(T))
@@ -505,7 +523,7 @@ class LIBPROTOBUF_EXPORT Arena {
   // Any objects allocated on this arena are unusable after this call. It also
   // returns the total space used by the arena which is the sums of the sizes
   // of the allocated blocks. This method is not thread-safe.
-  GOOGLE_ATTRIBUTE_NOINLINE uint64 Reset() {
+  GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE uint64 Reset() {
     uint64 space_allocated = SpaceAllocated();
     // Call the reset hook
     if (on_arena_reset_ != NULL) {
@@ -516,7 +534,7 @@ class LIBPROTOBUF_EXPORT Arena {
 
   // Adds |object| to a list of heap-allocated objects to be freed with |delete|
   // when the arena is destroyed or reset.
-  template <typename T> GOOGLE_ATTRIBUTE_NOINLINE
+  template <typename T> GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE
   void Own(T* object) {
     OwnInternal(object, google::protobuf::internal::is_convertible<T*, ::google::protobuf::Message*>());
   }
@@ -526,7 +544,7 @@ class LIBPROTOBUF_EXPORT Arena {
   // that it does not free the underlying memory with |delete|; hence, it is
   // normally only used for objects that are placement-newed into
   // arena-allocated memory.
-  template <typename T> GOOGLE_ATTRIBUTE_NOINLINE
+  template <typename T> GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE
   void OwnDestructor(T* object) {
     if (object != NULL) {
       impl_.AddCleanup(object, &internal::arena_destruct_object<T>);
@@ -537,8 +555,8 @@ class LIBPROTOBUF_EXPORT Arena {
   // will be manually called when the arena is destroyed or reset. This differs
   // from OwnDestructor() in that any member function may be specified, not only
   // the class destructor.
-  GOOGLE_ATTRIBUTE_NOINLINE void OwnCustomDestructor(void* object,
-                                              void (*destruct)(void*)) {
+  GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE void OwnCustomDestructor(
+      void* object, void (*destruct)(void*)) {
     impl_.AddCleanup(object, destruct);
   }
 
@@ -546,7 +564,7 @@ class LIBPROTOBUF_EXPORT Arena {
   // message, or NULL otherwise. This differs from value->GetArena() in that the
   // latter is a virtual call, while this method is a templated call that
   // resolves at compile-time.
-  template<typename T> GOOGLE_ATTRIBUTE_ALWAYS_INLINE
+  template<typename T> GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
   static ::google::protobuf::Arena* GetArena(const T* value) {
     return GetArenaInternal(value, is_arena_constructable<T>());
   }
@@ -612,6 +630,7 @@ class LIBPROTOBUF_EXPORT Arena {
   struct is_arena_constructable : InternalHelper<T>::is_arena_constructable {};
 
  private:
+  void CallDestructorHooks();
   void OnArenaAllocation(const std::type_info* allocated_type, size_t n) const;
   inline void AllocHook(const std::type_info* allocated_type, size_t n) const {
     if (GOOGLE_PREDICT_FALSE(hooks_cookie_ != NULL)) {
@@ -622,7 +641,7 @@ class LIBPROTOBUF_EXPORT Arena {
   // Allocate and also optionally call on_arena_allocation callback with the
   // allocated type info when the hooks are in place in ArenaOptions and
   // the cookie is not null.
-  template<typename T> GOOGLE_ATTRIBUTE_ALWAYS_INLINE
+  template<typename T> GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
   void* AllocateInternal(bool skip_explicit_ownership) {
     const size_t n = internal::AlignUpTo8(sizeof(T));
     AllocHook(RTTI_TYPE_ID(T), n);
@@ -640,26 +659,24 @@ class LIBPROTOBUF_EXPORT Arena {
   // as it can cause confusing API usages, and end up having double free in
   // user code. These are used only internally from LazyField and Repeated
   // fields, since they are designed to work in all mode combinations.
-  template <typename Msg>
-  GOOGLE_ATTRIBUTE_ALWAYS_INLINE static Msg* CreateMaybeMessage(Arena* arena,
-                                                         google::protobuf::internal::true_type) {
+  template <typename Msg> GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
+  static Msg* CreateMaybeMessage(Arena* arena, google::protobuf::internal::true_type) {
     return CreateMessage<Msg>(arena);
   }
 
-  template <typename T>
-  GOOGLE_ATTRIBUTE_ALWAYS_INLINE static T* CreateMaybeMessage(Arena* arena,
-                                                       google::protobuf::internal::false_type) {
+  template <typename T> GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
+  static T* CreateMaybeMessage(Arena* arena, google::protobuf::internal::false_type) {
     return Create<T>(arena);
   }
 
-  template <typename T>
-  GOOGLE_ATTRIBUTE_ALWAYS_INLINE static T* CreateMaybeMessage(Arena* arena) {
+  template <typename T> GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
+  static T* CreateMaybeMessage(Arena* arena) {
     return CreateMaybeMessage<T>(arena, is_arena_constructable<T>());
   }
 
   // Just allocate the required size for the given type assuming the
   // type has a trivial constructor.
-  template<typename T> GOOGLE_ATTRIBUTE_ALWAYS_INLINE
+  template<typename T> GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
   T* CreateInternalRawArray(size_t num_elements) {
     GOOGLE_CHECK_LE(num_elements,
              std::numeric_limits<size_t>::max() / sizeof(T))
@@ -671,84 +688,91 @@ class LIBPROTOBUF_EXPORT Arena {
   }
 
 #if LANG_CXX11
-  template <typename T, typename... Args> GOOGLE_ATTRIBUTE_ALWAYS_INLINE
+  template <typename T, typename... Args>
+  GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
   T* CreateInternal(bool skip_explicit_ownership, Args&&... args) {
     return new (AllocateInternal<T>(skip_explicit_ownership))
         T(std::forward<Args>(args)...);
   }
 #else
-  template <typename T> GOOGLE_ATTRIBUTE_ALWAYS_INLINE
+  template <typename T> GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
   T* CreateInternal(bool skip_explicit_ownership) {
     return new (AllocateInternal<T>(skip_explicit_ownership)) T();
   }
 
-  template <typename T, typename Arg> GOOGLE_ATTRIBUTE_ALWAYS_INLINE
+  template <typename T, typename Arg> GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
   T* CreateInternal(bool skip_explicit_ownership, const Arg& arg) {
     return new (AllocateInternal<T>(skip_explicit_ownership)) T(arg);
   }
 
   template <typename T, typename Arg1, typename Arg2>
-  GOOGLE_ATTRIBUTE_ALWAYS_INLINE T* CreateInternal(bool skip_explicit_ownership,
-                                            const Arg1& arg1,
-                                            const Arg2& arg2) {
+  GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
+  T* CreateInternal(bool skip_explicit_ownership,
+                    const Arg1& arg1,
+                    const Arg2& arg2) {
     return new (AllocateInternal<T>(skip_explicit_ownership)) T(arg1, arg2);
   }
 
   template <typename T, typename Arg1, typename Arg2, typename Arg3>
-  GOOGLE_ATTRIBUTE_ALWAYS_INLINE T* CreateInternal(bool skip_explicit_ownership,
-                                            const Arg1& arg1,
-                                            const Arg2& arg2,
-                                            const Arg3& arg3) {
+  GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
+  T* CreateInternal(bool skip_explicit_ownership,
+                    const Arg1& arg1,
+                    const Arg2& arg2,
+                    const Arg3& arg3) {
     return new (AllocateInternal<T>(skip_explicit_ownership))
         T(arg1, arg2, arg3);
   }
 
   template <typename T, typename Arg1, typename Arg2, typename Arg3,
             typename Arg4>
-  GOOGLE_ATTRIBUTE_ALWAYS_INLINE T* CreateInternal(bool skip_explicit_ownership,
-                                            const Arg1& arg1,
-                                            const Arg2& arg2,
-                                            const Arg3& arg3,
-                                            const Arg4& arg4) {
+  GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
+  T* CreateInternal(bool skip_explicit_ownership,
+                    const Arg1& arg1,
+                    const Arg2& arg2,
+                    const Arg3& arg3,
+                    const Arg4& arg4) {
     return new (AllocateInternal<T>(skip_explicit_ownership))
         T(arg1, arg2, arg3, arg4);
   }
 
   template <typename T, typename Arg1, typename Arg2, typename Arg3,
             typename Arg4, typename Arg5>
-  GOOGLE_ATTRIBUTE_ALWAYS_INLINE T* CreateInternal(bool skip_explicit_ownership,
-                                            const Arg1& arg1,
-                                            const Arg2& arg2,
-                                            const Arg3& arg3,
-                                            const Arg4& arg4,
-                                            const Arg5& arg5) {
+  GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
+  T* CreateInternal(bool skip_explicit_ownership,
+                    const Arg1& arg1,
+                    const Arg2& arg2,
+                    const Arg3& arg3,
+                    const Arg4& arg4,
+                    const Arg5& arg5) {
     return new (AllocateInternal<T>(skip_explicit_ownership))
         T(arg1, arg2, arg3, arg4, arg5);
   }
 
   template <typename T, typename Arg1, typename Arg2, typename Arg3,
             typename Arg4, typename Arg5, typename Arg6>
-  GOOGLE_ATTRIBUTE_ALWAYS_INLINE T* CreateInternal(bool skip_explicit_ownership,
-                                            const Arg1& arg1,
-                                            const Arg2& arg2,
-                                            const Arg3& arg3,
-                                            const Arg4& arg4,
-                                            const Arg5& arg5,
-                                            const Arg6& arg6) {
+  GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
+  T* CreateInternal(bool skip_explicit_ownership,
+                    const Arg1& arg1,
+                    const Arg2& arg2,
+                    const Arg3& arg3,
+                    const Arg4& arg4,
+                    const Arg5& arg5,
+                    const Arg6& arg6) {
     return new (AllocateInternal<T>(skip_explicit_ownership))
         T(arg1, arg2, arg3, arg4, arg5, arg6);
   }
 
   template <typename T, typename Arg1, typename Arg2, typename Arg3,
             typename Arg4, typename Arg5, typename Arg6, typename Arg7>
-  GOOGLE_ATTRIBUTE_ALWAYS_INLINE T* CreateInternal(bool skip_explicit_ownership,
-                                            const Arg1& arg1,
-                                            const Arg2& arg2,
-                                            const Arg3& arg3,
-                                            const Arg4& arg4,
-                                            const Arg5& arg5,
-                                            const Arg6& arg6,
-                                            const Arg7& arg7) {
+  GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
+  T* CreateInternal(bool skip_explicit_ownership,
+                    const Arg1& arg1,
+                    const Arg2& arg2,
+                    const Arg3& arg3,
+                    const Arg4& arg4,
+                    const Arg5& arg5,
+                    const Arg6& arg6,
+                    const Arg7& arg7) {
     return new (AllocateInternal<T>(skip_explicit_ownership))
         T(arg1, arg2, arg3, arg4, arg5, arg6, arg7);
   }
@@ -756,36 +780,46 @@ class LIBPROTOBUF_EXPORT Arena {
   template <typename T, typename Arg1, typename Arg2, typename Arg3,
             typename Arg4, typename Arg5, typename Arg6, typename Arg7,
             typename Arg8>
-  GOOGLE_ATTRIBUTE_ALWAYS_INLINE T* CreateInternal(bool skip_explicit_ownership,
-                                            const Arg1& arg1,
-                                            const Arg2& arg2,
-                                            const Arg3& arg3,
-                                            const Arg4& arg4,
-                                            const Arg5& arg5,
-                                            const Arg6& arg6,
-                                            const Arg7& arg7,
-                                            const Arg8& arg8) {
+  GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
+  T* CreateInternal(bool skip_explicit_ownership,
+                    const Arg1& arg1,
+                    const Arg2& arg2,
+                    const Arg3& arg3,
+                    const Arg4& arg4,
+                    const Arg5& arg5,
+                    const Arg6& arg6,
+                    const Arg7& arg7,
+                    const Arg8& arg8) {
     return new (AllocateInternal<T>(skip_explicit_ownership))
         T(arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8);
   }
+#endif
+#if LANG_CXX11
+  template <typename T, typename... Args>
+  GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE T* CreateMessageInternal(
+      Args&&... args) {
+    return InternalHelper<T>::Construct(
+        AllocateInternal<T>(InternalHelper<T>::is_destructor_skippable::value),
+        this, std::forward<Args>(args)...);
+  }
 #endif
   template <typename T>
-  GOOGLE_ATTRIBUTE_ALWAYS_INLINE T* CreateMessageInternal() {
+  GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE T* CreateMessageInternal() {
     return InternalHelper<T>::Construct(
         AllocateInternal<T>(InternalHelper<T>::is_destructor_skippable::value),
         this);
   }
 
-  template <typename T, typename Arg>
-  GOOGLE_ATTRIBUTE_ALWAYS_INLINE T* CreateMessageInternal(const Arg& arg) {
+  template <typename T, typename Arg> GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
+  T* CreateMessageInternal(const Arg& arg) {
     return InternalHelper<T>::Construct(
         AllocateInternal<T>(InternalHelper<T>::is_destructor_skippable::value),
         this, arg);
   }
 
   template <typename T, typename Arg1, typename Arg2>
-  GOOGLE_ATTRIBUTE_ALWAYS_INLINE T* CreateMessageInternal(const Arg1& arg1,
-                                                   const Arg2& arg2) {
+  GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
+  T* CreateMessageInternal(const Arg1& arg1, const Arg2& arg2) {
     return InternalHelper<T>::Construct(
         AllocateInternal<T>(InternalHelper<T>::is_destructor_skippable::value),
         this, arg1, arg2);
@@ -828,14 +862,14 @@ class LIBPROTOBUF_EXPORT Arena {
   // is a subtype of ::google::protobuf::Message and 'false_type' otherwise. Collapsing
   // all template instantiations to one for generic Message reduces code size,
   // using the virtual destructor instead.
-  template<typename T> GOOGLE_ATTRIBUTE_ALWAYS_INLINE
+  template<typename T> GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
   void OwnInternal(T* object, google::protobuf::internal::true_type) {
     if (object != NULL) {
       impl_.AddCleanup(object,
                        &internal::arena_delete_object< ::google::protobuf::Message>);
     }
   }
-  template<typename T> GOOGLE_ATTRIBUTE_ALWAYS_INLINE
+  template<typename T> GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
   void OwnInternal(T* object, google::protobuf::internal::false_type) {
     if (object != NULL) {
       impl_.AddCleanup(object, &internal::arena_delete_object<T>);
@@ -845,14 +879,15 @@ class LIBPROTOBUF_EXPORT Arena {
   // Implementation for GetArena(). Only message objects with
   // InternalArenaConstructable_ tags can be associated with an arena, and such
   // objects must implement a GetArenaNoVirtual() method.
-  template <typename T>
-  GOOGLE_ATTRIBUTE_ALWAYS_INLINE static ::google::protobuf::Arena* GetArenaInternal(
+  template <typename T> GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
+  static ::google::protobuf::Arena* GetArenaInternal(
       const T* value, google::protobuf::internal::true_type) {
     return InternalHelper<T>::GetArena(value);
   }
 
   template <typename T>
-  GOOGLE_ATTRIBUTE_ALWAYS_INLINE static ::google::protobuf::Arena* GetArenaInternal(
+  GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
+  static ::google::protobuf::Arena* GetArenaInternal(
       const T* /* value */, google::protobuf::internal::false_type) {
     return NULL;
   }

src/google/protobuf/arena_impl.h

@@ -42,6 +42,8 @@
 #include <google/protobuf/stubs/mutex.h>
 #include <google/protobuf/stubs/type_traits.h>
 
+#include <google/protobuf/stubs/port.h>
+
 namespace google {
 
 namespace protobuf {
@@ -131,7 +133,8 @@ class LIBPROTOBUF_EXPORT ArenaImpl {
     // aligned at a multiple of 8 bytes.
     size_t pos;
     size_t size;  // total size of the block.
-    GOOGLE_ATTRIBUTE_ALWAYS_INLINE size_t avail() const { return size - pos; }
+    GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
+    size_t avail() const { return size - pos; }
     // data follows
   };
 

src/google/protobuf/arenastring.h

@@ -33,12 +33,11 @@
 
 #include <string>
 
-#include <google/protobuf/stubs/logging.h>
+#include <google/protobuf/arena.h>
 #include <google/protobuf/stubs/common.h>
 #include <google/protobuf/stubs/fastmem.h>
-#include <google/protobuf/arena.h>
-
-
+#include <google/protobuf/stubs/logging.h>
+#include <google/protobuf/stubs/port.h>
 
 // This is the implementation of arena string fields written for the open-source
 // release. The ArenaStringPtr struct below is an internal implementation class
@@ -62,6 +61,12 @@ struct LIBPROTOBUF_EXPORT ArenaStringPtr {
     }
   }
 
+  inline void SetLite(const ::std::string* default_value,
+                      const ::std::string& value,
+                      ::google::protobuf::Arena* arena) {
+    Set(default_value, value, arena);
+  }
+
   // Basic accessors.
   inline const ::std::string& Get() const { return *ptr_; }
 
@@ -84,8 +89,9 @@ struct LIBPROTOBUF_EXPORT ArenaStringPtr {
     }
     ::std::string* released = NULL;
     if (arena != NULL) {
-      // ptr_ is owned by the arena -- we need to return a copy.
-      released = new ::std::string(*ptr_);
+      // ptr_ is owned by the arena.
+      released = new ::std::string;
+      released->swap(*ptr_);
     } else {
       released = ptr_;
     }
@@ -143,7 +149,7 @@ struct LIBPROTOBUF_EXPORT ArenaStringPtr {
   // Swaps internal pointers. Arena-safety semantics: this is guarded by the
   // logic in Swap()/UnsafeArenaSwap() at the message level, so this method is
   // 'unsafe' if called directly.
-  GOOGLE_ATTRIBUTE_ALWAYS_INLINE void Swap(ArenaStringPtr* other) {
+  GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE void Swap(ArenaStringPtr* other) {
     std::swap(ptr_, other->ptr_);
   }
 
@@ -291,15 +297,17 @@ struct LIBPROTOBUF_EXPORT ArenaStringPtr {
  private:
   ::std::string* ptr_;
 
-  GOOGLE_ATTRIBUTE_NOINLINE void CreateInstance(::google::protobuf::Arena* arena,
-                                         const ::std::string* initial_value) {
+  GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE
+  void CreateInstance(::google::protobuf::Arena* arena,
+                      const ::std::string* initial_value) {
     GOOGLE_DCHECK(initial_value != NULL);
     ptr_ = new ::std::string(*initial_value);
     if (arena != NULL) {
       arena->Own(ptr_);
     }
   }
-  GOOGLE_ATTRIBUTE_NOINLINE void CreateInstanceNoArena(const ::std::string* initial_value) {
+  GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE
+  void CreateInstanceNoArena(const ::std::string* initial_value) {
     GOOGLE_DCHECK(initial_value != NULL);
     ptr_ = new ::std::string(*initial_value);
   }

src/google/protobuf/compiler/code_generator.h

@@ -51,6 +51,7 @@ class FileDescriptor;
 
 namespace compiler {
 class AccessInfoMap;
+
 class Version;
 
 // Defined in this file.
@@ -165,7 +166,7 @@ typedef GeneratorContext OutputDirectory;
 //   "foo=bar,baz,qux=corge"
 // parses to the pairs:
 //   ("foo", "bar"), ("baz", ""), ("qux", "corge")
-LIBPROTOC_EXPORT void ParseGeneratorParameter(
+void ParseGeneratorParameter(
     const string&, std::vector<std::pair<string, string> >*);
 
 }  // namespace compiler

src/google/protobuf/compiler/command_line_interface.cc

@@ -69,9 +69,6 @@
 #include <google/protobuf/stubs/common.h>
 #include <google/protobuf/stubs/logging.h>
 #include <google/protobuf/stubs/stringprintf.h>
-#include <google/protobuf/compiler/importer.h>
-#include <google/protobuf/compiler/code_generator.h>
-#include <google/protobuf/compiler/plugin.pb.h>
 #include <google/protobuf/compiler/subprocess.h>
 #include <google/protobuf/compiler/zip_writer.h>
 #include <google/protobuf/compiler/plugin.pb.h>
@@ -83,16 +80,11 @@
 #include <google/protobuf/descriptor.h>
 #include <google/protobuf/dynamic_message.h>
 #include <google/protobuf/text_format.h>
-#include <google/protobuf/dynamic_message.h>
-#include <google/protobuf/io/coded_stream.h>
-#include <google/protobuf/io/zero_copy_stream_impl.h>
-#include <google/protobuf/io/printer.h>
-#include <google/protobuf/stubs/io_win32.h>
-#include <google/protobuf/stubs/logging.h>
 #include <google/protobuf/stubs/strutil.h>
 #include <google/protobuf/stubs/substitute.h>
 #include <google/protobuf/stubs/map_util.h>
 #include <google/protobuf/stubs/stl_util.h>
+#include <google/protobuf/stubs/io_win32.h>
 
 
 namespace google {
@@ -232,7 +224,7 @@ bool IsInstalledProtoPath(const string& path) {
   return access(file_path.c_str(), F_OK) != -1;
 }
 
-// Add the paths where google/protobuf/descritor.proto and other well-known
+// Add the paths where google/protobuf/descriptor.proto and other well-known
 // type protos are installed.
 void AddDefaultProtoPaths(vector<pair<string, string> >* paths) {
   // TODO(xiaofeng): The code currently only checks relative paths of where
@@ -876,6 +868,7 @@ int CommandLineInterface::Run(int argc, const char* const argv[]) {
     return 1;
   }
 
+
   // We construct a separate GeneratorContext for each output location.  Note
   // that two code generators may output to the same location, in which case
   // they should share a single GeneratorContext so that OpenForInsert() works.
@@ -1019,7 +1012,8 @@ bool CommandLineInterface::PopulateSimpleDescriptorDatabase(
     bool parsed = file_descriptor_set.ParseFromFileDescriptor(fd);
     if (close(fd) != 0) {
       std::cerr << descriptor_set_in_names_[i] << ": close: "
-                << strerror(errno);
+                << strerror(errno)
+                << std::endl;
       return false;
     }
 
@@ -1172,6 +1166,21 @@ bool CommandLineInterface::MakeInputsBeProtoPathRelative(
   return true;
 }
 
+bool CommandLineInterface::ExpandArgumentFile(const string& file,
+                                              std::vector<string>* arguments) {
+  // The argument file is searched in the working directory only. We don't
+  // use the proto import path here.
+  std::ifstream file_stream(file.c_str());
+  if (!file_stream.is_open()) {
+    return false;
+  }
+  string argument;
+  // We don't support any kind of shell expansion right now.
+  while (std::getline(file_stream, argument)) {
+    arguments->push_back(argument);
+  }
+  return true;
+}
 
 CommandLineInterface::ParseArgumentStatus
 CommandLineInterface::ParseArguments(int argc, const char* const argv[]) {
@@ -1179,11 +1188,19 @@ CommandLineInterface::ParseArguments(int argc, const char* const argv[]) {
 
   std::vector<string> arguments;
   for (int i = 1; i < argc; ++i) {
+    if (argv[i][0] == '@') {
+      if (!ExpandArgumentFile(argv[i] + 1, &arguments)) {
+        std::cerr << "Failed to open argument file: " << (argv[i] + 1)
+                  << std::endl;
+        return PARSE_ARGUMENT_FAIL;
+      }
+      continue;
+    }
     arguments.push_back(argv[i]);
   }
 
   // if no arguments are given, show help
-  if(arguments.empty()) {
+  if (arguments.empty()) {
     PrintHelpText();
     return PARSE_ARGUMENT_DONE_AND_EXIT;  // Exit without running compiler.
   }
@@ -1749,6 +1766,20 @@ void CommandLineInterface::PrintHelpText() {
               << string(19 - iter->first.size(), ' ')  // Spaces for alignment.
               << iter->second.help_text << std::endl;
   }
+  std::cerr <<
+"  @<filename>                 Read options and filenames from file. If a\n"
+"                              relative file path is specified, the file\n"
+"                              will be searched in the working directory.\n"
+"                              The --proto_path option will not affect how\n"
+"                              this argument file is searched. Content of\n"
+"                              the file will be expanded in the position of\n"
+"                              @<filename> as in the argument list. Note\n"
+"                              that shell expansion is not applied to the\n"
+"                              content of the file (i.e., you cannot use\n"
+"                              quotes, wildcards, escapes, commands, etc.).\n"
+"                              Each line corresponds to a single argument,\n"
+"                              even if it contains spaces."
+      << std::endl;
 }
 
 bool CommandLineInterface::GenerateOutput(

src/google/protobuf/compiler/command_line_interface.h

@@ -227,6 +227,9 @@ class LIBPROTOC_EXPORT CommandLineInterface {
   // Parse all command-line arguments.
   ParseArgumentStatus ParseArguments(int argc, const char* const argv[]);
 
+  // Read an argument file and append the file's content to the list of
+  // arguments. Return false if the file cannot be read.
+  bool ExpandArgumentFile(const string& file, std::vector<string>* arguments);
 
   // Parses a command-line argument into a name/value pair.  Returns
   // true if the next argument in the argv should be used as the value,

src/google/protobuf/compiler/command_line_interface_unittest.cc

@@ -35,6 +35,7 @@
 #include <fcntl.h>
 #include <sys/stat.h>
 #include <sys/types.h>
+
 #ifndef _MSC_VER
 #include <unistd.h>
 #endif
@@ -84,6 +85,7 @@ using google::protobuf::internal::win32::write;
 // which case tcmalloc will print warnings that fail the plugin tests.
 #if !GOOGLE_PROTOBUF_HEAP_CHECK_DRACONIAN
 
+
 namespace {
 
 bool FileExists(const string& path) {
@@ -1559,6 +1561,36 @@ TEST_F(CommandLineInterfaceTest, WriteDependencyManifestFileForAbsolutePath) {
 }
 #endif  // !_WIN32
 
+TEST_F(CommandLineInterfaceTest, TestArgumentFile) {
+  // Test parsing multiple input files using an argument file.
+
+  CreateTempFile("foo.proto",
+    "syntax = \"proto2\";\n"
+    "message Foo {}\n");
+  CreateTempFile("bar.proto",
+    "syntax = \"proto2\";\n"
+    "message Bar {}\n");
+  CreateTempFile("arguments.txt",
+                 "--test_out=$tmpdir\n"
+                 "--plug_out=$tmpdir\n"
+                 "--proto_path=$tmpdir\n"
+                 "--direct_dependencies_violation_msg=%s is not imported\n"
+                 "foo.proto\n"
+                 "bar.proto");
+
+  Run("protocol_compiler @$tmpdir/arguments.txt");
+
+  ExpectNoErrors();
+  ExpectGeneratedWithMultipleInputs("test_generator", "foo.proto,bar.proto",
+                                    "foo.proto", "Foo");
+  ExpectGeneratedWithMultipleInputs("test_generator", "foo.proto,bar.proto",
+                                    "bar.proto", "Bar");
+  ExpectGeneratedWithMultipleInputs("test_plugin", "foo.proto,bar.proto",
+                                    "foo.proto", "Foo");
+  ExpectGeneratedWithMultipleInputs("test_plugin", "foo.proto,bar.proto",
+                                    "bar.proto", "Bar");
+}
+
 
 // -------------------------------------------------------------------
 

src/google/protobuf/compiler/cpp/cpp_enum.cc

@@ -197,6 +197,8 @@ void EnumGenerator::GenerateSymbolImports(io::Printer* printer) {
   vars["nested_name"] = descriptor_->name();
   vars["classname"] = classname_;
   vars["constexpr"] = options_.proto_h ? "constexpr " : "";
+  vars["{"] = "";
+  vars["}"] = "";
   printer->Print(vars, "typedef $classname$ $nested_name$;\n");
 
   for (int j = 0; j < descriptor_->value_count(); j++) {
@@ -204,22 +206,27 @@ void EnumGenerator::GenerateSymbolImports(io::Printer* printer) {
     vars["deprecated_attr"] = descriptor_->value(j)->options().deprecated() ?
       "GOOGLE_PROTOBUF_DEPRECATED_ATTR " : "";
     printer->Print(vars,
-      "$deprecated_attr$static $constexpr$const $nested_name$ $tag$ =\n"
+      "$deprecated_attr$static $constexpr$const $nested_name$ ${$$tag$$}$ =\n"
       "  $classname$_$tag$;\n");
+    printer->Annotate("{", "}", descriptor_->value(j));
   }
 
   printer->Print(vars,
     "static inline bool $nested_name$_IsValid(int value) {\n"
     "  return $classname$_IsValid(value);\n"
     "}\n"
-    "static const $nested_name$ $nested_name$_MIN =\n"
-    "  $classname$_$nested_name$_MIN;\n"
-    "static const $nested_name$ $nested_name$_MAX =\n"
+    "static const $nested_name$ ${$$nested_name$_MIN$}$ =\n"
+    "  $classname$_$nested_name$_MIN;\n");
+  printer->Annotate("{", "}", descriptor_);
+  printer->Print(vars,
+    "static const $nested_name$ ${$$nested_name$_MAX$}$ =\n"
     "  $classname$_$nested_name$_MAX;\n");
+  printer->Annotate("{", "}", descriptor_);
   if (generate_array_size_) {
     printer->Print(vars,
-      "static const int $nested_name$_ARRAYSIZE =\n"
+      "static const int ${$$nested_name$_ARRAYSIZE$}$ =\n"
       "  $classname$_$nested_name$_ARRAYSIZE;\n");
+    printer->Annotate("{", "}", descriptor_);
   }
 
   if (HasDescriptorMethods(descriptor_->file(), options_)) {
@@ -242,27 +249,10 @@ void EnumGenerator::GenerateSymbolImports(io::Printer* printer) {
   }
 }
 
-void EnumGenerator::GenerateDescriptorInitializer(io::Printer* printer) {
-  std::map<string, string> vars;
-  vars["index"] = SimpleItoa(descriptor_->index());
-  vars["index_in_metadata"] = SimpleItoa(index_in_metadata_);
-
-  if (descriptor_->containing_type() == NULL) {
-    printer->Print(vars,
-                   "file_level_enum_descriptors[$index_in_metadata$] = "
-                   "file->enum_type($index$);\n");
-  } else {
-    vars["parent"] = ClassName(descriptor_->containing_type(), false);
-    printer->Print(vars,
-                   "file_level_enum_descriptors[$index_in_metadata$] = "
-                   "$parent$_descriptor->enum_type($index$);\n");
-  }
-}
-
-void EnumGenerator::GenerateMethods(io::Printer* printer) {
+void EnumGenerator::GenerateMethods(int idx, io::Printer* printer) {
   std::map<string, string> vars;
   vars["classname"] = classname_;
-  vars["index_in_metadata"] = SimpleItoa(index_in_metadata_);
+  vars["index_in_metadata"] = SimpleItoa(idx);
   vars["constexpr"] = options_.proto_h ? "constexpr " : "";
   vars["file_namespace"] = FileLevelNamespace(descriptor_->file()->name());
 

src/google/protobuf/compiler/cpp/cpp_enum.h

@@ -86,13 +86,10 @@ class EnumGenerator {
 
   // Source file stuff.
 
-  // Generate code that initializes the global variable storing the enum's
-  // descriptor.
-  void GenerateDescriptorInitializer(io::Printer* printer);
-
   // Generate non-inline methods related to the enum, such as IsValidValue().
-  // Goes in the .cc file.
-  void GenerateMethods(io::Printer* printer);
+  // Goes in the .cc file. EnumDescriptors are stored in an array, idx is
+  // the index in this array that corresponds with this enum.
+  void GenerateMethods(int idx, io::Printer* printer);
 
  private:
   const EnumDescriptor* descriptor_;
@@ -101,8 +98,6 @@ class EnumGenerator {
   // whether to generate the *_ARRAYSIZE constant.
   const bool generate_array_size_;
 
-  int index_in_metadata_;
-
   friend class FileGenerator;
   GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(EnumGenerator);
 };

src/google/protobuf/compiler/cpp/cpp_enum_field.cc

@@ -232,9 +232,8 @@ GenerateSwappingCode(io::Printer* printer) const {
 
 void EnumOneofFieldGenerator::
 GenerateConstructorCode(io::Printer* printer) const {
-  printer->Print(
-      variables_,
-      "_$classname$_default_instance_.$name$_ = $default$;\n");
+  printer->Print(variables_,
+                 "$ns$::_$classname$_default_instance_.$name$_ = $default$;\n");
 }
 
 // ===================================================================

src/google/protobuf/compiler/cpp/cpp_extension.cc

@@ -167,46 +167,6 @@ void ExtensionGenerator::GenerateDefinition(io::Printer* printer) {
     "  $name$($constant_name$, $default$);\n");
 }
 
-void ExtensionGenerator::GenerateRegistration(io::Printer* printer) {
-  std::map<string, string> vars;
-  vars["extendee"   ] = ExtendeeClassName(descriptor_);
-  vars["number"     ] = SimpleItoa(descriptor_->number());
-  vars["field_type" ] = SimpleItoa(static_cast<int>(descriptor_->type()));
-  vars["is_repeated"] = descriptor_->is_repeated() ? "true" : "false";
-  vars["is_packed"  ] = (descriptor_->is_repeated() &&
-                         descriptor_->options().packed())
-                        ? "true" : "false";
-
-  switch (descriptor_->cpp_type()) {
-    case FieldDescriptor::CPPTYPE_ENUM:
-      printer->Print(
-          vars,
-          "::google::protobuf::internal::ExtensionSet::RegisterEnumExtension(\n"
-          "  $extendee$::internal_default_instance(),\n"
-          "  $number$, $field_type$, $is_repeated$, $is_packed$,\n");
-      printer->Print(
-        "  &$type$_IsValid);\n",
-        "type", ClassName(descriptor_->enum_type(), true));
-      break;
-    case FieldDescriptor::CPPTYPE_MESSAGE:
-      printer->Print(
-          vars,
-          "::google::protobuf::internal::ExtensionSet::RegisterMessageExtension(\n"
-          "  $extendee$::internal_default_instance(),\n"
-          "  $number$, $field_type$, $is_repeated$, $is_packed$,\n");
-      printer->Print("  $type$::internal_default_instance());\n", "type",
-                     ClassName(descriptor_->message_type(), true));
-      break;
-    default:
-      printer->Print(
-          vars,
-          "::google::protobuf::internal::ExtensionSet::RegisterExtension(\n"
-          "  $extendee$::internal_default_instance(),\n"
-          "  $number$, $field_type$, $is_repeated$, $is_packed$);\n");
-      break;
-  }
-}
-
 }  // namespace cpp
 }  // namespace compiler
 }  // namespace protobuf

src/google/protobuf/compiler/cpp/cpp_extension.h

@@ -67,9 +67,6 @@ class ExtensionGenerator {
   // Source file stuff.
   void GenerateDefinition(io::Printer* printer);
 
-  // Generate code to register the extension.
-  void GenerateRegistration(io::Printer* printer);
-
  private:
   const FieldDescriptor* descriptor_;
   string type_traits_;

src/google/protobuf/compiler/cpp/cpp_field.cc

@@ -61,6 +61,7 @@ using internal::WireFormat;
 void SetCommonFieldVariables(const FieldDescriptor* descriptor,
                              std::map<string, string>* variables,
                              const Options& options) {
+  (*variables)["ns"] = Namespace(descriptor);
   (*variables)["name"] = FieldName(descriptor);
   (*variables)["index"] = SimpleItoa(descriptor->index());
   (*variables)["number"] = SimpleItoa(descriptor->number());

src/google/protobuf/compiler/cpp/cpp_file.h

@@ -35,10 +35,12 @@
 #ifndef GOOGLE_PROTOBUF_COMPILER_CPP_FILE_H__
 #define GOOGLE_PROTOBUF_COMPILER_CPP_FILE_H__
 
+#include <algorithm>
 #include <memory>
 #ifndef _SHARED_PTR_H
 #include <google/protobuf/stubs/shared_ptr.h>
 #endif
+#include <set>
 #include <string>
 #include <vector>
 #include <google/protobuf/stubs/common.h>
@@ -82,16 +84,21 @@ class FileGenerator {
                         const string& info_path);
   void GenerateSource(io::Printer* printer);
 
+  int NumMessages() const { return message_generators_.size(); }
+  // Similar to GenerateSource but generates only one message
+  void GenerateSourceForMessage(int idx, io::Printer* printer);
+  void GenerateGlobalSource(io::Printer* printer);
+
  private:
   // Internal type used by GenerateForwardDeclarations (defined in file.cc).
   class ForwardDeclarations;
 
-  // Generate the BuildDescriptors() procedure, which builds all descriptors
-  // for types defined in the file.
-  void GenerateBuildDescriptors(io::Printer* printer);
+  void GenerateSourceIncludes(io::Printer* printer);
+  void GenerateSourceDefaultInstance(int idx, io::Printer* printer);
 
-  void GenerateNamespaceOpeners(io::Printer* printer);
-  void GenerateNamespaceClosers(io::Printer* printer);
+  void GenerateInitForSCC(const SCC* scc, io::Printer* printer);
+  void GenerateInitializationCode(io::Printer* printer);
+  void GenerateReflectionInitializationCode(io::Printer* printer);
 
   // For other imports, generates their forward-declarations.
   void GenerateForwardDeclarations(io::Printer* printer);
@@ -143,11 +150,23 @@ class FileGenerator {
   // a breaking change so we prefer the #undef approach.
   void GenerateMacroUndefs(io::Printer* printer);
 
+  bool IsSCCRepresentative(const Descriptor* d) {
+    return GetSCCRepresentative(d) == d;
+  }
+  const Descriptor* GetSCCRepresentative(const Descriptor* d) {
+    return GetSCC(d)->GetRepresentative();
+  }
+  const SCC* GetSCC(const Descriptor* d) {
+    return scc_analyzer_.GetSCC(d);
+  }
+
+
   const FileDescriptor* file_;
   const Options options_;
 
   SCCAnalyzer scc_analyzer_;
 
+
   // Contains the post-order walk of all the messages (and child messages) in
   // this file. If you need a pre-order walk just reverse iterate.
   std::vector<MessageGenerator*> message_generators_;
@@ -155,10 +174,8 @@ class FileGenerator {
   std::vector<ServiceGenerator*> service_generators_;
   std::vector<ExtensionGenerator*> extension_generators_;
 
-  // These members are just for owning (and thus proper deleting). Some of the
-  // message_ and enum_generators above are owned by child messages.
-  google::protobuf::scoped_array<google::protobuf::scoped_ptr<MessageGenerator> >
-      message_generators_owner_;
+  // These members are just for owning (and thus proper deleting).
+  // Nested (enum/extension)_generators are owned by child messages.
   google::protobuf::scoped_array<google::protobuf::scoped_ptr<EnumGenerator> > enum_generators_owner_;
   google::protobuf::scoped_array<google::protobuf::scoped_ptr<ServiceGenerator> >
       service_generators_owner_;

src/google/protobuf/compiler/cpp/cpp_generator.cc

@@ -46,6 +46,7 @@
 #include <google/protobuf/io/printer.h>
 #include <google/protobuf/io/zero_copy_stream.h>
 #include <google/protobuf/descriptor.pb.h>
+#include <google/protobuf/stubs/strutil.h>
 
 namespace google {
 namespace protobuf {
@@ -84,7 +85,7 @@ bool CppGenerator::Generate(const FileDescriptor* file,
   // __declspec(dllimport) depending on what is being compiled.
   //
   Options file_options;
-
+  bool split_source = false;
   for (int i = 0; i < options.size(); i++) {
     if (options[i].first == "dllexport_decl") {
       file_options.dllexport_decl = options[i].second;
@@ -98,10 +99,14 @@ bool CppGenerator::Generate(const FileDescriptor* file,
       file_options.annotation_guard_name = options[i].second;
     } else if (options[i].first == "lite") {
       file_options.enforce_lite = true;
+    } else if (options[i].first == "lite_implicit_weak_fields") {
+      file_options.lite_implicit_weak_fields = true;
     } else if (options[i].first == "table_driven_parsing") {
       file_options.table_driven_parsing = true;
     } else if (options[i].first == "table_driven_serialization") {
       file_options.table_driven_serialization = true;
+    } else if (options[i].first == "split_source") {
+      split_source = true;
     } else {
       *error = "Unknown generator option: " + options[i].first;
       return false;
@@ -135,14 +140,13 @@ bool CppGenerator::Generate(const FileDescriptor* file,
     }
   }
 
-  basename.append(".pb");
   {
     google::protobuf::scoped_ptr<io::ZeroCopyOutputStream> output(
-        generator_context->Open(basename + ".h"));
+        generator_context->Open(basename + ".pb.h"));
     GeneratedCodeInfo annotations;
     io::AnnotationProtoCollector<GeneratedCodeInfo> annotation_collector(
         &annotations);
-    string info_path = basename + ".h.meta";
+    string info_path = basename + ".pb.h.meta";
     io::Printer printer(output.get(), '$', file_options.annotate_headers
                                                ? &annotation_collector
                                                : NULL);
@@ -156,9 +160,24 @@ bool CppGenerator::Generate(const FileDescriptor* file,
   }
 
   // Generate cc file.
-  {
+  if (split_source) {
+    {
+      // This is the global .cc file, containing enum/services/tables/reflection
+      google::protobuf::scoped_ptr<io::ZeroCopyOutputStream> output(
+          generator_context->Open(basename + ".pb.cc"));
+      io::Printer printer(output.get(), '$');
+      file_generator.GenerateGlobalSource(&printer);
+    }
+    for (int i = 0; i < file_generator.NumMessages(); i++) {
+      // TODO(gerbens) Agree on naming scheme.
+      google::protobuf::scoped_ptr<io::ZeroCopyOutputStream> output(
+          generator_context->Open(basename + "." + SimpleItoa(i) + ".cc"));
+      io::Printer printer(output.get(), '$');
+      file_generator.GenerateSourceForMessage(i, &printer);
+    }
+  } else {
     google::protobuf::scoped_ptr<io::ZeroCopyOutputStream> output(
-        generator_context->Open(basename + ".cc"));
+        generator_context->Open(basename + ".pb.cc"));
     io::Printer printer(output.get(), '$');
     file_generator.GenerateSource(&printer);
   }

src/google/protobuf/compiler/cpp/cpp_helpers.cc

@@ -32,10 +32,11 @@
 //  Based on original Protocol Buffers design by
 //  Sanjay Ghemawat, Jeff Dean, and others.
 
+#include <google/protobuf/stubs/hash.h>
 #include <limits>
 #include <map>
+#include <queue>
 #include <vector>
-#include <google/protobuf/stubs/hash.h>
 
 #include <google/protobuf/stubs/logging.h>
 #include <google/protobuf/stubs/common.h>
@@ -105,6 +106,30 @@ bool HasExtension(const Descriptor* descriptor) {
   return false;
 }
 
+// Encode [0..63] as 'A'-'Z', 'a'-'z', '0'-'9', '_'
+char Base63Char(int value) {
+  GOOGLE_CHECK_GE(value, 0);
+  if (value < 26) return 'A' + value;
+  value -= 26;
+  if (value < 26) return 'a' + value;
+  value -= 26;
+  if (value < 10) return '0' + value;
+  GOOGLE_CHECK_EQ(value, 10);
+  return '_';
+}
+
+// Given a c identifier has 63 legal characters we can't implement base64
+// encoding. So we return the k least significant "digits" in base 63.
+template <typename I>
+string Base63(I n, int k) {
+  string res;
+  while (k-- > 0) {
+    res += Base63Char(static_cast<int>(n % 63));
+    n /= 63;
+  }
+  return res;
+}
+
 }  // namespace
 
 string UnderscoresToCamelCase(const string& input, bool cap_next_letter) {
@@ -137,44 +162,63 @@ const char kThickSeparator[] =
 const char kThinSeparator[] =
   "// -------------------------------------------------------------------\n";
 
-string ClassName(const Descriptor* descriptor, bool qualified) {
-
-  // Find "outer", the descriptor of the top-level message in which
-  // "descriptor" is embedded.
-  const Descriptor* outer = descriptor;
-  while (outer->containing_type() != NULL) outer = outer->containing_type();
-
-  const string& outer_name = outer->full_name();
-  string inner_name = descriptor->full_name().substr(outer_name.size());
-
-  if (qualified) {
-    return "::" + DotsToColons(outer_name) + DotsToUnderscores(inner_name);
-  } else {
-    return outer->name() + DotsToUnderscores(inner_name);
+bool CanInitializeByZeroing(const FieldDescriptor* field) {
+  if (field->is_repeated() || field->is_extension()) return false;
+  switch (field->cpp_type()) {
+    case FieldDescriptor::CPPTYPE_ENUM:
+      return field->default_value_enum()->number() == 0;
+    case FieldDescriptor::CPPTYPE_INT32:
+      return field->default_value_int32() == 0;
+    case FieldDescriptor::CPPTYPE_INT64:
+      return field->default_value_int64() == 0;
+    case FieldDescriptor::CPPTYPE_UINT32:
+      return field->default_value_uint32() == 0;
+    case FieldDescriptor::CPPTYPE_UINT64:
+      return field->default_value_uint64() == 0;
+    case FieldDescriptor::CPPTYPE_FLOAT:
+      return field->default_value_float() == 0;
+    case FieldDescriptor::CPPTYPE_DOUBLE:
+      return field->default_value_double() == 0;
+    case FieldDescriptor::CPPTYPE_BOOL:
+      return field->default_value_bool() == false;
+    default:
+      return false;
   }
 }
 
-string ClassName(const EnumDescriptor* enum_descriptor, bool qualified) {
+string ClassName(const Descriptor* descriptor) {
+  const Descriptor* parent = descriptor->containing_type();
+  string res;
+  if (parent) res += ClassName(parent) + "_";
+  res += descriptor->name();
+  if (IsMapEntryMessage(descriptor)) res += "_DoNotUse";
+  return res;
+}
+
+string ClassName(const EnumDescriptor* enum_descriptor) {
   if (enum_descriptor->containing_type() == NULL) {
-    if (qualified) {
-      return "::" + DotsToColons(enum_descriptor->full_name());
-    } else {
-      return enum_descriptor->name();
-    }
+    return enum_descriptor->name();
   } else {
-    string result = ClassName(enum_descriptor->containing_type(), qualified);
-    result += '_';
-    result += enum_descriptor->name();
-    return result;
+    return ClassName(enum_descriptor->containing_type()) + "_" +
+           enum_descriptor->name();
   }
 }
 
+string Namespace(const string& package) {
+  if (package.empty()) return "";
+  return "::" + DotsToColons(package);
+}
+
 string DefaultInstanceName(const Descriptor* descriptor) {
   string prefix = descriptor->file()->package().empty() ? "" : "::";
   return prefix + DotsToColons(descriptor->file()->package()) + "::_" +
       ClassName(descriptor, false) + "_default_instance_";
 }
 
+string ReferenceFunctionName(const Descriptor* descriptor) {
+  return QualifiedClassName(descriptor) + "_Reference";
+}
+
 string DependentBaseClassTemplateName(const Descriptor* descriptor) {
   return ClassName(descriptor, false) + "_InternalBase";
 }
@@ -210,6 +254,30 @@ string EnumValueName(const EnumValueDescriptor* enum_value) {
   return result;
 }
 
+int EstimateAlignmentSize(const FieldDescriptor* field) {
+  if (field == NULL) return 0;
+  if (field->is_repeated()) return 8;
+  switch (field->cpp_type()) {
+    case FieldDescriptor::CPPTYPE_BOOL:
+      return 1;
+
+    case FieldDescriptor::CPPTYPE_INT32:
+    case FieldDescriptor::CPPTYPE_UINT32:
+    case FieldDescriptor::CPPTYPE_ENUM:
+    case FieldDescriptor::CPPTYPE_FLOAT:
+      return 4;
+
+    case FieldDescriptor::CPPTYPE_INT64:
+    case FieldDescriptor::CPPTYPE_UINT64:
+    case FieldDescriptor::CPPTYPE_DOUBLE:
+    case FieldDescriptor::CPPTYPE_STRING:
+    case FieldDescriptor::CPPTYPE_MESSAGE:
+      return 8;
+  }
+  GOOGLE_LOG(FATAL) << "Can't get here.";
+  return -1;  // Make compiler happy.
+}
+
 string FieldConstantName(const FieldDescriptor *field) {
   string field_name = UnderscoresToCamelCase(field->name(), true);
   string result = "k" + field_name + "FieldNumber";
@@ -484,19 +552,6 @@ string SafeFunctionName(const Descriptor* descriptor,
   return function_name;
 }
 
-bool StaticInitializersForced(const FileDescriptor* file,
-                              const Options& options) {
-  if (HasDescriptorMethods(file, options) || file->extension_count() > 0) {
-    return true;
-  }
-  for (int i = 0; i < file->message_type_count(); ++i) {
-    if (HasExtension(file->message_type(i))) {
-      return true;
-    }
-  }
-  return false;
-}
-
 
 static bool HasMapFields(const Descriptor* descriptor) {
   for (int i = 0; i < descriptor->field_count(); ++i) {
@@ -672,13 +727,11 @@ void Flatten(const Descriptor* descriptor,
 
 }  // namespace
 
-std::vector<const Descriptor*> FlattenMessagesInFile(
-    const FileDescriptor* file) {
-  std::vector<const Descriptor*> result;
+void FlattenMessagesInFile(const FileDescriptor* file,
+                           std::vector<const Descriptor*>* result) {
   for (int i = 0; i < file->message_type_count(); i++) {
-    Flatten(file->message_type(i), &result);
+    Flatten(file->message_type(i), result);
   }
-  return result;
 }
 
 bool HasWeakFields(const Descriptor* descriptor) {
@@ -689,6 +742,27 @@ bool HasWeakFields(const FileDescriptor* file) {
   return false;
 }
 
+bool UsingImplicitWeakFields(const FileDescriptor* file,
+                             const Options& options) {
+  return options.lite_implicit_weak_fields &&
+         GetOptimizeFor(file, options) == FileOptions::LITE_RUNTIME;
+}
+
+
+bool IsImplicitWeakField(const FieldDescriptor* field, const Options& options) {
+  return UsingImplicitWeakFields(field->file(), options) &&
+         field->type() == FieldDescriptor::TYPE_MESSAGE &&
+         !field->is_required() && !field->is_repeated() && !field->is_map() &&
+         field->containing_oneof() == NULL &&
+         field->message_type()->file() != field->file();
+}
+
+struct CompareDescriptors {
+  bool operator()(const Descriptor* a, const Descriptor* b) {
+    return a->full_name() < b->full_name();
+  }
+};
+
 SCCAnalyzer::NodeData SCCAnalyzer::DFS(const Descriptor* descriptor) {
   // Must not have visited already.
   GOOGLE_DCHECK_EQ(cache_.count(descriptor), 0);
@@ -728,10 +802,33 @@ SCCAnalyzer::NodeData SCCAnalyzer::DFS(const Descriptor* descriptor) {
 
       if (scc_desc == descriptor) break;
     }
+
+    // The order of descriptors is random and depends how this SCC was
+    // discovered. In-order to ensure maximum stability we sort it by name.
+    std::sort(scc->descriptors.begin(), scc->descriptors.end(),
+              CompareDescriptors());
+    AddChildren(scc);
   }
   return result;
 }
 
+void SCCAnalyzer::AddChildren(SCC* scc) {
+  std::set<const SCC*> seen;
+  for (int i = 0; i < scc->descriptors.size(); i++) {
+    const Descriptor* descriptor = scc->descriptors[i];
+    for (int j = 0; j < descriptor->field_count(); j++) {
+      const Descriptor* child_msg = descriptor->field(j)->message_type();
+      if (child_msg) {
+        const SCC* child = GetSCC(child_msg);
+        if (child == scc) continue;
+        if (seen.insert(child).second) {
+          scc->children.push_back(child);
+        }
+      }
+    }
+  }
+}
+
 MessageAnalysis SCCAnalyzer::GetSCCAnalysis(const SCC* scc) {
   if (analysis_cache_.count(scc)) return analysis_cache_[scc];
   MessageAnalysis result = MessageAnalysis();
@@ -784,6 +881,46 @@ MessageAnalysis SCCAnalyzer::GetSCCAnalysis(const SCC* scc) {
   return analysis_cache_[scc] = result;
 }
 
+void ListAllFields(const Descriptor* d,
+                   std::vector<const FieldDescriptor*>* fields) {
+  // Collect sub messages
+  for (int i = 0; i < d->nested_type_count(); i++) {
+    ListAllFields(d->nested_type(i), fields);
+  }
+  // Collect message level extensions.
+  for (int i = 0; i < d->extension_count(); i++) {
+    fields->push_back(d->extension(i));
+  }
+  // Add types of fields necessary
+  for (int i = 0; i < d->field_count(); i++) {
+    fields->push_back(d->field(i));
+  }
+}
+
+void ListAllFields(const FileDescriptor* d,
+                   std::vector<const FieldDescriptor*>* fields) {
+  // Collect file level message.
+  for (int i = 0; i < d->message_type_count(); i++) {
+    ListAllFields(d->message_type(i), fields);
+  }
+  // Collect message level extensions.
+  for (int i = 0; i < d->extension_count(); i++) {
+    fields->push_back(d->extension(i));
+  }
+}
+
+void ListAllTypesForServices(const FileDescriptor* fd,
+                             std::vector<const Descriptor*>* types) {
+  for (int i = 0; i < fd->service_count(); i++) {
+    const ServiceDescriptor* sd = fd->service(i);
+    for (int j = 0; j < sd->method_count(); j++) {
+      const MethodDescriptor* method = sd->method(j);
+      types->push_back(method->input_type());
+      types->push_back(method->output_type());
+    }
+  }
+}
+
 }  // namespace cpp
 }  // namespace compiler
 }  // namespace protobuf

src/google/protobuf/compiler/cpp/cpp_helpers.h

@@ -38,16 +38,13 @@
 #include <map>
 #include <string>
 #include <google/protobuf/compiler/cpp/cpp_options.h>
+#include <google/protobuf/io/printer.h>
 #include <google/protobuf/descriptor.pb.h>
 #include <google/protobuf/descriptor.h>
+#include <google/protobuf/stubs/strutil.h>
 
 namespace google {
 namespace protobuf {
-
-namespace io {
-class Printer;
-}
-
 namespace compiler {
 namespace cpp {
 
@@ -56,6 +53,31 @@ namespace cpp {
 extern const char kThickSeparator[];
 extern const char kThinSeparator[];
 
+// Name space of the proto file. This namespace is such that the string
+// "<namespace>::some_name" is the correct fully qualified namespace.
+// This means if the package is empty the namespace is "", and otherwise
+// the namespace is "::foo::bar::...::baz" without trailing semi-colons.
+string Namespace(const string& package);
+inline string Namespace(const FileDescriptor* d) {
+  return Namespace(d->package());
+}
+template <typename Desc>
+string Namespace(const Desc* d) {
+  return Namespace(d->file());
+}
+
+// Returns true if it's safe to reset "field" to zero.
+bool CanInitializeByZeroing(const FieldDescriptor* field);
+
+string ClassName(const Descriptor* descriptor);
+string ClassName(const EnumDescriptor* enum_descriptor);
+template <typename Desc>
+string QualifiedClassName(const Desc* d) {
+  return Namespace(d) + "::" + ClassName(d);
+}
+
+// DEPRECATED just use ClassName or QualifiedClassName, a boolean is very
+// unreadable at the callsite.
 // Returns the non-nested type name for the given type.  If "qualified" is
 // true, prefix the type with the full namespace.  For example, if you had:
 //   package foo.bar;
@@ -64,12 +86,22 @@ extern const char kThinSeparator[];
 //   ::foo::bar::Baz_Qux
 // While the non-qualified version would be:
 //   Baz_Qux
-string ClassName(const Descriptor* descriptor, bool qualified);
-string ClassName(const EnumDescriptor* enum_descriptor, bool qualified);
+inline string ClassName(const Descriptor* descriptor, bool qualified) {
+  return qualified ? QualifiedClassName(descriptor) : ClassName(descriptor);
+}
+
+inline string ClassName(const EnumDescriptor* descriptor, bool qualified) {
+  return qualified ? QualifiedClassName(descriptor) : ClassName(descriptor);
+}
 
 // Fully qualified name of the default_instance of this message.
 string DefaultInstanceName(const Descriptor* descriptor);
 
+// Returns the name of a no-op function that we can call to introduce a linker
+// dependency on the given message type. This is used to implement implicit weak
+// fields.
+string ReferenceFunctionName(const Descriptor* descriptor);
+
 // Name of the CRTP class template (for use with proto_h).
 // This is a class name, like "ProtoName_InternalBase".
 string DependentBaseClassTemplateName(const Descriptor* descriptor);
@@ -92,6 +124,12 @@ string FieldName(const FieldDescriptor* field);
 // Get the sanitized name that should be used for the given enum in C++ code.
 string EnumValueName(const EnumValueDescriptor* enum_value);
 
+// Returns an estimate of the compiler's alignment for the field.  This
+// can't guarantee to be correct because the generated code could be compiled on
+// different systems with different alignment rules.  The estimates below assume
+// 64-bit pointers.
+int EstimateAlignmentSize(const FieldDescriptor* field);
+
 // Get the unqualified name that should be used for a field's field
 // number constant.
 string FieldConstantName(const FieldDescriptor *field);
@@ -150,6 +188,12 @@ string FilenameIdentifier(const string& filename);
 // For each .proto file generates a unique namespace. In this namespace global
 // definitions are put to prevent collisions.
 string FileLevelNamespace(const string& filename);
+inline string FileLevelNamespace(const FileDescriptor* file) {
+  return FileLevelNamespace(file->name());
+}
+inline string FileLevelNamespace(const Descriptor* d) {
+  return FileLevelNamespace(d->file());
+}
 
 // Return the qualified C++ name for a file level symbol.
 string QualifiedFileLevelSymbol(const string& package, const string& name);
@@ -225,10 +269,6 @@ inline bool HasFastArraySerialization(const FileDescriptor* file,
   return GetOptimizeFor(file, options) == FileOptions::SPEED;
 }
 
-// Returns whether we have to generate code with static initializers.
-bool StaticInitializersForced(const FileDescriptor* file,
-                              const Options& options);
-
 
 inline bool IsMapEntryMessage(const Descriptor* descriptor) {
   return descriptor->options().map_entry();
@@ -289,12 +329,25 @@ inline ::google::protobuf::FileOptions_OptimizeMode GetOptimizeFor(
 }
 
 // This orders the messages in a .pb.cc as it's outputted by file.cc
-std::vector<const Descriptor*> FlattenMessagesInFile(
-    const FileDescriptor* file);
+void FlattenMessagesInFile(const FileDescriptor* file,
+                           std::vector<const Descriptor*>* result);
+inline std::vector<const Descriptor*> FlattenMessagesInFile(
+    const FileDescriptor* file) {
+  std::vector<const Descriptor*> result;
+  FlattenMessagesInFile(file, &result);
+  return result;
+}
 
 bool HasWeakFields(const Descriptor* desc);
 bool HasWeakFields(const FileDescriptor* desc);
 
+// Indicates whether we should use implicit weak fields for this file.
+bool UsingImplicitWeakFields(const FileDescriptor* file,
+                             const Options& options);
+
+// Indicates whether to treat this field as implicitly weak.
+bool IsImplicitWeakField(const FieldDescriptor* field, const Options& options);
+
 // Returns true if the "required" restriction check should be ignored for the
 // given field.
 inline static bool ShouldIgnoreRequiredFieldCheck(const FieldDescriptor* field,
@@ -302,8 +355,46 @@ inline static bool ShouldIgnoreRequiredFieldCheck(const FieldDescriptor* field,
   return false;
 }
 
+class LIBPROTOC_EXPORT NamespaceOpener {
+ public:
+  explicit NamespaceOpener(io::Printer* printer) : printer_(printer) {}
+  NamespaceOpener(const string& name, io::Printer* printer)
+      : printer_(printer) {
+    ChangeTo(name);
+  }
+  ~NamespaceOpener() { ChangeTo(""); }
+
+  void ChangeTo(const string& name) {
+    std::vector<string> new_stack_ =
+        Split(name, "::", true);
+    int len = std::min(name_stack_.size(), new_stack_.size());
+    int common_idx = 0;
+    while (common_idx < len) {
+      if (name_stack_[common_idx] != new_stack_[common_idx]) break;
+      common_idx++;
+    }
+    for (int i = name_stack_.size() - 1; i >= common_idx; i--) {
+      printer_->Print("}  // namespace $ns$\n", "ns", name_stack_[i]);
+    }
+    name_stack_.swap(new_stack_);
+    for (int i = common_idx; i < name_stack_.size(); i++) {
+      printer_->Print("namespace $ns$ {\n", "ns", name_stack_[i]);
+    }
+  }
+
+ private:
+  io::Printer* printer_;
+  std::vector<string> name_stack_;
+};
+
+// Description of each strongly connected component. Note that the order
+// of both the descriptors in this SCC and the order of children is
+// deterministic.
 struct SCC {
   std::vector<const Descriptor*> descriptors;
+  std::vector<const SCC*> children;
+
+  const Descriptor* GetRepresentative() const { return descriptors[0]; }
 };
 
 struct MessageAnalysis {
@@ -357,8 +448,16 @@ class LIBPROTOC_EXPORT SCCAnalyzer {
 
   // Tarjan's Strongly Connected Components algo
   NodeData DFS(const Descriptor* descriptor);
+
+  // Add the SCC's that are children of this SCC to its children.
+  void AddChildren(SCC* scc);
 };
 
+void ListAllFields(const FileDescriptor* d,
+                   std::vector<const FieldDescriptor*>* fields);
+void ListAllTypesForServices(const FileDescriptor* fd,
+                             std::vector<const Descriptor*>* types);
+
 }  // namespace cpp
 }  // namespace compiler
 }  // namespace protobuf

src/google/protobuf/compiler/cpp/cpp_map_field.cc

@@ -114,50 +114,13 @@ MapFieldGenerator::~MapFieldGenerator() {}
 
 void MapFieldGenerator::
 GeneratePrivateMembers(io::Printer* printer) const {
-  if (HasDescriptorMethods(descriptor_->file(), options_)) {
-    printer->Print(
-        variables_,
-        "public:\n"
-        "class $map_classname$ : public "
-        "::google::protobuf::internal::MapEntry<$map_classname$, \n"
-        "    $key_cpp$, $val_cpp$,\n"
-        "    $key_wire_type$,\n"
-        "    $val_wire_type$,\n"
-        "    $default_enum_value$ > {\n"
-        "public:\n"
-        "  typedef ::google::protobuf::internal::MapEntry<$map_classname$, \n"
-        "    $key_cpp$, $val_cpp$,\n"
-        "    $key_wire_type$,\n"
-        "    $val_wire_type$,\n"
-        "    $default_enum_value$ > SuperType;\n"
-        "  $map_classname$();\n"
-        "  $map_classname$(::google::protobuf::Arena* arena);\n"
-        "  void MergeFrom(const ::google::protobuf::Message& other) PROTOBUF_FINAL;\n"
-        "  void MergeFrom(const $map_classname$& other);\n"
-        "  static const Message* internal_default_instance() { return "
-        "reinterpret_cast<const "
-        "Message*>(&_$map_classname$_default_instance_); }\n"
-        "  ::google::protobuf::Metadata GetMetadata() const;\n"
-        "};\n");
-  } else {
-    printer->Print(variables_,
-                   "public:\n"
-                   "typedef ::google::protobuf::internal::MapEntryLite<\n"
-                   "    $key_cpp$, $val_cpp$,\n"
-                   "    $key_wire_type$,\n"
-                   "    $val_wire_type$,\n"
-                   "    $default_enum_value$ >\n"
-                   "    $map_classname$;\n");
-  }
   printer->Print(variables_,
-                 "private:\n"
                  "::google::protobuf::internal::MapField$lite$<\n"
                  "    $map_classname$,\n"
                  "    $key_cpp$, $val_cpp$,\n"
                  "    $key_wire_type$,\n"
                  "    $val_wire_type$,\n"
-                 "    $default_enum_value$ > $name$_;\n"
-                 "private:\n");
+                 "    $default_enum_value$ > $name$_;\n");
 }
 
 void MapFieldGenerator::
@@ -262,7 +225,7 @@ GenerateMergeFromCodedStream(io::Printer* printer) const {
       printer->Print(variables_,
           "    unknown_fields_stream.WriteVarint32($tag$u);\n"
           "    unknown_fields_stream.WriteVarint32(\n"
-          "        static_cast<google::protobuf::uint32>(data.size()));\n"
+          "        static_cast< ::google::protobuf::uint32>(data.size()));\n"
           "    unknown_fields_stream.WriteString(data);\n");
     }
 

src/google/protobuf/compiler/cpp/cpp_message.h

@@ -43,6 +43,7 @@
 #include <string>
 #include <google/protobuf/compiler/cpp/cpp_field.h>
 #include <google/protobuf/compiler/cpp/cpp_helpers.h>
+#include <google/protobuf/compiler/cpp/cpp_message_layout_helper.h>
 #include <google/protobuf/compiler/cpp/cpp_options.h>
 
 namespace google {
@@ -62,12 +63,10 @@ class ExtensionGenerator;      // extension.h
 class MessageGenerator {
  public:
   // See generator.cc for the meaning of dllexport_decl.
-  MessageGenerator(const Descriptor* descriptor, const Options& options,
-                   SCCAnalyzer* scc_analyzer);
+  MessageGenerator(const Descriptor* descriptor, int index_in_file_messages,
+                   const Options& options, SCCAnalyzer* scc_analyzer);
   ~MessageGenerator();
 
-  // Appends the pre-order walk of the nested generators to list.
-  void Flatten(std::vector<MessageGenerator*>* list);
   // Append the two types of nested generators to the corresponding vector.
   void AddGenerators(std::vector<EnumGenerator*>* enum_generators,
                      std::vector<ExtensionGenerator*>* extension_generators);
@@ -96,8 +95,8 @@ class MessageGenerator {
   // Generate extra fields
   void GenerateExtraDefaultFields(io::Printer* printer);
 
-  // Generates code that allocates the message's default instance.
-  void GenerateDefaultInstanceAllocator(io::Printer* printer);
+  // Generates code that creates default instances for fields.
+  void GenerateFieldDefaultInstances(io::Printer* printer);
 
   // Generates code that initializes the message's default instance.  This
   // is separate from allocating because all default instances must be
@@ -208,6 +207,7 @@ class MessageGenerator {
   std::vector<uint32> RequiredFieldsBitMask() const;
 
   const Descriptor* descriptor_;
+  int index_in_file_messages_;
   string classname_;
   Options options_;
   FieldGeneratorMap field_generators_;
@@ -218,7 +218,6 @@ class MessageGenerator {
   std::vector<const FieldDescriptor *> optimized_order_;
   std::vector<int> has_bit_indices_;
   int max_has_bit_index_;
-  google::protobuf::scoped_array<google::protobuf::scoped_ptr<MessageGenerator> > nested_generators_;
   google::protobuf::scoped_array<google::protobuf::scoped_ptr<EnumGenerator> > enum_generators_;
   google::protobuf::scoped_array<google::protobuf::scoped_ptr<ExtensionGenerator> > extension_generators_;
   int num_required_fields_;
@@ -227,9 +226,10 @@ class MessageGenerator {
   // table_driven_ indicates the generated message uses table-driven parsing.
   bool table_driven_;
 
-  int index_in_file_messages_;
+  google::protobuf::scoped_ptr<MessageLayoutHelper> message_layout_helper_;
 
   SCCAnalyzer* scc_analyzer_;
+  string scc_name_;
 
   friend class FileGenerator;
   GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(MessageGenerator);

src/google/protobuf/compiler/cpp/cpp_message_field.cc

@@ -45,13 +45,42 @@ namespace cpp {
 
 namespace {
 
+// When we are generating code for implicit weak fields, we need to insert some
+// additional casts. These functions return the casted expression if
+// implicit_weak_field is true but otherwise return the original expression.
+// Ordinarily a static_cast is enough to cast google::protobuf::MessageLite* to a class
+// deriving from it, but we need a reinterpret_cast in cases where the generated
+// message is forward-declared but its full definition is not visible.
+string StaticCast(const string& type, const string& expression,
+                  bool implicit_weak_field) {
+  if (implicit_weak_field) {
+    return "static_cast< " + type + " >(" + expression + ")";
+  } else {
+    return expression;
+  }
+}
+
+string ReinterpretCast(const string& type, const string& expression,
+                       bool implicit_weak_field) {
+  if (implicit_weak_field) {
+    return "reinterpret_cast< " + type + " >(" + expression + ")";
+  } else {
+    return expression;
+  }
+}
+
 void SetMessageVariables(const FieldDescriptor* descriptor,
                          std::map<string, string>* variables,
                          const Options& options) {
   SetCommonFieldVariables(descriptor, variables, options);
   (*variables)["type"] = FieldMessageTypeName(descriptor);
+  (*variables)["casted_member"] =
+      StaticCast((*variables)["type"] + "*", (*variables)["name"] + "_",
+                      IsImplicitWeakField(descriptor, options));
   (*variables)["type_default_instance"] =
       DefaultInstanceName(descriptor->message_type());
+  (*variables)["type_reference_function"] =
+      ReferenceFunctionName(descriptor->message_type());
   if (descriptor->options().weak() || !descriptor->containing_oneof()) {
     (*variables)["non_null_ptr_to_name"] =
         StrCat("this->", (*variables)["name"], "_");
@@ -85,7 +114,8 @@ MessageFieldGenerator::MessageFieldGenerator(const FieldDescriptor* descriptor,
                                              const Options& options)
     : FieldGenerator(options),
       descriptor_(descriptor),
-      dependent_field_(options.proto_h && IsFieldDependent(descriptor)) {
+      dependent_field_(options.proto_h && IsFieldDependent(descriptor)),
+      implicit_weak_field_(IsImplicitWeakField(descriptor, options)) {
   SetMessageVariables(descriptor, &variables_, options);
 }
 
@@ -93,7 +123,11 @@ MessageFieldGenerator::~MessageFieldGenerator() {}
 
 void MessageFieldGenerator::
 GeneratePrivateMembers(io::Printer* printer) const {
-  printer->Print(variables_, "$type$* $name$_;\n");
+  if (implicit_weak_field_) {
+    printer->Print(variables_, "google::protobuf::MessageLite* $name$_;\n");
+  } else {
+    printer->Print(variables_, "$type$* $name$_;\n");
+  }
 }
 
 void MessageFieldGenerator::
@@ -108,7 +142,11 @@ GenerateDependentAccessorDeclarations(io::Printer* printer) const {
   if (!dependent_field_) {
     return;
   }
-  // Arena manipulation code is out-of-line in the derived message class.
+  // Arena manipulation code is out-of-line in the derived message class. The
+  // one exception is unsafe_arena_release_; this method has to be inline so
+  // that when the implicit weak field optimization is enabled, the method does
+  // not introduce a strong dependency on the submessage type unless the
+  // accessor actually gets called somewhere.
   printer->Print(variables_,
                  "$deprecated_attr$$type$* ${$mutable_$name$$}$();\n");
   printer->Annotate("{", "}", descriptor_);
@@ -118,14 +156,22 @@ GenerateDependentAccessorDeclarations(io::Printer* printer) const {
                  "$deprecated_attr$void ${$set_allocated_$name$$}$"
                  "($type$* $name$);\n");
   printer->Annotate("{", "}", descriptor_);
+  if (SupportsArenas(descriptor_)) {
+    printer->Print(
+        variables_,
+        "$deprecated_attr$$type$* ${$unsafe_arena_release_$name$$}$();\n");
+    printer->Annotate("{", "}", descriptor_);
+  }
 }
 
 void MessageFieldGenerator::
 GenerateAccessorDeclarations(io::Printer* printer) const {
   if (SupportsArenas(descriptor_)) {
     printer->Print(variables_,
-       "private:\n"
-       "void _slow_mutable_$name$();\n");
+       "private:\n");
+    if (!implicit_weak_field_) {
+      printer->Print(variables_, "void _slow_mutable_$name$();\n");
+    }
     if (SupportsArenas(descriptor_->message_type())) {
       printer->Print(variables_,
        "void _slow_set_allocated_$name$(\n"
@@ -135,6 +181,16 @@ GenerateAccessorDeclarations(io::Printer* printer) const {
        "$type$* _slow_$release_name$();\n"
        "public:\n");
   }
+  if (implicit_weak_field_) {
+    // These private accessors are used by MergeFrom and
+    // MergePartialFromCodedStream, and their purpose is to provide access to
+    // the field without creating a strong dependency on the message type.
+    printer->Print(variables_,
+       "private:\n"
+       "const google::protobuf::MessageLite& _internal_$name$() const;\n"
+       "google::protobuf::MessageLite* _internal_mutable_$name$();\n"
+       "public:\n");
+  }
   GenerateGetterDeclaration(printer);
   if (!dependent_field_) {
     printer->Print(variables_,
@@ -146,12 +202,14 @@ GenerateAccessorDeclarations(io::Printer* printer) const {
                    "$deprecated_attr$void ${$set_allocated_$name$$}$"
                    "($type$* $name$);\n");
     printer->Annotate("{", "}", descriptor_);
+    if (SupportsArenas(descriptor_)) {
+      printer->Print(
+          variables_,
+          "$deprecated_attr$$type$* ${$unsafe_arena_release_$name$$}$();\n");
+      printer->Annotate("{", "}", descriptor_);
+    }
   }
   if (SupportsArenas(descriptor_)) {
-    printer->Print(
-        variables_,
-        "$deprecated_attr$$type$* ${$unsafe_arena_release_$name$$}$();\n");
-    printer->Annotate("{", "}", descriptor_);
     printer->Print(variables_,
                    "$deprecated_attr$void "
                    "${$unsafe_arena_set_allocated_$name$$}$(\n"
@@ -162,9 +220,39 @@ GenerateAccessorDeclarations(io::Printer* printer) const {
 
 void MessageFieldGenerator::GenerateNonInlineAccessorDefinitions(
     io::Printer* printer) const {
-  if (SupportsArenas(descriptor_)) {
+  if (implicit_weak_field_) {
     printer->Print(variables_,
-      "void $classname$::_slow_mutable_$name$() {\n");
+      "const google::protobuf::MessageLite& $classname$::_internal_$name$() const {\n"
+      "  if ($name$_ != NULL) {\n"
+      "    return *$name$_;\n"
+      "  } else if (&$type_default_instance$ != NULL) {\n"
+      "    return *reinterpret_cast<const google::protobuf::MessageLite*>(\n"
+      "        &$type_default_instance$);\n"
+      "  } else {\n"
+      "    return *reinterpret_cast<const google::protobuf::MessageLite*>(\n"
+      "        &::google::protobuf::internal::implicit_weak_message_default_instance);\n"
+      "  }\n"
+      "}\n");
+  }
+  if (SupportsArenas(descriptor_)) {
+    if (implicit_weak_field_) {
+      printer->Print(variables_,
+        "google::protobuf::MessageLite* $classname$::_internal_mutable_$name$() {\n"
+        "  $set_hasbit$\n"
+        "  if ($name$_ == NULL) {\n"
+        "    if (&$type_default_instance$ == NULL) {\n"
+        "      $name$_ = ::google::protobuf::Arena::CreateMessage<\n"
+        "          ::google::protobuf::internal::ImplicitWeakMessage>(\n"
+        "              GetArenaNoVirtual());\n"
+        "    } else {\n"
+        "      $name$_ = reinterpret_cast<const google::protobuf::MessageLite*>(\n"
+        "          &$type_default_instance$)->New(GetArenaNoVirtual());\n"
+        "    }\n"
+        "  }\n"
+        "  return $name$_;\n");
+    } else {
+      printer->Print(variables_,
+        "void $classname$::_slow_mutable_$name$() {\n");
       if (SupportsArenas(descriptor_->message_type())) {
         printer->Print(variables_,
           "  $name$_ = ::google::protobuf::Arena::CreateMessage< $type$ >(\n"
@@ -174,23 +262,27 @@ void MessageFieldGenerator::GenerateNonInlineAccessorDefinitions(
           "  $name$_ = ::google::protobuf::Arena::Create< $type$ >(\n"
           "      GetArenaNoVirtual());\n");
       }
+    }
     printer->Print(variables_,
       "}\n"
       "$type$* $classname$::_slow_$release_name$() {\n"
       "  if ($name$_ == NULL) {\n"
       "    return NULL;\n"
-      "  } else {\n"
-      "    $type$* temp = new $type$(*$name$_);\n"
-      "    $name$_ = NULL;\n"
-      "    return temp;\n"
+      "  } else {\n");
+    if (implicit_weak_field_) {
+      printer->Print(variables_,
+        "    google::protobuf::MessageLite* temp = $name$_->New();\n"
+        "    temp->CheckTypeAndMergeFrom(*$name$_);\n");
+    } else {
+      printer->Print(variables_,
+        "    $type$* temp = new $type$(*$name$_);\n");
+    }
+    printer->Print(variables_, "    $name$_ = NULL;\n");
+    printer->Print(
+        "    return $result$;\n", "result",
+        StaticCast(variables_.at("type") + "*", "temp", implicit_weak_field_));
+    printer->Print(variables_,
       "  }\n"
-      "}\n"
-      "$type$* $classname$::unsafe_arena_release_$name$() {\n"
-      "  // @@protoc_insertion_point(field_unsafe_arena_release:$full_name$)\n"
-      "  $clear_hasbit$\n"
-      "  $type$* temp = $name$_;\n"
-      "  $name$_ = NULL;\n"
-      "  return temp;\n"
       "}\n");
     if (SupportsArenas(descriptor_->message_type())) {
       // NOTE: the same logic is mirrored in weak_message_field.cc. Any
@@ -202,12 +294,23 @@ void MessageFieldGenerator::GenerateNonInlineAccessorDefinitions(
           "        ::google::protobuf::Arena::GetArena(*$name$) == NULL) {\n"
           "      message_arena->Own(*$name$);\n"
           "    } else if (message_arena !=\n"
-          "               ::google::protobuf::Arena::GetArena(*$name$)) {\n"
-          "      $type$* new_$name$ = \n"
-          "            ::google::protobuf::Arena::CreateMessage< $type$ >(\n"
-          "            message_arena);\n"
-          "      new_$name$->CopyFrom(**$name$);\n"
-          "      *$name$ = new_$name$;\n"
+          "               ::google::protobuf::Arena::GetArena(*$name$)) {\n");
+      if (implicit_weak_field_) {
+        printer->Print(variables_,
+            "      google::protobuf::MessageLite* new_$name$ =\n"
+            "          reinterpret_cast<const google::protobuf::MessageLite*>(\n"
+            "          &$type_default_instance$)->New(GetArenaNoVirtual());\n"
+            "      new_$name$->CheckTypeAndMergeFrom(**$name$);\n"
+            "      *$name$ = static_cast< $type$* >(new_$name$);\n");
+      } else {
+        printer->Print(variables_,
+            "      $type$* new_$name$ =\n"
+            "            ::google::protobuf::Arena::CreateMessage< $type$ >(\n"
+            "            message_arena);\n"
+            "      new_$name$->CopyFrom(**$name$);\n"
+            "      *$name$ = new_$name$;\n");
+      }
+      printer->Print(variables_,
           "    }\n"
           "}\n");
     }
@@ -228,6 +331,20 @@ void MessageFieldGenerator::GenerateNonInlineAccessorDefinitions(
       "  // @@protoc_insertion_point(field_unsafe_arena_set_allocated"
       ":$full_name$)\n"
       "}\n");
+  } else if (implicit_weak_field_) {
+    printer->Print(variables_,
+        "google::protobuf::MessageLite* $classname$::_internal_mutable_$name$() {\n"
+        "  $set_hasbit$\n"
+        "  if ($name$_ == NULL) {\n"
+        "    if (&$type_default_instance$ == NULL) {\n"
+        "      $name$_ = new ::google::protobuf::internal::ImplicitWeakMessage;\n"
+        "    } else {\n"
+        "      $name$_ = reinterpret_cast<const google::protobuf::MessageLite*>(\n"
+        "          &$type_default_instance$)->New();\n"
+        "    }\n"
+        "  }\n"
+        "  return $name$_;\n"
+        "}\n");
   }
 }
 
@@ -243,6 +360,10 @@ GenerateDependentInlineAccessorDefinitions(io::Printer* printer) const {
   variables["dependent_classname"] =
       DependentBaseClassTemplateName(descriptor_->containing_type()) + "<T>";
   variables["this_message"] = DependentBaseDownCast();
+  variables["casted_reference"] =
+      ReinterpretCast(variables["dependent_typename"] + "*&",
+                      variables["this_message"] + variables["name"] + "_",
+                      implicit_weak_field_);
   if (!variables["set_hasbit"].empty()) {
     variables["set_hasbit"] =
         variables["this_message"] + variables["set_hasbit"];
@@ -255,19 +376,39 @@ GenerateDependentInlineAccessorDefinitions(io::Printer* printer) const {
   if (SupportsArenas(descriptor_)) {
     printer->Print(variables,
       "template <class T>\n"
-      "inline $type$* $dependent_classname$::mutable_$name$() {\n"
+      "inline $type$* $dependent_classname$::mutable_$name$() {\n");
+    if (implicit_weak_field_) {
+      printer->Print(variables, "  $type_reference_function$();\n");
+    }
+    printer->Print(variables,
       "  $set_hasbit$\n"
-      "  $dependent_typename$*& $name$_ = $this_message$$name$_;\n"
-      "  if ($name$_ == NULL) {\n"
-      "    $this_message$_slow_mutable_$name$();\n"
+      "  $dependent_typename$*& $name$_ = $casted_reference$;\n"
+      "  if ($name$_ == NULL) {\n");
+    if (implicit_weak_field_) {
+      if (SupportsArenas(descriptor_->message_type())) {
+        printer->Print(variables,
+            "    $name$_ = reinterpret_cast<$dependent_typename$*>(\n"
+            "        reinterpret_cast<const google::protobuf::MessageLite*>(\n"
+            "        &$type_default_instance$)->New(\n"
+            "        $this_message$GetArenaNoVirtual()));\n");
+      }
+    } else {
+      printer->Print(variables,
+        "    $this_message$_slow_mutable_$name$();\n");
+    }
+    printer->Print(variables,
       "  }\n"
       "  // @@protoc_insertion_point(field_mutable:$full_name$)\n"
       "  return $name$_;\n"
       "}\n"
       "template <class T>\n"
       "inline $type$* $dependent_classname$::$release_name$() {\n"
-      "  // @@protoc_insertion_point(field_release:$full_name$)\n"
-      "  $dependent_typename$*& $name$_ = $this_message$$name$_;\n"
+      "  // @@protoc_insertion_point(field_release:$full_name$)\n");
+    if (implicit_weak_field_) {
+      printer->Print(variables, "  $type_reference_function$();\n");
+    }
+    printer->Print(variables,
+      "  $dependent_typename$*& $name$_ = $casted_reference$;\n"
       "  $clear_hasbit$\n"
       "  if ($this_message$GetArenaNoVirtual() != NULL) {\n"
       "    return $this_message$_slow_$release_name$();\n"
@@ -281,7 +422,7 @@ GenerateDependentInlineAccessorDefinitions(io::Printer* printer) const {
       "inline void $dependent_classname$::"
       "set_allocated_$name$($type$* $name$) {\n"
       "  ::google::protobuf::Arena* message_arena = $this_message$GetArenaNoVirtual();\n"
-      "  $dependent_typename$*& $name$_ = $this_message$$name$_;\n"
+      "  $dependent_typename$*& $name$_ = $casted_reference$;\n"
       "  if (message_arena == NULL) {\n"
       "    delete $name$_;\n"
       "  }\n"
@@ -311,13 +452,27 @@ GenerateDependentInlineAccessorDefinitions(io::Printer* printer) const {
       "  }\n"
       // TODO(dlj): move insertion points to message class.
       "  // @@protoc_insertion_point(field_set_allocated:$full_name$)\n"
+      "}\n"
+      "template <class T>\n"
+      "inline $type$* $dependent_classname$::unsafe_arena_release_$name$() {\n"
+      "  // @@protoc_insertion_point("
+      "field_unsafe_arena_release:$full_name$)\n");
+    if (implicit_weak_field_) {
+      printer->Print(variables, "  $type_reference_function$();\n");
+    }
+    printer->Print(variables,
+      "  $clear_hasbit$\n"
+      "  $dependent_typename$*& $name$_ = $casted_reference$;\n"
+      "  $dependent_typename$* temp = $name$_;\n"
+      "  $name$_ = NULL;\n"
+      "  return temp;\n"
       "}\n");
   } else {
     printer->Print(variables,
       "template <class T>\n"
       "inline $type$* $dependent_classname$::mutable_$name$() {\n"
       "  $set_hasbit$\n"
-      "  $dependent_typename$*& $name$_ = $this_message$$name$_;\n"
+      "  $dependent_typename$*& $name$_ = $casted_reference$;\n"
       "  if ($name$_ == NULL) {\n"
       "    $name$_ = new $dependent_typename$;\n"
       "  }\n"
@@ -326,9 +481,13 @@ GenerateDependentInlineAccessorDefinitions(io::Printer* printer) const {
       "}\n"
       "template <class T>\n"
       "inline $type$* $dependent_classname$::$release_name$() {\n"
-      "  // @@protoc_insertion_point(field_release:$full_name$)\n"
+      "  // @@protoc_insertion_point(field_release:$full_name$)\n");
+    if (implicit_weak_field_) {
+      printer->Print(variables, "  $type_reference_function$();\n");
+    }
+    printer->Print(variables,
       "  $clear_hasbit$\n"
-      "  $dependent_typename$*& $name$_ = $this_message$$name$_;\n"
+      "  $dependent_typename$*& $name$_ = $casted_reference$;\n"
       "  $dependent_typename$* temp = $name$_;\n"
       "  $name$_ = NULL;\n"
       "  return temp;\n"
@@ -336,7 +495,7 @@ GenerateDependentInlineAccessorDefinitions(io::Printer* printer) const {
       "template <class T>\n"
       "inline void $dependent_classname$::"
       "set_allocated_$name$($type$* $name$) {\n"
-      "  $dependent_typename$*& $name$_ = $this_message$$name$_;\n"
+      "  $dependent_typename$*& $name$_ = $casted_reference$;\n"
       "  delete $name$_;\n");
 
     if (SupportsArenas(descriptor_->message_type())) {
@@ -366,9 +525,16 @@ GenerateInlineAccessorDefinitions(io::Printer* printer,
                                   bool is_inline) const {
   std::map<string, string> variables(variables_);
   variables["inline"] = is_inline ? "inline " : "";
+  variables["const_member"] = ReinterpretCast(
+      "const " + variables["type"] + "*", variables["name"] + "_",
+      implicit_weak_field_);
   printer->Print(variables,
-    "$inline$const $type$& $classname$::$name$() const {\n"
-    "  const $type$* p = $name$_;\n"
+    "$inline$const $type$& $classname$::$name$() const {\n");
+  if (implicit_weak_field_) {
+    printer->Print(variables, "  $type_reference_function$();\n");
+  }
+  printer->Print(variables,
+    "  const $type$* p = $const_member$;\n"
     "  // @@protoc_insertion_point(field_get:$full_name$)\n"
     "  return p != NULL ? *p : *reinterpret_cast<const $type$*>(\n"
     "      &$type_default_instance$);\n"
@@ -381,11 +547,18 @@ GenerateInlineAccessorDefinitions(io::Printer* printer,
       "$inline$"
       "$type$* $classname$::mutable_$name$() {\n"
       "  $set_hasbit$\n"
-      "  if ($name$_ == NULL) {\n"
-      "    _slow_mutable_$name$();\n"
+      "  if ($name$_ == NULL) {\n");
+    if (implicit_weak_field_) {
+      printer->Print(variables,
+        "    _internal_mutable_$name$();\n");
+    } else {
+      printer->Print(variables,
+        "    _slow_mutable_$name$();\n");
+    }
+    printer->Print(variables,
       "  }\n"
       "  // @@protoc_insertion_point(field_mutable:$full_name$)\n"
-      "  return $name$_;\n"
+      "  return $casted_member$;\n"
       "}\n"
       "$inline$"
       "$type$* $classname$::$release_name$() {\n"
@@ -394,7 +567,7 @@ GenerateInlineAccessorDefinitions(io::Printer* printer,
       "  if (GetArenaNoVirtual() != NULL) {\n"
       "    return _slow_$release_name$();\n"
       "  } else {\n"
-      "    $type$* temp = $name$_;\n"
+      "    $type$* temp = $casted_member$;\n"
       "    $name$_ = NULL;\n"
       "    return temp;\n"
       "  }\n"
@@ -429,6 +602,19 @@ GenerateInlineAccessorDefinitions(io::Printer* printer,
       "    $clear_hasbit$\n"
       "  }\n"
       "  // @@protoc_insertion_point(field_set_allocated:$full_name$)\n"
+      "}\n"
+      "$inline$"
+      "$type$* $classname$::unsafe_arena_release_$name$() {\n"
+      "  // @@protoc_insertion_point("
+      "field_unsafe_arena_release:$full_name$)\n");
+    if (implicit_weak_field_) {
+      printer->Print(variables, "  $type_reference_function$();\n");
+    }
+    printer->Print(variables,
+      "  $clear_hasbit$\n"
+      "  $type$* temp = $casted_member$;\n"
+      "  $name$_ = NULL;\n"
+      "  return temp;\n"
       "}\n");
   } else {
     printer->Print(variables,
@@ -439,13 +625,13 @@ GenerateInlineAccessorDefinitions(io::Printer* printer,
       "    $name$_ = new $type$;\n"
       "  }\n"
       "  // @@protoc_insertion_point(field_mutable:$full_name$)\n"
-      "  return $name$_;\n"
+      "  return $casted_member$;\n"
       "}\n"
       "$inline$"
       "$type$* $classname$::$release_name$() {\n"
       "  // @@protoc_insertion_point(field_release:$full_name$)\n"
       "  $clear_hasbit$\n"
-      "  $type$* temp = $name$_;\n"
+      "  $type$* temp = $casted_member$;\n"
       "  $name$_ = NULL;\n"
       "  return temp;\n"
       "}\n"
@@ -485,6 +671,9 @@ GenerateClearingCode(io::Printer* printer) const {
       "if ($this_message$GetArenaNoVirtual() == NULL && "
       "$this_message$$name$_ != NULL) delete $this_message$$name$_;\n"
       "$this_message$$name$_ = NULL;\n");
+  } else if (implicit_weak_field_) {
+    printer->Print(variables,
+      "if ($this_message$$name$_ != NULL) $this_message$$name$_->Clear();\n");
   } else {
     printer->Print(variables,
       "if ($this_message$$name$_ != NULL) $this_message$$name$_->"
@@ -502,6 +691,10 @@ GenerateMessageClearingCode(io::Printer* printer) const {
       "  delete $name$_;\n"
       "}\n"
       "$name$_ = NULL;\n");
+  } else if (implicit_weak_field_) {
+    printer->Print(variables_,
+      "GOOGLE_DCHECK($name$_ != NULL);\n"
+      "$name$_->Clear();\n");
   } else {
     printer->Print(variables_,
       "GOOGLE_DCHECK($name$_ != NULL);\n"
@@ -511,8 +704,14 @@ GenerateMessageClearingCode(io::Printer* printer) const {
 
 void MessageFieldGenerator::
 GenerateMergingCode(io::Printer* printer) const {
-  printer->Print(variables_,
-    "mutable_$name$()->$type$::MergeFrom(from.$name$());\n");
+  if (implicit_weak_field_) {
+    printer->Print(variables_,
+      "_internal_mutable_$name$()->CheckTypeAndMergeFrom(\n"
+      "    from._internal_$name$());\n");
+  } else {
+    printer->Print(variables_,
+      "mutable_$name$()->$type$::MergeFrom(from.$name$());\n");
+  }
 }
 
 void MessageFieldGenerator::
@@ -557,17 +756,24 @@ GenerateCopyConstructorCode(io::Printer* printer) const {
   // wasn't copied, so both of these methods allocate the submessage on the
   // heap.
 
-  printer->Print(variables_,
-    "if (from.has_$name$()) {\n"
-    "  $name$_ = new $type$(*from.$name$_);\n"
-    "} else {\n"
-    "  $name$_ = NULL;\n"
-    "}\n");
+  string new_expression = (implicit_weak_field_ ? "from.$name$_->New()"
+                                                : "new $type$(*from.$name$_)");
+  string output =
+      "if (from.has_$name$()) {\n"
+      "  $name$_ = " + new_expression + ";\n"
+      "} else {\n"
+      "  $name$_ = NULL;\n"
+      "}\n";
+  printer->Print(variables_, output.c_str());
 }
 
 void MessageFieldGenerator::
 GenerateMergeFromCodedStream(io::Printer* printer) const {
-  if (descriptor_->type() == FieldDescriptor::TYPE_MESSAGE) {
+  if (implicit_weak_field_) {
+    printer->Print(variables_,
+      "DO_(::google::protobuf::internal::WireFormatLite::ReadMessage(\n"
+      "     input, _internal_mutable_$name$()));\n");
+  } else if (descriptor_->type() == FieldDescriptor::TYPE_MESSAGE) {
     printer->Print(variables_,
       "DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual(\n"
       "     input, mutable_$name$()));\n");
@@ -595,9 +801,11 @@ GenerateSerializeWithCachedSizesToArray(io::Printer* printer) const {
 
 void MessageFieldGenerator::
 GenerateByteSize(io::Printer* printer) const {
-  printer->Print(variables_,
+  std::map<string, string> variables = variables_;
+  variables["no_virtual"] = (implicit_weak_field_ ? "" : "NoVirtual");
+  printer->Print(variables,
     "total_size += $tag_size$ +\n"
-    "  ::google::protobuf::internal::WireFormatLite::$declared_type$SizeNoVirtual(\n"
+    "  ::google::protobuf::internal::WireFormatLite::$declared_type$Size$no_virtual$(\n"
     "    *$non_null_ptr_to_name$);\n");
 }
 
@@ -771,13 +979,15 @@ void MessageOneofFieldGenerator::InternalGenerateInlineAccessorDefinitions(
       "  }\n"
       "  // @@protoc_insertion_point(field_set_allocated:$full_name$)\n"
       "}\n"
-      "$inline$ $type$* $classname$::unsafe_arena_release_$name$() {\n"
+      "$tmpl$"
+      "$inline$"
+      "$type$* $dependent_classname$::unsafe_arena_release_$name$() {\n"
       "  // @@protoc_insertion_point(field_unsafe_arena_release"
       ":$full_name$)\n"
-      "  if (has_$name$()) {\n"
-      "    clear_has_$oneof_name$();\n"
-      "    $type$* temp = $oneof_prefix$$name$_;\n"
-      "    $oneof_prefix$$name$_ = NULL;\n"
+      "  if ($this_message$has_$name$()) {\n"
+      "    $this_message$clear_has_$oneof_name$();\n"
+      "    $dependent_typename$* temp = $this_message$$oneof_prefix$$name$_;\n"
+      "    $this_message$$oneof_prefix$$name$_ = NULL;\n"
       "    return temp;\n"
       "  } else {\n"
       "    return NULL;\n"

src/google/protobuf/compiler/cpp/cpp_message_field.h

@@ -44,6 +44,8 @@ namespace protobuf {
 namespace compiler {
 namespace cpp {
 
+bool IsImplicitWeakField(const FieldDescriptor* field, const Options& options);
+
 class MessageFieldGenerator : public FieldGenerator {
  public:
   MessageFieldGenerator(const FieldDescriptor* descriptor,
@@ -78,6 +80,7 @@ class MessageFieldGenerator : public FieldGenerator {
 
   const FieldDescriptor* descriptor_;
   const bool dependent_field_;
+  const bool implicit_weak_field_;
   std::map<string, string> variables_;
 
  private:

src/google/protobuf/compiler/cpp/cpp_message_layout_helper.h

@@ -0,0 +1,61 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: seongkim@google.com (Seong Beom Kim)
+//  Based on original Protocol Buffers design by
+//  Sanjay Ghemawat, Jeff Dean, and others.
+
+#ifndef GOOGLE_PROTOBUF_COMPILER_CPP_MESSAGE_LAYOUT_HELPER_H__
+#define GOOGLE_PROTOBUF_COMPILER_CPP_MESSAGE_LAYOUT_HELPER_H__
+
+#include <google/protobuf/compiler/cpp/cpp_options.h>
+#include <google/protobuf/descriptor.h>
+
+namespace google {
+namespace protobuf {
+namespace compiler {
+namespace cpp {
+
+// Provides an abstract interface to optimize message layout
+// by rearranging the fields of a message.
+class MessageLayoutHelper {
+ public:
+  virtual ~MessageLayoutHelper() {}
+
+  virtual void OptimizeLayout(std::vector<const FieldDescriptor*>* fields,
+                              const Options& options) = 0;
+};
+
+}  // namespace cpp
+}  // namespace compiler
+}  // namespace protobuf
+
+}  // namespace google
+#endif  // GOOGLE_PROTOBUF_COMPILER_CPP_MESSAGE_LAYOUT_HELPER_H__

src/google/protobuf/compiler/cpp/cpp_options.h

@@ -39,6 +39,8 @@
 namespace google {
 namespace protobuf {
 namespace compiler {
+class AccessInfoMap;
+
 namespace cpp {
 
 // Generator options (see generator.cc for a description of each):
@@ -50,7 +52,9 @@ struct Options {
         annotate_headers(false),
         enforce_lite(false),
         table_driven_parsing(false),
-        table_driven_serialization(false) {}
+        table_driven_serialization(false),
+        lite_implicit_weak_fields(false),
+        access_info_map(NULL) {}
 
   string dllexport_decl;
   bool safe_boundary_check;
@@ -60,8 +64,10 @@ struct Options {
   bool enforce_lite;
   bool table_driven_parsing;
   bool table_driven_serialization;
+  bool lite_implicit_weak_fields;
   string annotation_pragma_name;
   string annotation_guard_name;
+  const AccessInfoMap* access_info_map;
 };
 
 }  // namespace cpp

src/google/protobuf/compiler/cpp/cpp_padding_optimizer.cc

@@ -0,0 +1,220 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <google/protobuf/compiler/cpp/cpp_padding_optimizer.h>
+
+#include <google/protobuf/compiler/cpp/cpp_helpers.h>
+
+namespace google {
+namespace protobuf {
+namespace compiler {
+namespace cpp {
+
+namespace {
+
+// FieldGroup is just a helper for PaddingOptimizer below. It holds a vector of
+// fields that are grouped together because they have compatible alignment, and
+// a preferred location in the final field ordering.
+class FieldGroup {
+ public:
+  FieldGroup() : preferred_location_(0) {}
+
+  // A group with a single field.
+  FieldGroup(float preferred_location, const FieldDescriptor* field)
+      : preferred_location_(preferred_location), fields_(1, field) {}
+
+  // Append the fields in 'other' to this group.
+  void Append(const FieldGroup& other) {
+    if (other.fields_.empty()) {
+      return;
+    }
+    // Preferred location is the average among all the fields, so we weight by
+    // the number of fields on each FieldGroup object.
+    preferred_location_ = (preferred_location_ * fields_.size() +
+                           (other.preferred_location_ * other.fields_.size())) /
+                          (fields_.size() + other.fields_.size());
+    fields_.insert(fields_.end(), other.fields_.begin(), other.fields_.end());
+  }
+
+  void SetPreferredLocation(float location) { preferred_location_ = location; }
+  const std::vector<const FieldDescriptor*>& fields() const { return fields_; }
+
+  // FieldGroup objects sort by their preferred location.
+  bool operator<(const FieldGroup& other) const {
+    return preferred_location_ < other.preferred_location_;
+  }
+
+ private:
+  // "preferred_location_" is an estimate of where this group should go in the
+  // final list of fields.  We compute this by taking the average index of each
+  // field in this group in the original ordering of fields.  This is very
+  // approximate, but should put this group close to where its member fields
+  // originally went.
+  float preferred_location_;
+  std::vector<const FieldDescriptor*> fields_;
+  // We rely on the default copy constructor and operator= so this type can be
+  // used in a vector.
+};
+
+}  // namespace
+
+// Reorder 'fields' so that if the fields are output into a c++ class in the new
+// order, fields of similar family (see below) are together and within each
+// family, alignment padding is minimized.
+//
+// We try to do this while keeping each field as close as possible to its field
+// number order so that we don't reduce cache locality much for function that
+// access each field in order.  Originally, OptimizePadding used declaration
+// order for its decisions, but generated code minus the serializer/parsers uses
+// the output of OptimizePadding as well (stored in
+// MessageGenerator::optimized_order_).  Since the serializers use field number
+// order, we use that as a tie-breaker.
+//
+// We classify each field into a particular "family" of fields, that we perform
+// the same operation on in our generated functions.
+//
+// REPEATED is placed first, as the C++ compiler automatically initializes
+// these fields in layout order.
+//
+// STRING is grouped next, as our Clear/SharedCtor/SharedDtor walks it and
+// calls ArenaStringPtr::Destroy on each.
+//
+//
+// MESSAGE is grouped next, as our Clear/SharedDtor code walks it and calls
+// delete on each.  We initialize these fields with a NULL pointer (see
+// MessageFieldGenerator::GenerateConstructorCode), which allows them to be
+// memset.
+//
+// ZERO_INITIALIZABLE is memset in Clear/SharedCtor
+//
+// OTHER these fields are initialized one-by-one.
+void PaddingOptimizer::OptimizeLayout(
+    std::vector<const FieldDescriptor*>* fields, const Options& options) {
+  // The sorted numeric order of Family determines the declaration order in the
+  // memory layout.
+  enum Family {
+    REPEATED = 0,
+    STRING = 1,
+    MESSAGE = 3,
+    ZERO_INITIALIZABLE = 4,
+    OTHER = 5,
+    kMaxFamily
+  };
+
+  // First divide fields into those that align to 1 byte, 4 bytes or 8 bytes.
+  std::vector<FieldGroup> aligned_to_1[kMaxFamily];
+  std::vector<FieldGroup> aligned_to_4[kMaxFamily];
+  std::vector<FieldGroup> aligned_to_8[kMaxFamily];
+  for (int i = 0; i < fields->size(); ++i) {
+    const FieldDescriptor* field = (*fields)[i];
+
+    Family f = OTHER;
+    if (field->is_repeated()) {
+      f = REPEATED;
+    } else if (field->cpp_type() == FieldDescriptor::CPPTYPE_STRING) {
+      f = STRING;
+    } else if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
+      f = MESSAGE;
+
+    } else if (CanInitializeByZeroing(field)) {
+      f = ZERO_INITIALIZABLE;
+    }
+
+    const int j = field->number();
+    switch (EstimateAlignmentSize(field)) {
+      case 1:
+        aligned_to_1[f].push_back(FieldGroup(j, field));
+        break;
+      case 4:
+        aligned_to_4[f].push_back(FieldGroup(j, field));
+        break;
+      case 8:
+        aligned_to_8[f].push_back(FieldGroup(j, field));
+        break;
+      default:
+        GOOGLE_LOG(FATAL) << "Unknown alignment size " << EstimateAlignmentSize(field)
+                   << "for a field " << field->full_name() << ".";
+    }
+  }
+
+  // For each family, group fields to optimize padding.
+  for (int f = 0; f < kMaxFamily; f++) {
+    // Now group fields aligned to 1 byte into sets of 4, and treat those like a
+    // single field aligned to 4 bytes.
+    for (int i = 0; i < aligned_to_1[f].size(); i += 4) {
+      FieldGroup field_group;
+      for (int j = i; j < aligned_to_1[f].size() && j < i + 4; ++j) {
+        field_group.Append(aligned_to_1[f][j]);
+      }
+      aligned_to_4[f].push_back(field_group);
+    }
+    // Sort by preferred location to keep fields as close to their field number
+    // order as possible.  Using stable_sort ensures that the output is
+    // consistent across runs.
+    std::stable_sort(aligned_to_4[f].begin(), aligned_to_4[f].end());
+
+    // Now group fields aligned to 4 bytes (or the 4-field groups created above)
+    // into pairs, and treat those like a single field aligned to 8 bytes.
+    for (int i = 0; i < aligned_to_4[f].size(); i += 2) {
+      FieldGroup field_group;
+      for (int j = i; j < aligned_to_4[f].size() && j < i + 2; ++j) {
+        field_group.Append(aligned_to_4[f][j]);
+      }
+      if (i == aligned_to_4[f].size() - 1) {
+        if (f == OTHER) {
+          // Move incomplete 4-byte block to the beginning.  This is done to
+          // pair with the (possible) leftover blocks from the
+          // ZERO_INITIALIZABLE family.
+          field_group.SetPreferredLocation(-1);
+        } else {
+          // Move incomplete 4-byte block to the end.
+          field_group.SetPreferredLocation(fields->size() + 1);
+        }
+      }
+      aligned_to_8[f].push_back(field_group);
+    }
+    // Sort by preferred location.
+    std::stable_sort(aligned_to_8[f].begin(), aligned_to_8[f].end());
+  }
+
+  // Now pull out all the FieldDescriptors in order.
+  fields->clear();
+  for (int f = 0; f < kMaxFamily; ++f) {
+    for (int i = 0; i < aligned_to_8[f].size(); ++i) {
+      fields->insert(fields->end(), aligned_to_8[f][i].fields().begin(),
+                     aligned_to_8[f][i].fields().end());
+    }
+  }
+}
+
+}  // namespace cpp
+}  // namespace compiler
+}  // namespace protobuf
+}  // namespace google

src/google/protobuf/compiler/cpp/cpp_padding_optimizer.h

@@ -0,0 +1,64 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: seongkim@google.com (Seong Beom Kim)
+//  Based on original Protocol Buffers design by
+//  Sanjay Ghemawat, Jeff Dean, and others.
+
+#ifndef GOOGLE_PROTOBUF_COMPILER_CPP_PADDING_OPTIMIZER_H__
+#define GOOGLE_PROTOBUF_COMPILER_CPP_PADDING_OPTIMIZER_H__
+
+#include <google/protobuf/compiler/cpp/cpp_message_layout_helper.h>
+
+namespace google {
+namespace protobuf {
+namespace compiler {
+namespace cpp {
+
+// Rearranges the fields of a message to minimize padding.
+// Fields are grouped by the type and the size.
+// For example, grouping four boolean fields and one int32
+// field results in zero padding overhead. See OptimizeLayout's
+// comment for details.
+class PaddingOptimizer : public MessageLayoutHelper {
+ public:
+  PaddingOptimizer() {}
+  ~PaddingOptimizer() {}
+
+  void OptimizeLayout(std::vector<const FieldDescriptor*>* fields,
+                      const Options& options);
+};
+
+}  // namespace cpp
+}  // namespace compiler
+}  // namespace protobuf
+
+}  // namespace google
+#endif  // GOOGLE_PROTOBUF_COMPILER_CPP_PADDING_OPTIMIZER_H__

src/google/protobuf/compiler/cpp/cpp_primitive_field.cc

@@ -245,9 +245,8 @@ GenerateSwappingCode(io::Printer* printer) const {
 
 void PrimitiveOneofFieldGenerator::
 GenerateConstructorCode(io::Printer* printer) const {
-  printer->Print(
-      variables_,
-      "_$classname$_default_instance_.$name$_ = $default$;\n");
+  printer->Print(variables_,
+                 "$ns$::_$classname$_default_instance_.$name$_ = $default$;\n");
 }
 
 void PrimitiveOneofFieldGenerator::
@@ -433,7 +432,7 @@ GenerateSerializeWithCachedSizesToArray(io::Printer* printer) const {
       "    ::google::protobuf::internal::WireFormatLite::WIRETYPE_LENGTH_DELIMITED,\n"
       "    target);\n"
       "  target = ::google::protobuf::io::CodedOutputStream::WriteVarint32ToArray(\n"
-      "      static_cast< ::google::protobuf::uint32>(\n"
+      "      static_cast< ::google::protobuf::int32>(\n"
       "          _$name$_cached_byte_size_), target);\n"
       "  target = ::google::protobuf::internal::WireFormatLite::\n"
       "    Write$declared_type$NoTagToArray(this->$name$_, target);\n"

src/google/protobuf/compiler/cpp/cpp_string_field.cc

@@ -58,8 +58,8 @@ void SetStringVariables(const FieldDescriptor* descriptor,
   (*variables)["default_variable"] =
       descriptor->default_value_string().empty()
           ? "&::google::protobuf::internal::GetEmptyStringAlreadyInited()"
-          : "&" + (*variables)["classname"] + "::" + default_variable_string +
-                ".get()";
+          : "&" + Namespace(descriptor) + "::" + (*variables)["classname"] +
+                "::" + default_variable_string + ".get()";
   (*variables)["pointer_type"] =
       descriptor->type() == FieldDescriptor::TYPE_BYTES ? "void" : "char";
   (*variables)["null_check"] = "GOOGLE_DCHECK(value != NULL);\n";
@@ -71,6 +71,9 @@ void SetStringVariables(const FieldDescriptor* descriptor,
   (*variables)["full_name"] = descriptor->full_name();
 
   (*variables)["string_piece"] = "::std::string";
+
+  (*variables)["lite"] =
+      HasDescriptorMethods(descriptor->file(), options) ? "" : "Lite";
 }
 
 }  // namespace
@@ -79,7 +82,8 @@ void SetStringVariables(const FieldDescriptor* descriptor,
 
 StringFieldGenerator::StringFieldGenerator(const FieldDescriptor* descriptor,
                                            const Options& options)
-    : FieldGenerator(options), descriptor_(descriptor) {
+    : FieldGenerator(options), descriptor_(descriptor),
+    lite_(!HasDescriptorMethods(descriptor->file(), options)) {
   SetStringVariables(descriptor, &variables_, options);
 }
 
@@ -207,13 +211,13 @@ GenerateInlineAccessorDefinitions(io::Printer* printer,
         "}\n"
         "$inline$void $classname$::set_$name$(const ::std::string& value) {\n"
         "  $set_hasbit$\n"
-        "  $name$_.Set($default_variable$, value, GetArenaNoVirtual());\n"
+        "  $name$_.Set$lite$($default_variable$, value, GetArenaNoVirtual());\n"
         "  // @@protoc_insertion_point(field_set:$full_name$)\n"
         "}\n"
         "#if LANG_CXX11\n"
         "$inline$void $classname$::set_$name$(::std::string&& value) {\n"
         "  $set_hasbit$\n"
-        "  $name$_.Set(\n"
+        "  $name$_.Set$lite$(\n"
         "    $default_variable$, ::std::move(value), GetArenaNoVirtual());\n"
         "  // @@protoc_insertion_point(field_set_rvalue:$full_name$)\n"
         "}\n"
@@ -221,7 +225,7 @@ GenerateInlineAccessorDefinitions(io::Printer* printer,
         "$inline$void $classname$::set_$name$(const char* value) {\n"
         "  $null_check$"
         "  $set_hasbit$\n"
-        "  $name$_.Set($default_variable$, $string_piece$(value),\n"
+        "  $name$_.Set$lite$($default_variable$, $string_piece$(value),\n"
         "              GetArenaNoVirtual());\n"
         "  // @@protoc_insertion_point(field_set_char:$full_name$)\n"
         "}\n"
@@ -229,7 +233,7 @@ GenerateInlineAccessorDefinitions(io::Printer* printer,
         "void $classname$::set_$name$(const $pointer_type$* value,\n"
         "    size_t size) {\n"
         "  $set_hasbit$\n"
-        "  $name$_.Set($default_variable$, $string_piece$(\n"
+        "  $name$_.Set$lite$($default_variable$, $string_piece$(\n"
         "      reinterpret_cast<const char*>(value), size), "
         "GetArenaNoVirtual());\n"
         "  // @@protoc_insertion_point(field_set_pointer:$full_name$)\n"
@@ -453,7 +457,7 @@ GenerateCopyConstructorCode(io::Printer* printer) const {
   if (SupportsArenas(descriptor_) || descriptor_->containing_oneof() != NULL) {
     // TODO(gpike): improve this
     printer->Print(variables_,
-      "$name$_.Set($default_variable$, from.$name$(),\n"
+      "$name$_.Set$lite$($default_variable$, from.$name$(),\n"
       "  GetArenaNoVirtual());\n");
   } else {
     printer->Print(variables_,
@@ -480,13 +484,14 @@ GenerateDestructorCode(io::Printer* printer) const {
 void StringFieldGenerator::
 GenerateDefaultInstanceAllocator(io::Printer* printer) const {
   if (!descriptor_->default_value_string().empty()) {
-    printer->Print(variables_,
-                   "$classname$::$default_variable_name$.DefaultConstruct();\n"
-                   "*$classname$::$default_variable_name$.get_mutable() = "
-                   "::std::string($default$, $default_length$);\n"
-                   "::google::protobuf::internal::OnShutdownDestroyString(\n"
-                   "    $classname$::$default_variable_name$.get_mutable());\n"
-                   );
+    printer->Print(
+        variables_,
+        "$ns$::$classname$::$default_variable_name$.DefaultConstruct();\n"
+        "*$ns$::$classname$::$default_variable_name$.get_mutable() = "
+        "::std::string($default$, $default_length$);\n"
+        "::google::protobuf::internal::OnShutdownDestroyString(\n"
+        "    $ns$::$classname$::$default_variable_name$.get_mutable());\n"
+    );
   }
 }
 
@@ -572,7 +577,7 @@ GenerateInlineAccessorDefinitions(io::Printer* printer,
         "    set_has_$name$();\n"
         "    $oneof_prefix$$name$_.UnsafeSetDefault($default_variable$);\n"
         "  }\n"
-        "  $oneof_prefix$$name$_.Set($default_variable$, value,\n"
+        "  $oneof_prefix$$name$_.Set$lite$($default_variable$, value,\n"
         "      GetArenaNoVirtual());\n"
         "  // @@protoc_insertion_point(field_set:$full_name$)\n"
         "}\n"
@@ -584,7 +589,7 @@ GenerateInlineAccessorDefinitions(io::Printer* printer,
         "    set_has_$name$();\n"
         "    $oneof_prefix$$name$_.UnsafeSetDefault($default_variable$);\n"
         "  }\n"
-        "  $oneof_prefix$$name$_.Set(\n"
+        "  $oneof_prefix$$name$_.Set$lite$(\n"
         "    $default_variable$, ::std::move(value), GetArenaNoVirtual());\n"
         "  // @@protoc_insertion_point(field_set_rvalue:$full_name$)\n"
         "}\n"
@@ -596,7 +601,7 @@ GenerateInlineAccessorDefinitions(io::Printer* printer,
         "    set_has_$name$();\n"
         "    $oneof_prefix$$name$_.UnsafeSetDefault($default_variable$);\n"
         "  }\n"
-        "  $oneof_prefix$$name$_.Set($default_variable$,\n"
+        "  $oneof_prefix$$name$_.Set$lite$($default_variable$,\n"
         "      $string_piece$(value), GetArenaNoVirtual());\n"
         "  // @@protoc_insertion_point(field_set_char:$full_name$)\n"
         "}\n"
@@ -608,7 +613,8 @@ GenerateInlineAccessorDefinitions(io::Printer* printer,
         "    set_has_$name$();\n"
         "    $oneof_prefix$$name$_.UnsafeSetDefault($default_variable$);\n"
         "  }\n"
-        "  $oneof_prefix$$name$_.Set($default_variable$, $string_piece$(\n"
+        "  $oneof_prefix$$name$_.Set$lite$(\n"
+        "      $default_variable$, $string_piece$(\n"
         "      reinterpret_cast<const char*>(value), size),\n"
         "      GetArenaNoVirtual());\n"
         "  // @@protoc_insertion_point(field_set_pointer:$full_name$)\n"
@@ -806,7 +812,7 @@ void StringOneofFieldGenerator::
 GenerateConstructorCode(io::Printer* printer) const {
   printer->Print(
       variables_,
-      "_$classname$_default_instance_.$name$_.UnsafeSetDefault(\n"
+      "$ns$::_$classname$_default_instance_.$name$_.UnsafeSetDefault(\n"
       "    $default_variable$);\n");
 }
 

src/google/protobuf/compiler/cpp/cpp_string_field.h

@@ -73,6 +73,7 @@ class StringFieldGenerator : public FieldGenerator {
  protected:
   const FieldDescriptor* descriptor_;
   std::map<string, string> variables_;
+  const bool lite_;
 
  private:
   GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(StringFieldGenerator);

src/google/protobuf/compiler/cpp/cpp_test_bad_identifiers.proto

@@ -151,6 +151,10 @@ enum ConflictingEnum {  // NO_PROTO3
 
 message DummyMessage {}
 
+// Message names that could conflict.
+message Shutdown {}
+message TableStruct {}
+
 service TestConflictingMethodNames {
   rpc Closure(DummyMessage) returns (DummyMessage);
 }

src/google/protobuf/compiler/cpp/cpp_unittest.cc

@@ -744,21 +744,6 @@ TEST(GeneratedMessageTest, NonEmptyMergeFrom) {
   TestUtil::ExpectAllFieldsSet(message1);
 }
 
-#if !defined(PROTOBUF_TEST_NO_DESCRIPTORS) || \
-    !defined(GOOGLE_PROTOBUF_NO_RTTI)
-#ifdef PROTOBUF_HAS_DEATH_TEST
-#ifndef NDEBUG
-
-TEST(GeneratedMessageTest, MergeFromSelf) {
-  unittest::TestAllTypes message;
-  EXPECT_DEATH(message.MergeFrom(message), "pb[.]cc.*Check failed:");
-  EXPECT_DEATH(message.MergeFrom(implicit_cast<const Message&>(message)),
-               "pb[.]cc.*Check failed:");
-}
-
-#endif  // NDEBUG
-#endif  // PROTOBUF_HAS_DEATH_TEST
-#endif  // !PROTOBUF_TEST_NO_DESCRIPTORS || !GOOGLE_PROTOBUF_NO_RTTI
 
 // Test the generated SerializeWithCachedSizesToArray(),
 TEST(GeneratedMessageTest, SerializationToArray) {

src/google/protobuf/compiler/importer.cc

@@ -54,13 +54,17 @@
 #include <google/protobuf/compiler/parser.h>
 #include <google/protobuf/io/tokenizer.h>
 #include <google/protobuf/io/zero_copy_stream_impl.h>
-#include <google/protobuf/stubs/io_win32.h>
 #include <google/protobuf/stubs/strutil.h>
+#include <google/protobuf/stubs/io_win32.h>
 
 #ifdef _WIN32
 #include <ctype.h>
 #endif
 
+namespace google {
+namespace protobuf {
+namespace compiler {
+
 #ifdef _MSC_VER
 // DO NOT include <io.h>, instead create functions in io_win32.{h,cc} and import
 // them like we do below.
@@ -68,10 +72,6 @@ using google::protobuf::internal::win32::access;
 using google::protobuf::internal::win32::open;
 #endif
 
-namespace google {
-namespace protobuf {
-namespace compiler {
-
 // Returns true if the text looks like a Windows-style absolute path, starting
 // with a drive letter.  Example:  "C:\foo".  TODO(kenton):  Share this with
 // copy in command_line_interface.cc?

src/google/protobuf/compiler/java/java_helpers.h

@@ -136,6 +136,13 @@ inline string ShortMutableJavaClassName(const Descriptor* descriptor) {
   return descriptor->name();
 }
 
+// Whether the given descriptor is for one of the core descriptor protos. We
+// cannot currently use the new runtime with core protos since there is a
+// bootstrapping problem with obtaining their descriptors.
+inline bool IsDescriptorProto(const Descriptor* descriptor) {
+  return descriptor->file()->name() == "google/protobuf/descriptor.proto";
+}
+
 
 // Whether we should generate multiple java files for messages.
 inline bool MultipleJavaFiles(
@@ -371,6 +378,10 @@ inline bool IsAnyMessage(const Descriptor* descriptor) {
   return descriptor->full_name() == "google.protobuf.Any";
 }
 
+inline bool IsWrappersProtoFile(const FileDescriptor* descriptor) {
+  return descriptor->name() == "google/protobuf/wrappers.proto";
+}
+
 inline bool CheckUtf8(const FieldDescriptor* descriptor) {
   return descriptor->file()->syntax() == FileDescriptor::SYNTAX_PROTO3 ||
       descriptor->file()->options().java_string_check_utf8();
@@ -379,6 +390,10 @@ inline bool CheckUtf8(const FieldDescriptor* descriptor) {
 inline string GeneratedCodeVersionSuffix() {
   return "V3";
 }
+
+inline bool EnableExperimentalRuntime(Context* context) {
+  return false;
+}
 }  // namespace java
 }  // namespace compiler
 }  // namespace protobuf

src/google/protobuf/compiler/java/java_message.cc

@@ -379,7 +379,10 @@ void ImmutableMessageGenerator::Generate(io::Printer* printer) {
     "}\n");
 
   if (context_->HasGeneratedMethods(descriptor_)) {
-    GenerateParsingConstructor(printer);
+    if (!EnableExperimentalRuntime(context_) ||
+        IsDescriptorProto(descriptor_)) {
+      GenerateParsingConstructor(printer);
+    }
   }
 
   GenerateDescriptorMethods(printer);
@@ -537,6 +540,17 @@ void ImmutableMessageGenerator::Generate(io::Printer* printer) {
       "\n",
       "classname", name_resolver_->GetImmutableClassName(descriptor_));
 
+  // 'of' method for Wrappers
+  if (IsWrappersProtoFile(descriptor_->file())) {
+    printer->Print(
+        "public static $classname$ of($field_type$ value) {\n"
+        "  return newBuilder().setValue(value).build();\n"
+        "}\n"
+        "\n",
+        "classname", name_resolver_->GetImmutableClassName(descriptor_),
+        "field_type", PrimitiveTypeName(GetJavaType(descriptor_->field(0))));
+  }
+
   GenerateParser(printer);
 
   printer->Print(
@@ -576,95 +590,103 @@ GenerateMessageSerializationMethods(io::Printer* printer) {
     "public void writeTo(com.google.protobuf.CodedOutputStream output)\n"
     "                    throws java.io.IOException {\n");
   printer->Indent();
-  if (HasPackedFields(descriptor_)) {
-    // writeTo(CodedOutputStream output) might be invoked without
-    // getSerializedSize() ever being called, but we need the memoized
-    // sizes in case this message has packed fields. Rather than emit checks for
-    // each packed field, just call getSerializedSize() up front.
-    // In most cases, getSerializedSize() will have already been called anyway
-    // by one of the wrapper writeTo() methods, making this call cheap.
-    printer->Print(
-      "getSerializedSize();\n");
-  }
 
-  if (descriptor_->extension_range_count() > 0) {
-    if (descriptor_->options().message_set_wire_format()) {
-      printer->Print(
-        "com.google.protobuf.GeneratedMessage$ver$\n"
-        "  .ExtendableMessage<$classname$>.ExtensionWriter\n"
-        "    extensionWriter = newMessageSetExtensionWriter();\n",
-        "classname", name_resolver_->GetImmutableClassName(descriptor_),
-        "ver", GeneratedCodeVersionSuffix());
-    } else {
-      printer->Print(
-        "com.google.protobuf.GeneratedMessage$ver$\n"
-        "  .ExtendableMessage<$classname$>.ExtensionWriter\n"
-        "    extensionWriter = newExtensionWriter();\n",
-        "classname", name_resolver_->GetImmutableClassName(descriptor_),
-        "ver", GeneratedCodeVersionSuffix());
+  if (EnableExperimentalRuntime(context_) && !IsDescriptorProto(descriptor_)) {
+    printer->Print("writeToInternal(output);\n");
+  } else {
+    if (HasPackedFields(descriptor_)) {
+      // writeTo(CodedOutputStream output) might be invoked without
+      // getSerializedSize() ever being called, but we need the memoized
+      // sizes in case this message has packed fields. Rather than emit checks
+      // for each packed field, just call getSerializedSize() up front. In most
+      // cases, getSerializedSize() will have already been called anyway by one
+      // of the wrapper writeTo() methods, making this call cheap.
+      printer->Print("getSerializedSize();\n");
     }
-  }
 
-  // Merge the fields and the extension ranges, both sorted by field number.
-  for (int i = 0, j = 0;
-       i < descriptor_->field_count() || j < sorted_extensions.size();
-       ) {
-    if (i == descriptor_->field_count()) {
-      GenerateSerializeOneExtensionRange(printer, sorted_extensions[j++]);
-    } else if (j == sorted_extensions.size()) {
-      GenerateSerializeOneField(printer, sorted_fields[i++]);
-    } else if (sorted_fields[i]->number() < sorted_extensions[j]->start) {
-      GenerateSerializeOneField(printer, sorted_fields[i++]);
-    } else {
-      GenerateSerializeOneExtensionRange(printer, sorted_extensions[j++]);
+    if (descriptor_->extension_range_count() > 0) {
+      if (descriptor_->options().message_set_wire_format()) {
+        printer->Print(
+            "com.google.protobuf.GeneratedMessage$ver$\n"
+            "  .ExtendableMessage<$classname$>.ExtensionWriter\n"
+            "    extensionWriter = newMessageSetExtensionWriter();\n",
+            "classname", name_resolver_->GetImmutableClassName(descriptor_),
+            "ver", GeneratedCodeVersionSuffix());
+      } else {
+        printer->Print(
+            "com.google.protobuf.GeneratedMessage$ver$\n"
+            "  .ExtendableMessage<$classname$>.ExtensionWriter\n"
+            "    extensionWriter = newExtensionWriter();\n",
+            "classname", name_resolver_->GetImmutableClassName(descriptor_),
+            "ver", GeneratedCodeVersionSuffix());
+      }
     }
-  }
 
-  if (descriptor_->options().message_set_wire_format()) {
-    printer->Print(
-        "unknownFields.writeAsMessageSetTo(output);\n");
-  } else {
-    printer->Print(
-        "unknownFields.writeTo(output);\n");
+    // Merge the fields and the extension ranges, both sorted by field number.
+    for (int i = 0, j = 0;
+         i < descriptor_->field_count() || j < sorted_extensions.size();) {
+      if (i == descriptor_->field_count()) {
+        GenerateSerializeOneExtensionRange(printer, sorted_extensions[j++]);
+      } else if (j == sorted_extensions.size()) {
+        GenerateSerializeOneField(printer, sorted_fields[i++]);
+      } else if (sorted_fields[i]->number() < sorted_extensions[j]->start) {
+        GenerateSerializeOneField(printer, sorted_fields[i++]);
+      } else {
+        GenerateSerializeOneExtensionRange(printer, sorted_extensions[j++]);
+      }
+    }
+
+    if (descriptor_->options().message_set_wire_format()) {
+      printer->Print("unknownFields.writeAsMessageSetTo(output);\n");
+    } else {
+      printer->Print("unknownFields.writeTo(output);\n");
+    }
   }
 
   printer->Outdent();
   printer->Print(
-    "}\n"
-    "\n"
-    "public int getSerializedSize() {\n"
-    "  int size = memoizedSize;\n"
-    "  if (size != -1) return size;\n"
-    "\n"
-    "  size = 0;\n");
+      "}\n"
+      "\n"
+      "public int getSerializedSize() {\n"
+      "  int size = memoizedSize;\n"
+      "  if (size != -1) return size;\n"
+      "\n");
   printer->Indent();
+  if (EnableExperimentalRuntime(context_) && !IsDescriptorProto(descriptor_)) {
+    printer->Print(
+        "memoizedSize = getSerializedSizeInternal();\n"
+        "return memoizedSize;\n");
+  } else {
 
-  for (int i = 0; i < descriptor_->field_count(); i++) {
-    field_generators_.get(sorted_fields[i]).GenerateSerializedSizeCode(printer);
-  }
+    printer->Print("size = 0;\n");
+
+    for (int i = 0; i < descriptor_->field_count(); i++) {
+      field_generators_.get(sorted_fields[i])
+          .GenerateSerializedSizeCode(printer);
+    }
+
+    if (descriptor_->extension_range_count() > 0) {
+      if (descriptor_->options().message_set_wire_format()) {
+        printer->Print("size += extensionsSerializedSizeAsMessageSet();\n");
+      } else {
+        printer->Print("size += extensionsSerializedSize();\n");
+      }
+    }
 
-  if (descriptor_->extension_range_count() > 0) {
     if (descriptor_->options().message_set_wire_format()) {
       printer->Print(
-        "size += extensionsSerializedSizeAsMessageSet();\n");
+          "size += unknownFields.getSerializedSizeAsMessageSet();\n");
     } else {
-      printer->Print(
-        "size += extensionsSerializedSize();\n");
+      printer->Print("size += unknownFields.getSerializedSize();\n");
     }
-  }
 
-  if (descriptor_->options().message_set_wire_format()) {
-    printer->Print(
-        "size += unknownFields.getSerializedSizeAsMessageSet();\n");
-  } else {
     printer->Print(
-        "size += unknownFields.getSerializedSize();\n");
+        "memoizedSize = size;\n"
+        "return size;\n");
   }
 
   printer->Outdent();
   printer->Print(
-    "  memoizedSize = size;\n"
-    "  return size;\n"
     "}\n"
     "\n");
 }
@@ -1199,7 +1221,10 @@ GenerateParsingConstructor(io::Printer* printer) {
 
   // Initialize all fields to default.
   printer->Print(
-      "this();\n");
+      "this();\n"
+      "if (extensionRegistry == null) {\n"
+      "  throw new java.lang.NullPointerException();\n"
+      "}\n");
 
   // Use builder bits to track mutable repeated fields.
   int totalBuilderBits = 0;
@@ -1335,10 +1360,18 @@ void ImmutableMessageGenerator::GenerateParser(io::Printer* printer) {
       "    com.google.protobuf.ExtensionRegistryLite extensionRegistry)\n"
       "    throws com.google.protobuf.InvalidProtocolBufferException {\n",
       "classname", descriptor_->name());
-  if (context_->HasGeneratedMethods(descriptor_)) {
+  if (EnableExperimentalRuntime(context_) && !IsDescriptorProto(descriptor_)) {
+    printer->Indent();
     printer->Print(
-        "    return new $classname$(input, extensionRegistry);\n",
+        "$classname$ msg = new $classname$();\n"
+        "msg.mergeFromInternal(input, extensionRegistry);\n"
+        "msg.makeImmutableInternal();\n"
+        "return msg;\n",
         "classname", descriptor_->name());
+    printer->Outdent();
+  } else if (context_->HasGeneratedMethods(descriptor_)) {
+    printer->Print("  return new $classname$(input, extensionRegistry);\n",
+                   "classname", descriptor_->name());
   } else {
     // When parsing constructor isn't generated, use builder to parse
     // messages. Note, will fallback to use reflection based mergeFieldFrom()

src/google/protobuf/compiler/java/java_message_lite.cc

@@ -332,17 +332,19 @@ void ImmutableMessageLiteGenerator::Generate(io::Printer* printer) {
     printer->Print("\n");
   }
 
-  GenerateMessageSerializationMethods(printer);
+  if (!EnableExperimentalRuntime(context_)) {
+    GenerateMessageSerializationMethods(printer);
+  }
 
   GenerateParseFromMethods(printer);
   GenerateBuilder(printer);
 
   if (HasRequiredFields(descriptor_)) {
     // Memoizes whether the protocol buffer is fully initialized (has all
-    // required fields). -1 means not yet computed. 0 means false and 1 means
-    // true.
+    // required fields). 0 means false, 1 means true, and all other values
+    // mean not yet computed.
     printer->Print(
-      "private byte memoizedIsInitialized = -1;\n");
+      "private byte memoizedIsInitialized = 2;\n");
   }
 
   printer->Print(
@@ -415,13 +417,33 @@ void ImmutableMessageLiteGenerator::Generate(io::Printer* printer) {
     "      }\n"
     "    }\n"
     "  }\n"
-    "  return PARSER;\n"
-    "}\n",
+    "  return PARSER;\n",
     "classname", name_resolver_->GetImmutableClassName(descriptor_));
 
   printer->Outdent();
-  printer->Outdent();
 
+  if (HasRequiredFields(descriptor_)) {
+    printer->Print(
+        "}\n"
+        "case GET_MEMOIZED_IS_INITIALIZED: {\n"
+        "  return memoizedIsInitialized;\n"
+        "}\n"
+        "case SET_MEMOIZED_IS_INITIALIZED: {\n"
+        "  memoizedIsInitialized = (byte) (arg0 == null ? 0 : 1);\n"
+        "  return null;\n"
+        "}\n");
+  } else {
+    printer->Print(
+        "}\n"
+        "case GET_MEMOIZED_IS_INITIALIZED: {\n"
+        "  return (byte) 1;\n"
+        "}\n"
+        "case SET_MEMOIZED_IS_INITIALIZED: {\n"
+        "  return null;\n"
+        "}\n");
+  }
+
+  printer->Outdent();
   printer->Print(
     "  }\n"
     "  throw new UnsupportedOperationException();\n"
@@ -455,6 +477,17 @@ void ImmutableMessageLiteGenerator::Generate(io::Printer* printer) {
       "\n",
       "classname", name_resolver_->GetImmutableClassName(descriptor_));
 
+  // 'of' method for Wrappers
+  if (IsWrappersProtoFile(descriptor_->file())) {
+    printer->Print(
+        "public static $classname$ of($field_type$ value) {\n"
+        "  return newBuilder().setValue(value).build();\n"
+        "}\n"
+        "\n",
+        "classname", name_resolver_->GetImmutableClassName(descriptor_),
+        "field_type", PrimitiveTypeName(GetJavaType(descriptor_->field(0))));
+  }
+
   GenerateParser(printer);
 
   // Extensions must be declared after the DEFAULT_INSTANCE is initialized
@@ -490,36 +523,34 @@ GenerateMessageSerializationMethods(io::Printer* printer) {
   if (HasPackedFields(descriptor_)) {
     // writeTo(CodedOutputStream output) might be invoked without
     // getSerializedSize() ever being called, but we need the memoized
-    // sizes in case this message has packed fields. Rather than emit checks for
-    // each packed field, just call getSerializedSize() up front.
-    // In most cases, getSerializedSize() will have already been called anyway
-    // by one of the wrapper writeTo() methods, making this call cheap.
-    printer->Print(
-      "getSerializedSize();\n");
+    // sizes in case this message has packed fields. Rather than emit checks
+    // for each packed field, just call getSerializedSize() up front. In most
+    // cases, getSerializedSize() will have already been called anyway by one
+    // of the wrapper writeTo() methods, making this call cheap.
+    printer->Print("getSerializedSize();\n");
   }
 
   if (descriptor_->extension_range_count() > 0) {
     if (descriptor_->options().message_set_wire_format()) {
       printer->Print(
-        "com.google.protobuf.GeneratedMessageLite\n"
-        "  .ExtendableMessage<$classname$, $classname$.Builder>\n"
-        "    .ExtensionWriter extensionWriter =\n"
-        "      newMessageSetExtensionWriter();\n",
-        "classname", name_resolver_->GetImmutableClassName(descriptor_));
+          "com.google.protobuf.GeneratedMessageLite\n"
+          "  .ExtendableMessage<$classname$, $classname$.Builder>\n"
+          "    .ExtensionWriter extensionWriter =\n"
+          "      newMessageSetExtensionWriter();\n",
+          "classname", name_resolver_->GetImmutableClassName(descriptor_));
     } else {
       printer->Print(
-        "com.google.protobuf.GeneratedMessageLite\n"
-        "  .ExtendableMessage<$classname$, $classname$.Builder>\n"
-        "    .ExtensionWriter extensionWriter =\n"
-        "      newExtensionWriter();\n",
-        "classname", name_resolver_->GetImmutableClassName(descriptor_));
+          "com.google.protobuf.GeneratedMessageLite\n"
+          "  .ExtendableMessage<$classname$, $classname$.Builder>\n"
+          "    .ExtensionWriter extensionWriter =\n"
+          "      newExtensionWriter();\n",
+          "classname", name_resolver_->GetImmutableClassName(descriptor_));
     }
   }
 
   // Merge the fields and the extension ranges, both sorted by field number.
   for (int i = 0, j = 0;
-       i < descriptor_->field_count() || j < sorted_extensions.size();
-       ) {
+       i < descriptor_->field_count() || j < sorted_extensions.size();) {
     if (i == descriptor_->field_count()) {
       GenerateSerializeOneExtensionRange(printer, sorted_extensions[j++]);
     } else if (j == sorted_extensions.size()) {
@@ -532,23 +563,22 @@ GenerateMessageSerializationMethods(io::Printer* printer) {
   }
 
   if (descriptor_->options().message_set_wire_format()) {
-    printer->Print(
-      "unknownFields.writeAsMessageSetTo(output);\n");
+    printer->Print("unknownFields.writeAsMessageSetTo(output);\n");
   } else {
-    printer->Print(
-      "unknownFields.writeTo(output);\n");
+    printer->Print("unknownFields.writeTo(output);\n");
   }
 
   printer->Outdent();
   printer->Print(
-    "}\n"
-    "\n"
-    "public int getSerializedSize() {\n"
-    "  int size = memoizedSerializedSize;\n"
-    "  if (size != -1) return size;\n"
-    "\n"
-    "  size = 0;\n");
+      "}\n"
+      "\n"
+      "public int getSerializedSize() {\n"
+      "  int size = memoizedSerializedSize;\n"
+      "  if (size != -1) return size;\n"
+      "\n");
   printer->Indent();
+  printer->Print(
+      "size = 0;\n");
 
   for (int i = 0; i < descriptor_->field_count(); i++) {
     field_generators_.get(sorted_fields[i]).GenerateSerializedSizeCode(printer);
@@ -556,26 +586,24 @@ GenerateMessageSerializationMethods(io::Printer* printer) {
 
   if (descriptor_->extension_range_count() > 0) {
     if (descriptor_->options().message_set_wire_format()) {
-      printer->Print(
-        "size += extensionsSerializedSizeAsMessageSet();\n");
+      printer->Print("size += extensionsSerializedSizeAsMessageSet();\n");
     } else {
-      printer->Print(
-        "size += extensionsSerializedSize();\n");
+      printer->Print("size += extensionsSerializedSize();\n");
     }
   }
 
   if (descriptor_->options().message_set_wire_format()) {
-    printer->Print(
-      "size += unknownFields.getSerializedSizeAsMessageSet();\n");
+    printer->Print("size += unknownFields.getSerializedSizeAsMessageSet();\n");
   } else {
-    printer->Print(
-      "size += unknownFields.getSerializedSize();\n");
+    printer->Print("size += unknownFields.getSerializedSize();\n");
   }
 
+  printer->Print(
+      "memoizedSerializedSize = size;\n"
+      "return size;\n");
+
   printer->Outdent();
   printer->Print(
-    "  memoizedSerializedSize = size;\n"
-    "  return size;\n"
     "}\n"
     "\n");
 }
@@ -702,7 +730,10 @@ void ImmutableMessageLiteGenerator::GenerateDynamicMethodIsInitialized(
     return;
   }
 
-  // Don't directly compare to -1 to avoid an Android x86 JIT bug.
+  // TODO(xiaofeng): Remove this when b/64445758 is fixed. We don't need to
+  // check memoizedIsInitialized here because the caller does that already,
+  // but right now proguard proto shrinker asserts on the bytecode layout of
+  // this code so it can't be removed until proguard is updated.
   printer->Print(
     "byte isInitialized = memoizedIsInitialized;\n"
     "if (isInitialized == 1) return DEFAULT_INSTANCE;\n"
@@ -720,9 +751,6 @@ void ImmutableMessageLiteGenerator::GenerateDynamicMethodIsInitialized(
     if (field->is_required()) {
       printer->Print(
         "if (!has$name$()) {\n"
-        "  if (shouldMemoize) {\n"
-        "    memoizedIsInitialized = 0;\n"
-        "  }\n"
         "  return null;\n"
         "}\n",
         "name", info->capitalized_name);
@@ -739,9 +767,6 @@ void ImmutableMessageLiteGenerator::GenerateDynamicMethodIsInitialized(
         case FieldDescriptor::LABEL_REQUIRED:
           printer->Print(
             "if (!get$name$().isInitialized()) {\n"
-            "  if (shouldMemoize) {\n"
-            "    memoizedIsInitialized = 0;\n"
-            "  }\n"
             "  return null;\n"
             "}\n",
             "type", name_resolver_->GetImmutableClassName(
@@ -765,9 +790,6 @@ void ImmutableMessageLiteGenerator::GenerateDynamicMethodIsInitialized(
           }
           printer->Print(
             "  if (!get$name$().isInitialized()) {\n"
-            "    if (shouldMemoize) {\n"
-            "      memoizedIsInitialized = 0;\n"
-            "    }\n"
             "    return null;\n"
             "  }\n"
             "}\n",
@@ -778,9 +800,6 @@ void ImmutableMessageLiteGenerator::GenerateDynamicMethodIsInitialized(
             printer->Print(
               "for ($type$ item : get$name$Map().values()) {\n"
               "  if (!item.isInitialized()) {\n"
-              "    if (shouldMemoize) {\n"
-              "      memoizedIsInitialized = 0;\n"
-              "    }\n"
               "    return null;\n"
               "  }\n"
               "}\n",
@@ -791,9 +810,6 @@ void ImmutableMessageLiteGenerator::GenerateDynamicMethodIsInitialized(
             printer->Print(
               "for (int i = 0; i < get$name$Count(); i++) {\n"
               "  if (!get$name$(i).isInitialized()) {\n"
-              "    if (shouldMemoize) {\n"
-              "      memoizedIsInitialized = 0;\n"
-              "    }\n"
               "    return null;\n"
               "  }\n"
               "}\n",
@@ -809,16 +825,10 @@ void ImmutableMessageLiteGenerator::GenerateDynamicMethodIsInitialized(
   if (descriptor_->extension_range_count() > 0) {
     printer->Print(
       "if (!extensionsAreInitialized()) {\n"
-      "  if (shouldMemoize) {\n"
-      "    memoizedIsInitialized = 0;\n"
-      "  }\n"
       "  return null;\n"
       "}\n");
   }
 
-  printer->Print(
-    "if (shouldMemoize) memoizedIsInitialized = 1;\n");
-
   printer->Print(
     "return DEFAULT_INSTANCE;\n"
     "\n");
@@ -946,99 +956,107 @@ void ImmutableMessageLiteGenerator::GenerateDynamicMethodMergeFromStream(
       "com.google.protobuf.CodedInputStream input =\n"
       "    (com.google.protobuf.CodedInputStream) arg0;\n"
       "com.google.protobuf.ExtensionRegistryLite extensionRegistry =\n"
-      "    (com.google.protobuf.ExtensionRegistryLite) arg1;\n");
+      "    (com.google.protobuf.ExtensionRegistryLite) arg1;\n"
+      "if (extensionRegistry == null) {\n"
+      "  throw new java.lang.NullPointerException();\n"
+      "}\n");
   printer->Print(
       "try {\n");
   printer->Indent();
-  printer->Print(
-    "boolean done = false;\n"
-    "while (!done) {\n");
-  printer->Indent();
-
-  printer->Print(
-    "int tag = input.readTag();\n"
-    "switch (tag) {\n");
-  printer->Indent();
+  if (EnableExperimentalRuntime(context_)) {
+    printer->Print(
+        "mergeFromInternal(input, extensionRegistry);\n"
+        "return DEFAULT_INSTANCE;\n");
+  } else {
+    printer->Print(
+        "boolean done = false;\n"
+        "while (!done) {\n");
+    printer->Indent();
 
-  printer->Print(
-    "case 0:\n"          // zero signals EOF / limit reached
-    "  done = true;\n"
-    "  break;\n");
+    printer->Print(
+        "int tag = input.readTag();\n"
+        "switch (tag) {\n");
+    printer->Indent();
 
-  if (descriptor_->extension_range_count() > 0) {
-    if (descriptor_->options().message_set_wire_format()) {
-      printer->Print(
-          "default: {\n"
-          "  if (!parseUnknownFieldAsMessageSet(\n"
-          "      getDefaultInstanceForType(), input, extensionRegistry,\n"
-          "      tag)) {\n"
-          "    done = true;\n"  // it's an endgroup tag
-          "  }\n"
-          "  break;\n"
-          "}\n");
+    printer->Print(
+        "case 0:\n"  // zero signals EOF / limit reached
+        "  done = true;\n"
+        "  break;\n");
+
+    if (descriptor_->extension_range_count() > 0) {
+      if (descriptor_->options().message_set_wire_format()) {
+        printer->Print(
+            "default: {\n"
+            "  if (!parseUnknownFieldAsMessageSet(\n"
+            "      getDefaultInstanceForType(), input, extensionRegistry,\n"
+            "      tag)) {\n"
+            "    done = true;\n"  // it's an endgroup tag
+            "  }\n"
+            "  break;\n"
+            "}\n");
+      } else {
+        printer->Print(
+            "default: {\n"
+            "  if (!parseUnknownField(getDefaultInstanceForType(),\n"
+            "      input, extensionRegistry, tag)) {\n"
+            "    done = true;\n"  // it's an endgroup tag
+            "  }\n"
+            "  break;\n"
+            "}\n");
+      }
     } else {
       printer->Print(
           "default: {\n"
-          "  if (!parseUnknownField(getDefaultInstanceForType(),\n"
-          "      input, extensionRegistry, tag)) {\n"
+          "  if (!parseUnknownField(tag, input)) {\n"
           "    done = true;\n"  // it's an endgroup tag
           "  }\n"
           "  break;\n"
           "}\n");
     }
-  } else {
-    printer->Print(
-      "default: {\n"
-      "  if (!parseUnknownField(tag, input)) {\n"
-      "    done = true;\n"  // it's an endgroup tag
-      "  }\n"
-      "  break;\n"
-      "}\n");
-  }
 
-  google::protobuf::scoped_array<const FieldDescriptor * > sorted_fields(
-      SortFieldsByNumber(descriptor_));
-  for (int i = 0; i < descriptor_->field_count(); i++) {
-    const FieldDescriptor* field = sorted_fields[i];
-    uint32 tag = WireFormatLite::MakeTag(field->number(),
-      WireFormat::WireTypeForFieldType(field->type()));
-
-    printer->Print(
-      "case $tag$: {\n",
-      "tag", SimpleItoa(static_cast<int32>(tag)));
-    printer->Indent();
-
-    field_generators_.get(field).GenerateParsingCode(printer);
-
-    printer->Outdent();
-    printer->Print(
-      "  break;\n"
-      "}\n");
+    google::protobuf::scoped_array<const FieldDescriptor* > sorted_fields(
+        SortFieldsByNumber(descriptor_));
+    for (int i = 0; i < descriptor_->field_count(); i++) {
+      const FieldDescriptor* field = sorted_fields[i];
+      uint32 tag = WireFormatLite::MakeTag(
+          field->number(), WireFormat::WireTypeForFieldType(field->type()));
 
-    if (field->is_packable()) {
-      // To make packed = true wire compatible, we generate parsing code from a
-      // packed version of this field regardless of field->options().packed().
-      uint32 packed_tag = WireFormatLite::MakeTag(field->number(),
-        WireFormatLite::WIRETYPE_LENGTH_DELIMITED);
-      printer->Print(
-        "case $tag$: {\n",
-        "tag", SimpleItoa(static_cast<int32>(packed_tag)));
+      printer->Print("case $tag$: {\n", "tag",
+                     SimpleItoa(static_cast<int32>(tag)));
       printer->Indent();
 
-      field_generators_.get(field).GenerateParsingCodeFromPacked(printer);
+      field_generators_.get(field).GenerateParsingCode(printer);
 
       printer->Outdent();
       printer->Print(
         "  break;\n"
         "}\n");
+
+      if (field->is_packable()) {
+        // To make packed = true wire compatible, we generate parsing code from
+        // a packed version of this field regardless of
+        // field->options().packed().
+        uint32 packed_tag = WireFormatLite::MakeTag(
+            field->number(), WireFormatLite::WIRETYPE_LENGTH_DELIMITED);
+        printer->Print("case $tag$: {\n", "tag",
+                       SimpleItoa(static_cast<int32>(packed_tag)));
+        printer->Indent();
+
+        field_generators_.get(field).GenerateParsingCodeFromPacked(printer);
+
+        printer->Outdent();
+        printer->Print(
+            "  break;\n"
+            "}\n");
+      }
     }
-  }
 
-  printer->Outdent();
-  printer->Outdent();
-  printer->Print(
-      "  }\n"     // switch (tag)
-      "}\n");     // while (!done)
+    printer->Outdent();
+    printer->Outdent();
+    printer->Print(
+        "  }\n"  // switch (tag)
+        "}\n");  // while (!done)
+  }
 
   printer->Outdent();
   printer->Print(

src/google/protobuf/compiler/java/java_shared_code_generator.cc

@@ -140,13 +140,16 @@ void SharedCodeGenerator::GenerateDescriptors(io::Printer* printer) {
     "java.lang.String[] descriptorData = {\n");
   printer->Indent();
 
-  // Only write 40 bytes per line.
+  // Limit the number of bytes per line.
   static const int kBytesPerLine = 40;
+  // Limit the number of lines per string part.
+  static const int kLinesPerPart = 400;
+  // Every block of bytes, start a new string literal, in order to avoid the
+  // 64k length limit. Note that this value needs to be <64k.
+  static const int kBytesPerPart = kBytesPerLine * kLinesPerPart;
   for (int i = 0; i < file_data.size(); i += kBytesPerLine) {
     if (i > 0) {
-      // Every 400 lines, start a new string literal, in order to avoid the
-      // 64k length limit.
-      if (i % 400 == 0) {
+      if (i % kBytesPerPart == 0) {
         printer->Print(",\n");
       } else {
         printer->Print(" +\n");

src/google/protobuf/compiler/js/js_generator.cc

@@ -1082,6 +1082,40 @@ string JSReturnClause(const FieldDescriptor* desc) {
   return "";
 }
 
+string JSTypeTag(const FieldDescriptor* desc) {
+  switch (desc->type()) {
+    case FieldDescriptor::TYPE_DOUBLE:
+    case FieldDescriptor::TYPE_FLOAT:
+      return "Float";
+    case FieldDescriptor::TYPE_INT32:
+    case FieldDescriptor::TYPE_UINT32:
+    case FieldDescriptor::TYPE_INT64:
+    case FieldDescriptor::TYPE_UINT64:
+    case FieldDescriptor::TYPE_FIXED32:
+    case FieldDescriptor::TYPE_FIXED64:
+    case FieldDescriptor::TYPE_SINT32:
+    case FieldDescriptor::TYPE_SINT64:
+    case FieldDescriptor::TYPE_SFIXED32:
+    case FieldDescriptor::TYPE_SFIXED64:
+      if (IsIntegralFieldWithStringJSType(desc)) {
+        return "StringInt";
+      } else {
+        return "Int";
+      }
+    case FieldDescriptor::TYPE_BOOL:
+      return "Boolean";
+    case FieldDescriptor::TYPE_STRING:
+      return "String";
+    case FieldDescriptor::TYPE_BYTES:
+      return "Bytes";
+    case FieldDescriptor::TYPE_ENUM:
+      return "Enum";
+    default:
+      assert(false);
+  }
+  return "";
+}
+
 string JSReturnDoc(const GeneratorOptions& options,
                    const FieldDescriptor* desc) {
   return "";
@@ -2569,26 +2603,44 @@ void Generator::GenerateClassField(const GeneratorOptions& options,
               /* singular_if_not_packed = */ false),
           "returndoc", JSReturnDoc(options, field));
     }
-    printer->Print(
-        "$class$.prototype.set$name$ = function(value) {\n"
-        "  jspb.Message.set$oneoftag$Field(this, $index$",
-        "class", GetMessagePath(options, field->containing_type()),
-        "name", JSGetterName(options, field),
-        "oneoftag", (field->containing_oneof() ? "Oneof" : ""),
-        "index", JSFieldIndex(field));
-    printer->Print(
-        "$oneofgroup$, $type$value$rptvalueinit$$typeclose$);$returnvalue$\n"
-        "};\n"
-        "\n"
-        "\n",
-        "type",
-        untyped ? "/** @type{string|number|boolean|Array|undefined} */(" : "",
-        "typeclose", untyped ? ")" : "",
-        "oneofgroup",
-        (field->containing_oneof() ? (", " + JSOneofArray(options, field))
-                                   : ""),
-        "returnvalue", JSReturnClause(field), "rptvalueinit",
-        (field->is_repeated() ? " || []" : ""));
+
+    if (field->file()->syntax() == FileDescriptor::SYNTAX_PROTO3 &&
+        !field->is_repeated() && !field->is_map() &&
+        !HasFieldPresence(options, field)) {
+      // Proto3 non-repeated and non-map fields without presence use the
+      // setProto3*Field function.
+      printer->Print(
+          "$class$.prototype.set$name$ = function(value) {\n"
+          "  jspb.Message.setProto3$typetag$Field(this, $index$, "
+          "value);$returnvalue$\n"
+          "};\n"
+          "\n"
+          "\n",
+          "class", GetMessagePath(options, field->containing_type()), "name",
+          JSGetterName(options, field), "typetag", JSTypeTag(field), "index",
+          JSFieldIndex(field), "returnvalue", JSReturnClause(field));
+    } else {
+      // Otherwise, use the regular setField function.
+      printer->Print(
+          "$class$.prototype.set$name$ = function(value) {\n"
+          "  jspb.Message.set$oneoftag$Field(this, $index$",
+          "class", GetMessagePath(options, field->containing_type()), "name",
+          JSGetterName(options, field), "oneoftag",
+          (field->containing_oneof() ? "Oneof" : ""), "index",
+          JSFieldIndex(field));
+      printer->Print(
+          "$oneofgroup$, $type$value$rptvalueinit$$typeclose$);$returnvalue$\n"
+          "};\n"
+          "\n"
+          "\n",
+          "type",
+          untyped ? "/** @type{string|number|boolean|Array|undefined} */(" : "",
+          "typeclose", untyped ? ")" : "", "oneofgroup",
+          (field->containing_oneof() ? (", " + JSOneofArray(options, field))
+                                     : ""),
+          "returnvalue", JSReturnClause(field), "rptvalueinit",
+          (field->is_repeated() ? " || []" : ""));
+    }
 
     if (untyped) {
       printer->Print(

src/google/protobuf/compiler/mock_code_generator.cc

@@ -57,12 +57,12 @@
 #include <google/protobuf/stubs/substitute.h>
 #include <gtest/gtest.h>
 
-#ifdef major		
-#undef major		
-#endif		
-#ifdef minor		
-#undef minor		
-#endif		
+#ifdef major
+#undef major
+#endif
+#ifdef minor
+#undef minor
+#endif
 
 namespace google {
 namespace protobuf {