Merge branch 'sync-piper' into sync-stage

conformance/binary_json_conformance_suite.cc

@@ -556,24 +556,24 @@ void BinaryAndJsonConformanceSuite::RunValidProtobufTestWithMessage(
                        equivalent_text_format, is_proto3);
 }
 
-// According to proto3 JSON specification, JSON serializers follow more strict
+// According to proto JSON specification, JSON serializers follow more strict
 // rules than parsers (e.g., a serializer must serialize int32 values as JSON
 // numbers while the parser is allowed to accept them as JSON strings). This
-// method allows strict checking on a proto3 JSON serializer by inspecting
+// method allows strict checking on a proto JSON serializer by inspecting
 // the JSON output directly.
 void BinaryAndJsonConformanceSuite::RunValidJsonTestWithValidator(
     const string& test_name, ConformanceLevel level, const string& input_json,
-    const Validator& validator) {
-  TestAllTypesProto3 prototype;
-  ConformanceRequestSetting setting(
-      level, conformance::JSON, conformance::JSON,
-      conformance::JSON_TEST,
-      prototype, test_name, input_json);
+    const Validator& validator, bool is_proto3) {
+  std::unique_ptr<Message> prototype = NewTestMessage(is_proto3);
+  ConformanceRequestSetting setting(level, conformance::JSON, conformance::JSON,
+                                    conformance::JSON_TEST, *prototype,
+                                    test_name, input_json);
   const ConformanceRequest& request = setting.GetRequest();
   ConformanceResponse response;
   string effective_test_name =
       StrCat(setting.ConformanceLevelToString(level),
-                   ".Proto3.JsonInput.", test_name, ".Validator");
+                   is_proto3 ? ".Proto3.JsonInput." : ".Proto2.JsonInput.",
+                   test_name, ".Validator");
 
   RunTest(effective_test_name, request, &response);
 
@@ -1800,7 +1800,8 @@ void BinaryAndJsonConformanceSuite::RunJsonTestsForFieldNameConvention() {
             value.isMember("fieldName2") &&
             value.isMember("FieldName3") &&
             value.isMember("fieldName4");
-      });
+      },
+      true);
   RunValidJsonTestWithValidator(
       "FieldNameWithNumbers", REQUIRED,
       R"({
@@ -1810,7 +1811,8 @@ void BinaryAndJsonConformanceSuite::RunJsonTestsForFieldNameConvention() {
       [](const Json::Value& value) {
         return value.isMember("field0name5") &&
             value.isMember("field0Name6");
-      });
+      },
+      true);
   RunValidJsonTestWithValidator(
       "FieldNameWithMixedCases", REQUIRED,
       R"({
@@ -1828,7 +1830,8 @@ void BinaryAndJsonConformanceSuite::RunJsonTestsForFieldNameConvention() {
             value.isMember("FieldName10") &&
             value.isMember("FIELDNAME11") &&
             value.isMember("FIELDName12");
-      });
+      },
+      true);
   RunValidJsonTestWithValidator(
       "FieldNameWithDoubleUnderscores", RECOMMENDED,
       R"({
@@ -1846,7 +1849,22 @@ void BinaryAndJsonConformanceSuite::RunJsonTestsForFieldNameConvention() {
             value.isMember("fieldName16") &&
             value.isMember("fieldName17") &&
             value.isMember("FieldName18");
-      });
+      },
+      true);
+  RunValidJsonTestWithValidator(
+      "StoresDefaultPrimitive", REQUIRED,
+      R"({
+        "FieldName13": 0
+      })",
+      [](const Json::Value& value) { return value.isMember("FieldName13"); },
+      false);
+  RunValidJsonTestWithValidator(
+      "SkipsDefaultPrimitive", REQUIRED,
+      R"({
+        "FieldName13": 0
+      })",
+      [](const Json::Value& value) { return !value.isMember("FieldName13"); },
+      true);
 }
 
 void BinaryAndJsonConformanceSuite::RunJsonTestsForNonRepeatedTypes() {
@@ -1995,19 +2013,19 @@ void BinaryAndJsonConformanceSuite::RunJsonTestsForNonRepeatedTypes() {
 
   // 64-bit values are serialized as strings.
   RunValidJsonTestWithValidator(
-      "Int64FieldBeString", RECOMMENDED,
-      R"({"optionalInt64": 1})",
+      "Int64FieldBeString", RECOMMENDED, R"({"optionalInt64": 1})",
       [](const Json::Value& value) {
         return value["optionalInt64"].type() == Json::stringValue &&
             value["optionalInt64"].asString() == "1";
-      });
+      },
+      true);
   RunValidJsonTestWithValidator(
-      "Uint64FieldBeString", RECOMMENDED,
-      R"({"optionalUint64": 1})",
+      "Uint64FieldBeString", RECOMMENDED, R"({"optionalUint64": 1})",
       [](const Json::Value& value) {
         return value["optionalUint64"].type() == Json::stringValue &&
             value["optionalUint64"].asString() == "1";
-      });
+      },
+      true);
 
   // Bool fields.
   RunValidJsonTest(
@@ -2223,12 +2241,12 @@ void BinaryAndJsonConformanceSuite::RunJsonTestsForNonRepeatedTypes() {
       "optional_nested_enum: BAR");
   // Unknown enum values are represented as numeric values.
   RunValidJsonTestWithValidator(
-      "EnumFieldUnknownValue", REQUIRED,
-      R"({"optionalNestedEnum": 123})",
+      "EnumFieldUnknownValue", REQUIRED, R"({"optionalNestedEnum": 123})",
       [](const Json::Value& value) {
         return value["optionalNestedEnum"].type() == Json::intValue &&
             value["optionalNestedEnum"].asInt() == 123;
-      });
+      },
+      true);
 
   // String fields.
   RunValidJsonTest(
@@ -2712,25 +2730,29 @@ void BinaryAndJsonConformanceSuite::RunJsonTestsForWrapperTypes() {
       R"({"optionalDuration": "1.000000000s"})",
       [](const Json::Value& value) {
         return value["optionalDuration"].asString() == "1s";
-      });
+      },
+      true);
   RunValidJsonTestWithValidator(
       "DurationHas3FractionalDigits", RECOMMENDED,
       R"({"optionalDuration": "1.010000000s"})",
       [](const Json::Value& value) {
         return value["optionalDuration"].asString() == "1.010s";
-      });
+      },
+      true);
   RunValidJsonTestWithValidator(
       "DurationHas6FractionalDigits", RECOMMENDED,
       R"({"optionalDuration": "1.000010000s"})",
       [](const Json::Value& value) {
         return value["optionalDuration"].asString() == "1.000010s";
-      });
+      },
+      true);
   RunValidJsonTestWithValidator(
       "DurationHas9FractionalDigits", RECOMMENDED,
       R"({"optionalDuration": "1.000000010s"})",
       [](const Json::Value& value) {
         return value["optionalDuration"].asString() == "1.000000010s";
-      });
+      },
+      true);
 
   // Timestamp
   RunValidJsonTest(
@@ -2794,34 +2816,39 @@ void BinaryAndJsonConformanceSuite::RunJsonTestsForWrapperTypes() {
       R"({"optionalTimestamp": "1969-12-31T16:00:00-08:00"})",
       [](const Json::Value& value) {
         return value["optionalTimestamp"].asString() == "1970-01-01T00:00:00Z";
-      });
+      },
+      true);
   RunValidJsonTestWithValidator(
       "TimestampHasZeroFractionalDigit", RECOMMENDED,
       R"({"optionalTimestamp": "1970-01-01T00:00:00.000000000Z"})",
       [](const Json::Value& value) {
         return value["optionalTimestamp"].asString() == "1970-01-01T00:00:00Z";
-      });
+      },
+      true);
   RunValidJsonTestWithValidator(
       "TimestampHas3FractionalDigits", RECOMMENDED,
       R"({"optionalTimestamp": "1970-01-01T00:00:00.010000000Z"})",
       [](const Json::Value& value) {
         return value["optionalTimestamp"].asString() ==
                "1970-01-01T00:00:00.010Z";
-      });
+      },
+      true);
   RunValidJsonTestWithValidator(
       "TimestampHas6FractionalDigits", RECOMMENDED,
       R"({"optionalTimestamp": "1970-01-01T00:00:00.000010000Z"})",
       [](const Json::Value& value) {
         return value["optionalTimestamp"].asString() ==
                "1970-01-01T00:00:00.000010Z";
-      });
+      },
+      true);
   RunValidJsonTestWithValidator(
       "TimestampHas9FractionalDigits", RECOMMENDED,
       R"({"optionalTimestamp": "1970-01-01T00:00:00.000000010Z"})",
       [](const Json::Value& value) {
         return value["optionalTimestamp"].asString() ==
                "1970-01-01T00:00:00.000000010Z";
-      });
+      },
+      true);
 }
 
 void BinaryAndJsonConformanceSuite::RunJsonTestsForFieldMask() {

conformance/binary_json_conformance_suite.h

@@ -95,7 +95,8 @@ class BinaryAndJsonConformanceSuite : public ConformanceTestSuite {
   void RunValidJsonTestWithValidator(const std::string& test_name,
                                      ConformanceLevel level,
                                      const std::string& input_json,
-                                     const Validator& validator);
+                                     const Validator& validator,
+                                     bool is_proto3);
   void ExpectParseFailureForJson(const std::string& test_name,
                                  ConformanceLevel level,
                                  const std::string& input_json);

conformance/failure_list_java.txt

@@ -45,3 +45,4 @@ Required.Proto3.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValu
 Required.Proto3.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE
 Required.Proto2.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE
 Required.Proto2.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE
+Required.Proto2.JsonInput.StoresDefaultPrimitive.Validator

csharp/src/Google.Protobuf.Test.TestProtos/UnittestProto3Optional.cs

@@ -25,7 +25,7 @@ namespace ProtobufUnittest {
       byte[] descriptorData = global::System.Convert.FromBase64String(
           string.Concat(
             "Ci5nb29nbGUvcHJvdG9idWYvdW5pdHRlc3RfcHJvdG8zX29wdGlvbmFsLnBy",
-            "b3RvEhFwcm90b2J1Zl91bml0dGVzdCKBCgoSVGVzdFByb3RvM09wdGlvbmFs",
+            "b3RvEhFwcm90b2J1Zl91bml0dGVzdCKxCgoSVGVzdFByb3RvM09wdGlvbmFs",
             "EhsKDm9wdGlvbmFsX2ludDMyGAEgASgFSACIAQESGwoOb3B0aW9uYWxfaW50",
             "NjQYAiABKANIAYgBARIcCg9vcHRpb25hbF91aW50MzIYAyABKA1IAogBARIc",
             "Cg9vcHRpb25hbF91aW50NjQYBCABKARIA4gBARIcCg9vcHRpb25hbF9zaW50",
@@ -42,24 +42,25 @@ namespace ProtobufUnittest {
             "dGVkX21lc3NhZ2UYEyABKAsyMy5wcm90b2J1Zl91bml0dGVzdC5UZXN0UHJv",
             "dG8zT3B0aW9uYWwuTmVzdGVkTWVzc2FnZUICKAFIEYgBARJTChRvcHRpb25h",
             "bF9uZXN0ZWRfZW51bRgVIAEoDjIwLnByb3RvYnVmX3VuaXR0ZXN0LlRlc3RQ",
-            "cm90bzNPcHRpb25hbC5OZXN0ZWRFbnVtSBKIAQEaJwoNTmVzdGVkTWVzc2Fn",
-            "ZRIPCgJiYhgBIAEoBUgAiAEBQgUKA19iYiJKCgpOZXN0ZWRFbnVtEg8KC1VO",
-            "U1BFQ0lGSUVEEAASBwoDRk9PEAESBwoDQkFSEAISBwoDQkFaEAMSEAoDTkVH",
-            "EP///////////wFCEQoPX29wdGlvbmFsX2ludDMyQhEKD19vcHRpb25hbF9p",
-            "bnQ2NEISChBfb3B0aW9uYWxfdWludDMyQhIKEF9vcHRpb25hbF91aW50NjRC",
-            "EgoQX29wdGlvbmFsX3NpbnQzMkISChBfb3B0aW9uYWxfc2ludDY0QhMKEV9v",
-            "cHRpb25hbF9maXhlZDMyQhMKEV9vcHRpb25hbF9maXhlZDY0QhQKEl9vcHRp",
-            "b25hbF9zZml4ZWQzMkIUChJfb3B0aW9uYWxfc2ZpeGVkNjRCEQoPX29wdGlv",
-            "bmFsX2Zsb2F0QhIKEF9vcHRpb25hbF9kb3VibGVCEAoOX29wdGlvbmFsX2Jv",
-            "b2xCEgoQX29wdGlvbmFsX3N0cmluZ0IRCg9fb3B0aW9uYWxfYnl0ZXNCEAoO",
-            "X29wdGlvbmFsX2NvcmRCGgoYX29wdGlvbmFsX25lc3RlZF9tZXNzYWdlQhYK",
-            "FF9sYXp5X25lc3RlZF9tZXNzYWdlQhcKFV9vcHRpb25hbF9uZXN0ZWRfZW51",
-            "bUIlCiFjb20uZ29vZ2xlLnByb3RvYnVmLnRlc3RpbmcucHJvdG9QAWIGcHJv",
-            "dG8z"));
+            "cm90bzNPcHRpb25hbC5OZXN0ZWRFbnVtSBKIAQESFgoOc2luZ3VsYXJfaW50",
+            "MzIYFiABKAUSFgoOc2luZ3VsYXJfaW50NjQYFyABKAMaJwoNTmVzdGVkTWVz",
+            "c2FnZRIPCgJiYhgBIAEoBUgAiAEBQgUKA19iYiJKCgpOZXN0ZWRFbnVtEg8K",
+            "C1VOU1BFQ0lGSUVEEAASBwoDRk9PEAESBwoDQkFSEAISBwoDQkFaEAMSEAoD",
+            "TkVHEP///////////wFCEQoPX29wdGlvbmFsX2ludDMyQhEKD19vcHRpb25h",
+            "bF9pbnQ2NEISChBfb3B0aW9uYWxfdWludDMyQhIKEF9vcHRpb25hbF91aW50",
+            "NjRCEgoQX29wdGlvbmFsX3NpbnQzMkISChBfb3B0aW9uYWxfc2ludDY0QhMK",
+            "EV9vcHRpb25hbF9maXhlZDMyQhMKEV9vcHRpb25hbF9maXhlZDY0QhQKEl9v",
+            "cHRpb25hbF9zZml4ZWQzMkIUChJfb3B0aW9uYWxfc2ZpeGVkNjRCEQoPX29w",
+            "dGlvbmFsX2Zsb2F0QhIKEF9vcHRpb25hbF9kb3VibGVCEAoOX29wdGlvbmFs",
+            "X2Jvb2xCEgoQX29wdGlvbmFsX3N0cmluZ0IRCg9fb3B0aW9uYWxfYnl0ZXNC",
+            "EAoOX29wdGlvbmFsX2NvcmRCGgoYX29wdGlvbmFsX25lc3RlZF9tZXNzYWdl",
+            "QhYKFF9sYXp5X25lc3RlZF9tZXNzYWdlQhcKFV9vcHRpb25hbF9uZXN0ZWRf",
+            "ZW51bUIlCiFjb20uZ29vZ2xlLnByb3RvYnVmLnRlc3RpbmcucHJvdG9QAWIG",
+            "cHJvdG8z"));
       descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData,
           new pbr::FileDescriptor[] { },
           new pbr::GeneratedClrTypeInfo(null, null, new pbr::GeneratedClrTypeInfo[] {
-            new pbr::GeneratedClrTypeInfo(typeof(global::ProtobufUnittest.TestProto3Optional), global::ProtobufUnittest.TestProto3Optional.Parser, new[]{ "OptionalInt32", "OptionalInt64", "OptionalUint32", "OptionalUint64", "OptionalSint32", "OptionalSint64", "OptionalFixed32", "OptionalFixed64", "OptionalSfixed32", "OptionalSfixed64", "OptionalFloat", "OptionalDouble", "OptionalBool", "OptionalString", "OptionalBytes", "OptionalCord", "OptionalNestedMessage", "LazyNestedMessage", "OptionalNestedEnum" }, new[]{ "OptionalInt32", "OptionalInt64", "OptionalUint32", "OptionalUint64", "OptionalSint32", "OptionalSint64", "OptionalFixed32", "OptionalFixed64", "OptionalSfixed32", "OptionalSfixed64", "OptionalFloat", "OptionalDouble", "OptionalBool", "OptionalString", "OptionalBytes", "OptionalCord", "OptionalNestedMessage", "LazyNestedMessage", "OptionalNestedEnum" }, new[]{ typeof(global::ProtobufUnittest.TestProto3Optional.Types.NestedEnum) }, null, new pbr::GeneratedClrTypeInfo[] { new pbr::GeneratedClrTypeInfo(typeof(global::ProtobufUnittest.TestProto3Optional.Types.NestedMessage), global::ProtobufUnittest.TestProto3Optional.Types.NestedMessage.Parser, new[]{ "Bb" }, new[]{ "Bb" }, null, null, null)})
+            new pbr::GeneratedClrTypeInfo(typeof(global::ProtobufUnittest.TestProto3Optional), global::ProtobufUnittest.TestProto3Optional.Parser, new[]{ "OptionalInt32", "OptionalInt64", "OptionalUint32", "OptionalUint64", "OptionalSint32", "OptionalSint64", "OptionalFixed32", "OptionalFixed64", "OptionalSfixed32", "OptionalSfixed64", "OptionalFloat", "OptionalDouble", "OptionalBool", "OptionalString", "OptionalBytes", "OptionalCord", "OptionalNestedMessage", "LazyNestedMessage", "OptionalNestedEnum", "SingularInt32", "SingularInt64" }, new[]{ "OptionalInt32", "OptionalInt64", "OptionalUint32", "OptionalUint64", "OptionalSint32", "OptionalSint64", "OptionalFixed32", "OptionalFixed64", "OptionalSfixed32", "OptionalSfixed64", "OptionalFloat", "OptionalDouble", "OptionalBool", "OptionalString", "OptionalBytes", "OptionalCord", "OptionalNestedMessage", "LazyNestedMessage", "OptionalNestedEnum" }, new[]{ typeof(global::ProtobufUnittest.TestProto3Optional.Types.NestedEnum) }, null, new pbr::GeneratedClrTypeInfo[] { new pbr::GeneratedClrTypeInfo(typeof(global::ProtobufUnittest.TestProto3Optional.Types.NestedMessage), global::ProtobufUnittest.TestProto3Optional.Types.NestedMessage.Parser, new[]{ "Bb" }, new[]{ "Bb" }, null, null, null)})
           }));
     }
     #endregion
@@ -112,6 +113,8 @@ namespace ProtobufUnittest {
       optionalNestedMessage_ = other.optionalNestedMessage_ != null ? other.optionalNestedMessage_.Clone() : null;
       lazyNestedMessage_ = other.lazyNestedMessage_ != null ? other.lazyNestedMessage_.Clone() : null;
       optionalNestedEnum_ = other.optionalNestedEnum_;
+      singularInt32_ = other.singularInt32_;
+      singularInt64_ = other.singularInt64_;
       _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields);
     }
 
@@ -516,6 +519,31 @@ namespace ProtobufUnittest {
       _hasBits0 &= ~8192;
     }
 
+    /// <summary>Field number for the "singular_int32" field.</summary>
+    public const int SingularInt32FieldNumber = 22;
+    private int singularInt32_;
+    /// <summary>
+    /// Add some non-optional fields to verify we can mix them.
+    /// </summary>
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public int SingularInt32 {
+      get { return singularInt32_; }
+      set {
+        singularInt32_ = value;
+      }
+    }
+
+    /// <summary>Field number for the "singular_int64" field.</summary>
+    public const int SingularInt64FieldNumber = 23;
+    private long singularInt64_;
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public long SingularInt64 {
+      get { return singularInt64_; }
+      set {
+        singularInt64_ = value;
+      }
+    }
+
     [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
     public override bool Equals(object other) {
       return Equals(other as TestProto3Optional);
@@ -548,6 +576,8 @@ namespace ProtobufUnittest {
       if (!object.Equals(OptionalNestedMessage, other.OptionalNestedMessage)) return false;
       if (!object.Equals(LazyNestedMessage, other.LazyNestedMessage)) return false;
       if (OptionalNestedEnum != other.OptionalNestedEnum) return false;
+      if (SingularInt32 != other.SingularInt32) return false;
+      if (SingularInt64 != other.SingularInt64) return false;
       return Equals(_unknownFields, other._unknownFields);
     }
 
@@ -573,6 +603,8 @@ namespace ProtobufUnittest {
       if (optionalNestedMessage_ != null) hash ^= OptionalNestedMessage.GetHashCode();
       if (lazyNestedMessage_ != null) hash ^= LazyNestedMessage.GetHashCode();
       if (HasOptionalNestedEnum) hash ^= OptionalNestedEnum.GetHashCode();
+      if (SingularInt32 != 0) hash ^= SingularInt32.GetHashCode();
+      if (SingularInt64 != 0L) hash ^= SingularInt64.GetHashCode();
       if (_unknownFields != null) {
         hash ^= _unknownFields.GetHashCode();
       }
@@ -662,6 +694,14 @@ namespace ProtobufUnittest {
         output.WriteRawTag(168, 1);
         output.WriteEnum((int) OptionalNestedEnum);
       }
+      if (SingularInt32 != 0) {
+        output.WriteRawTag(176, 1);
+        output.WriteInt32(SingularInt32);
+      }
+      if (SingularInt64 != 0L) {
+        output.WriteRawTag(184, 1);
+        output.WriteInt64(SingularInt64);
+      }
       if (_unknownFields != null) {
         _unknownFields.WriteTo(output);
       }
@@ -727,6 +767,12 @@ namespace ProtobufUnittest {
       if (HasOptionalNestedEnum) {
         size += 2 + pb::CodedOutputStream.ComputeEnumSize((int) OptionalNestedEnum);
       }
+      if (SingularInt32 != 0) {
+        size += 2 + pb::CodedOutputStream.ComputeInt32Size(SingularInt32);
+      }
+      if (SingularInt64 != 0L) {
+        size += 2 + pb::CodedOutputStream.ComputeInt64Size(SingularInt64);
+      }
       if (_unknownFields != null) {
         size += _unknownFields.CalculateSize();
       }
@@ -801,6 +847,12 @@ namespace ProtobufUnittest {
       if (other.HasOptionalNestedEnum) {
         OptionalNestedEnum = other.OptionalNestedEnum;
       }
+      if (other.SingularInt32 != 0) {
+        SingularInt32 = other.SingularInt32;
+      }
+      if (other.SingularInt64 != 0L) {
+        SingularInt64 = other.SingularInt64;
+      }
       _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields);
     }
 
@@ -899,6 +951,14 @@ namespace ProtobufUnittest {
             OptionalNestedEnum = (global::ProtobufUnittest.TestProto3Optional.Types.NestedEnum) input.ReadEnum();
             break;
           }
+          case 176: {
+            SingularInt32 = input.ReadInt32();
+            break;
+          }
+          case 184: {
+            SingularInt64 = input.ReadInt64();
+            break;
+          }
         }
       }
     }

python/google/protobuf/pyext/map_container.cc

@@ -125,9 +125,9 @@ static bool PyStringToSTL(PyObject* py_string, std::string* stl_string) {
   }
 }
 
-static bool PythonToMapKey(PyObject* obj,
-                           const FieldDescriptor* field_descriptor,
-                           MapKey* key) {
+static bool PythonToMapKey(MapContainer* self, PyObject* obj, MapKey* key) {
+  const FieldDescriptor* field_descriptor =
+      self->parent_field_descriptor->message_type()->map_key();
   switch (field_descriptor->cpp_type()) {
     case FieldDescriptor::CPPTYPE_INT32: {
       GOOGLE_CHECK_GET_INT32(obj, value, false);
@@ -171,8 +171,9 @@ static bool PythonToMapKey(PyObject* obj,
   return true;
 }
 
-static PyObject* MapKeyToPython(const FieldDescriptor* field_descriptor,
-                                const MapKey& key) {
+static PyObject* MapKeyToPython(MapContainer* self, const MapKey& key) {
+  const FieldDescriptor* field_descriptor =
+      self->parent_field_descriptor->message_type()->map_key();
   switch (field_descriptor->cpp_type()) {
     case FieldDescriptor::CPPTYPE_INT32:
       return PyInt_FromLong(key.GetInt32Value());
@@ -196,8 +197,9 @@ static PyObject* MapKeyToPython(const FieldDescriptor* field_descriptor,
 
 // This is only used for ScalarMap, so we don't need to handle the
 // CPPTYPE_MESSAGE case.
-PyObject* MapValueRefToPython(const FieldDescriptor* field_descriptor,
-                              const MapValueRef& value) {
+PyObject* MapValueRefToPython(MapContainer* self, const MapValueRef& value) {
+  const FieldDescriptor* field_descriptor =
+      self->parent_field_descriptor->message_type()->map_value();
   switch (field_descriptor->cpp_type()) {
     case FieldDescriptor::CPPTYPE_INT32:
       return PyInt_FromLong(value.GetInt32Value());
@@ -227,10 +229,11 @@ PyObject* MapValueRefToPython(const FieldDescriptor* field_descriptor,
 
 // This is only used for ScalarMap, so we don't need to handle the
 // CPPTYPE_MESSAGE case.
-static bool PythonToMapValueRef(PyObject* obj,
-                                const FieldDescriptor* field_descriptor,
+static bool PythonToMapValueRef(MapContainer* self, PyObject* obj,
                                 bool allow_unknown_enum_values,
                                 MapValueRef* value_ref) {
+  const FieldDescriptor* field_descriptor =
+      self->parent_field_descriptor->message_type()->map_value();
   switch (field_descriptor->cpp_type()) {
     case FieldDescriptor::CPPTYPE_INT32: {
       GOOGLE_CHECK_GET_INT32(obj, value, false);
@@ -357,7 +360,7 @@ PyObject* MapReflectionFriend::Contains(PyObject* _self, PyObject* key) {
   const Reflection* reflection = message->GetReflection();
   MapKey map_key;
 
-  if (!PythonToMapKey(key, self->key_field_descriptor, &map_key)) {
+  if (!PythonToMapKey(self, key, &map_key)) {
     return NULL;
   }
 
@@ -391,18 +394,6 @@ MapContainer* NewScalarMapContainer(
   self->parent_field_descriptor = parent_field_descriptor;
   self->version = 0;
 
-  self->key_field_descriptor =
-      parent_field_descriptor->message_type()->FindFieldByName("key");
-  self->value_field_descriptor =
-      parent_field_descriptor->message_type()->FindFieldByName("value");
-
-  if (self->key_field_descriptor == NULL ||
-      self->value_field_descriptor == NULL) {
-    PyErr_Format(PyExc_KeyError,
-                 "Map entry descriptor did not have key/value fields");
-    return NULL;
-  }
-
   return self;
 }
 
@@ -415,7 +406,7 @@ PyObject* MapReflectionFriend::ScalarMapGetItem(PyObject* _self,
   MapKey map_key;
   MapValueRef value;
 
-  if (!PythonToMapKey(key, self->key_field_descriptor, &map_key)) {
+  if (!PythonToMapKey(self, key, &map_key)) {
     return NULL;
   }
 
@@ -424,7 +415,7 @@ PyObject* MapReflectionFriend::ScalarMapGetItem(PyObject* _self,
     self->version++;
   }
 
-  return MapValueRefToPython(self->value_field_descriptor, value);
+  return MapValueRefToPython(self, value);
 }
 
 int MapReflectionFriend::ScalarMapSetItem(PyObject* _self, PyObject* key,
@@ -436,7 +427,7 @@ int MapReflectionFriend::ScalarMapSetItem(PyObject* _self, PyObject* key,
   MapKey map_key;
   MapValueRef value;
 
-  if (!PythonToMapKey(key, self->key_field_descriptor, &map_key)) {
+  if (!PythonToMapKey(self, key, &map_key)) {
     return -1;
   }
 
@@ -447,10 +438,11 @@ int MapReflectionFriend::ScalarMapSetItem(PyObject* _self, PyObject* key,
     reflection->InsertOrLookupMapValue(message, self->parent_field_descriptor,
                                        map_key, &value);
 
-    return PythonToMapValueRef(v, self->value_field_descriptor,
-                               reflection->SupportsUnknownEnumValues(), &value)
-               ? 0
-               : -1;
+    if (!PythonToMapValueRef(self, v, reflection->SupportsUnknownEnumValues(),
+                             &value)) {
+      return -1;
+    }
+    return 0;
   } else {
     // Delete key from map.
     if (reflection->DeleteMapValue(message, self->parent_field_descriptor,
@@ -505,13 +497,11 @@ PyObject* MapReflectionFriend::ScalarMapToStr(PyObject* _self) {
            message, self->parent_field_descriptor);
        it != reflection->MapEnd(message, self->parent_field_descriptor);
        ++it) {
-    key.reset(MapKeyToPython(self->key_field_descriptor,
-                             it.GetKey()));
+    key.reset(MapKeyToPython(self, it.GetKey()));
     if (key == NULL) {
       return NULL;
     }
-    value.reset(MapValueRefToPython(self->value_field_descriptor,
-                                    it.GetValueRef()));
+    value.reset(MapValueRefToPython(self, it.GetValueRef()));
     if (value == NULL) {
       return NULL;
     }
@@ -655,22 +645,9 @@ MessageMapContainer* NewMessageMapContainer(
   self->parent_field_descriptor = parent_field_descriptor;
   self->version = 0;
 
-  self->key_field_descriptor =
-      parent_field_descriptor->message_type()->FindFieldByName("key");
-  self->value_field_descriptor =
-      parent_field_descriptor->message_type()->FindFieldByName("value");
-
   Py_INCREF(message_class);
   self->message_class = message_class;
 
-  if (self->key_field_descriptor == NULL ||
-      self->value_field_descriptor == NULL) {
-    Py_DECREF(self);
-    PyErr_SetString(PyExc_KeyError,
-                    "Map entry descriptor did not have key/value fields");
-    return NULL;
-  }
-
   return self;
 }
 
@@ -692,7 +669,7 @@ int MapReflectionFriend::MessageMapSetItem(PyObject* _self, PyObject* key,
 
   self->version++;
 
-  if (!PythonToMapKey(key, self->key_field_descriptor, &map_key)) {
+  if (!PythonToMapKey(self, key, &map_key)) {
     return -1;
   }
 
@@ -732,7 +709,7 @@ PyObject* MapReflectionFriend::MessageMapGetItem(PyObject* _self,
   MapKey map_key;
   MapValueRef value;
 
-  if (!PythonToMapKey(key, self->key_field_descriptor, &map_key)) {
+  if (!PythonToMapKey(self, key, &map_key)) {
     return NULL;
   }
 
@@ -759,8 +736,7 @@ PyObject* MapReflectionFriend::MessageMapToStr(PyObject* _self) {
            message, self->parent_field_descriptor);
        it != reflection->MapEnd(message, self->parent_field_descriptor);
        ++it) {
-    key.reset(MapKeyToPython(self->key_field_descriptor,
-                             it.GetKey()));
+    key.reset(MapKeyToPython(self, it.GetKey()));
     if (key == NULL) {
       return NULL;
     }
@@ -961,8 +937,7 @@ PyObject* MapReflectionFriend::IterNext(PyObject* _self) {
     return NULL;
   }
 
-  PyObject* ret = MapKeyToPython(self->container->key_field_descriptor,
-                                 self->iter->GetKey());
+  PyObject* ret = MapKeyToPython(self->container, self->iter->GetKey());
 
   ++(*self->iter);
 

python/google/protobuf/pyext/map_container.h

@@ -54,9 +54,6 @@ struct MapContainer : public ContainerBase {
   // Use to get a mutable message when necessary.
   Message* GetMutableMessage();
 
-  // Cache some descriptors, used to convert keys and values.
-  const FieldDescriptor* key_field_descriptor;
-  const FieldDescriptor* value_field_descriptor;
   // We bump this whenever we perform a mutation, to invalidate existing
   // iterators.
   uint64 version;

src/google/protobuf/compiler/cpp/cpp_unittest.inc

@@ -44,11 +44,10 @@
 // correctly and produces the interfaces we expect, which is why this test
 // is written this way.
 
-#include <google/protobuf/compiler/cpp/cpp_unittest.h>
-
 #include <memory>
 #include <vector>
 
+#include <google/protobuf/compiler/cpp/cpp_unittest.h>
 #include <google/protobuf/unittest_no_arena.pb.h>
 #include <google/protobuf/stubs/strutil.h>
 #if !defined(GOOGLE_PROTOBUF_CMAKE_BUILD) && !defined(_MSC_VER)

src/google/protobuf/compiler/java/java_helpers.cc

@@ -308,7 +308,7 @@ std::string ExtraMessageOrBuilderInterfaces(const Descriptor* descriptor) {
 
 std::string FieldConstantName(const FieldDescriptor* field) {
   std::string name = field->name() + "_FIELD_NUMBER";
-  UpperString(&name);
+  ToUpper(&name);
   return name;
 }
 

src/google/protobuf/compiler/python/python_generator.cc

@@ -584,7 +584,7 @@ void Generator::PrintTopLevelExtensions() const {
   for (int i = 0; i < file_->extension_count(); ++i) {
     const FieldDescriptor& extension_field = *file_->extension(i);
     std::string constant_name = extension_field.name() + "_FIELD_NUMBER";
-    UpperString(&constant_name);
+    ToUpper(&constant_name);
     printer_->Print("$constant_name$ = $number$\n", "constant_name",
                     constant_name, "number",
                     StrCat(extension_field.number()));
@@ -1262,7 +1262,7 @@ std::string Generator::ModuleLevelDescriptorName(
   // The C++ implementation doesn't guard against this either.  Leaving
   // it for now...
   std::string name = NamePrefixedWithNestedTypes(descriptor, "_");
-  UpperString(&name);
+  ToUpper(&name);
   // Module-private for now.  Easy to make public later; almost impossible
   // to make private later.
   name = "_" + name;
@@ -1292,7 +1292,7 @@ std::string Generator::ModuleLevelMessageName(
 std::string Generator::ModuleLevelServiceDescriptorName(
     const ServiceDescriptor& descriptor) const {
   std::string name = descriptor.name();
-  UpperString(&name);
+  ToUpper(&name);
   name = "_" + name;
   if (descriptor.file() != file_) {
     name = ModuleAlias(descriptor.file()->name()) + "." + name;

src/google/protobuf/descriptor.cc

@@ -1341,6 +1341,10 @@ DescriptorPool* NewGeneratedPool() {
 
 }  // anonymous namespace
 
+DescriptorDatabase* DescriptorPool::internal_generated_database() {
+  return GeneratedDatabase();
+}
+
 DescriptorPool* DescriptorPool::internal_generated_pool() {
   static DescriptorPool* generated_pool =
       internal::OnShutdownDelete(NewGeneratedPool());

src/google/protobuf/descriptor.h

@@ -1842,6 +1842,11 @@ class PROTOBUF_EXPORT DescriptorPool {
   // the underlay takes precedence.
   static DescriptorPool* internal_generated_pool();
 
+  // For internal use only:  Gets a non-const pointer to the generated
+  // descriptor database.
+  // Only used for testing.
+  static DescriptorDatabase* internal_generated_database();
+
   // For internal use only:  Changes the behavior of BuildFile() such that it
   // allows the file to make reference to message types declared in other files
   // which it did not officially declare as dependencies.

src/google/protobuf/descriptor_database_unittest.cc

@@ -34,20 +34,21 @@
 //
 // This file makes extensive use of RFC 3092.  :)
 
+#include <google/protobuf/descriptor_database.h>
+
 #include <algorithm>
 #include <memory>
 
+#include <google/protobuf/stubs/logging.h>
+#include <google/protobuf/stubs/common.h>
 #include <google/protobuf/descriptor.pb.h>
 #include <google/protobuf/descriptor.h>
-#include <google/protobuf/descriptor_database.h>
 #include <google/protobuf/text_format.h>
-
-#include <google/protobuf/stubs/logging.h>
-#include <google/protobuf/stubs/common.h>
 #include <gmock/gmock.h>
 #include <google/protobuf/testing/googletest.h>
 #include <gtest/gtest.h>
 
+
 namespace google {
 namespace protobuf {
 namespace {
@@ -798,6 +799,7 @@ TEST_F(MergedDescriptorDatabaseTest, FindAllExtensionNumbers) {
   }
 }
 
+
 }  // anonymous namespace
 }  // namespace protobuf
 }  // namespace google

src/google/protobuf/generated_message_reflection.cc

@@ -1061,7 +1061,7 @@ void Reflection::ListFields(const Message& message,
         if (oneof_case_array[containing_oneof->index()] == field->number()) {
           output->push_back(field);
         }
-      } else if (has_bits) {
+      } else if (has_bits && has_bits_indices[i] != -1) {
         // Equivalent to: HasBit(message, field)
         if (IsIndexInHasBitSet(has_bits, has_bits_indices[i])) {
           output->push_back(field);

src/google/protobuf/map_test.cc

@@ -2139,7 +2139,7 @@ TEST(GeneratedMapFieldTest, SerializationToArray) {
   unittest::TestMap message1, message2;
   std::string data;
   MapTestUtil::SetMapFields(&message1);
-  int size = message1.ByteSize();
+  size_t size = message1.ByteSizeLong();
   data.resize(size);
   uint8* start = reinterpret_cast<uint8*>(::google::protobuf::string_as_array(&data));
   uint8* end = message1.SerializeWithCachedSizesToArray(start);
@@ -2152,7 +2152,7 @@ TEST(GeneratedMapFieldTest, SerializationToArray) {
 TEST(GeneratedMapFieldTest, SerializationToStream) {
   unittest::TestMap message1, message2;
   MapTestUtil::SetMapFields(&message1);
-  int size = message1.ByteSize();
+  size_t size = message1.ByteSizeLong();
   std::string data;
   data.resize(size);
   {
@@ -2369,7 +2369,7 @@ TEST(GeneratedMapFieldTest, MissedValueTextFormat) {
 
   EXPECT_TRUE(TextFormat::ParseFromString(text, &message));
   EXPECT_EQ(1, message.map_int32_foreign_message().size());
-  EXPECT_EQ(11, message.ByteSize());
+  EXPECT_EQ(11, message.ByteSizeLong());
 }
 
 TEST(GeneratedMapFieldTest, UnknownFieldWireFormat) {
@@ -2481,7 +2481,7 @@ TEST(GeneratedMapFieldReflectionTest, SpaceUsed) {
   MapReflectionTester reflection_tester(unittest::TestMap::descriptor());
   reflection_tester.SetMapFieldsViaReflection(&message);
 
-  EXPECT_LT(0, message.GetReflection()->SpaceUsed(message));
+  EXPECT_LT(0, message.GetReflection()->SpaceUsedLong(message));
 }
 
 TEST(GeneratedMapFieldReflectionTest, Accessors) {
@@ -2767,7 +2767,7 @@ TEST_F(MapFieldInDynamicMessageTest, DynamicMapReflection) {
 }
 
 TEST_F(MapFieldInDynamicMessageTest, MapSpaceUsed) {
-  // Test that SpaceUsed() works properly
+  // Test that SpaceUsedLong() works properly
 
   // Since we share the implementation with generated messages, we don't need
   // to test very much here.  Just make sure it appears to be working.
@@ -2775,10 +2775,10 @@ TEST_F(MapFieldInDynamicMessageTest, MapSpaceUsed) {
   std::unique_ptr<Message> message(map_prototype_->New());
   MapReflectionTester reflection_tester(map_descriptor_);
 
-  int initial_space_used = message->SpaceUsed();
+  int initial_space_used = message->SpaceUsedLong();
 
   reflection_tester.SetMapFieldsViaReflection(message.get());
-  EXPECT_LT(initial_space_used, message->SpaceUsed());
+  EXPECT_LT(initial_space_used, message->SpaceUsedLong());
 }
 
 TEST_F(MapFieldInDynamicMessageTest, RecursiveMap) {
@@ -2952,9 +2952,9 @@ TEST(WireFormatForMapFieldTest, MapByteSize) {
   unittest::TestMap message;
   MapTestUtil::SetMapFields(&message);
 
-  EXPECT_EQ(message.ByteSize(), WireFormat::ByteSize(message));
+  EXPECT_EQ(message.ByteSizeLong(), WireFormat::ByteSize(message));
   message.Clear();
-  EXPECT_EQ(0, message.ByteSize());
+  EXPECT_EQ(0, message.ByteSizeLong());
   EXPECT_EQ(0, WireFormat::ByteSize(message));
 }
 
@@ -2967,7 +2967,7 @@ TEST(WireFormatForMapFieldTest, SerializeMap) {
 
   // Serialize using the generated code.
   {
-    message.ByteSize();
+    message.ByteSizeLong();
     io::StringOutputStream raw_output(&generated_data);
     io::CodedOutputStream output(&raw_output);
     message.SerializeWithCachedSizes(&output);
@@ -2978,7 +2978,7 @@ TEST(WireFormatForMapFieldTest, SerializeMap) {
   {
     io::StringOutputStream raw_output(&dynamic_data);
     io::CodedOutputStream output(&raw_output);
-    int size = WireFormat::ByteSize(message);
+    size_t size = WireFormat::ByteSize(message);
     WireFormat::SerializeWithCachedSizes(message, size, &output);
     ASSERT_FALSE(output.HadError());
   }
@@ -3026,7 +3026,7 @@ TEST(WireFormatForMapFieldTest, MapByteSizeDynamicMessage) {
   std::string expected_serialized_data;
   dynamic_message->SerializeToString(&expected_serialized_data);
   int expected_size = expected_serialized_data.size();
-  EXPECT_EQ(dynamic_message->ByteSize(), expected_size);
+  EXPECT_EQ(dynamic_message->ByteSizeLong(), expected_size);
 
   std::unique_ptr<Message> message2;
   message2.reset(factory.GetPrototype(unittest::TestMap::descriptor())->New());
@@ -3040,27 +3040,27 @@ TEST(WireFormatForMapFieldTest, MapByteSizeDynamicMessage) {
   reflection->RemoveLast(dynamic_message.get(), field);
   dynamic_message->MergeFrom(*message2);
   dynamic_message->MergeFrom(*message2);
-  // The map field is marked as STATE_MODIFIED_REPEATED, ByteSize() will use
+  // The map field is marked as STATE_MODIFIED_REPEATED, ByteSizeLong() will use
   // repeated field which have duplicate keys to calculate.
-  int duplicate_size = dynamic_message->ByteSize();
+  size_t duplicate_size = dynamic_message->ByteSizeLong();
   EXPECT_TRUE(duplicate_size > expected_size);
   std::string duplicate_serialized_data;
   dynamic_message->SerializeToString(&duplicate_serialized_data);
-  EXPECT_EQ(dynamic_message->ByteSize(), duplicate_serialized_data.size());
+  EXPECT_EQ(dynamic_message->ByteSizeLong(), duplicate_serialized_data.size());
 
   // Force the map field to mark with map CLEAN
   EXPECT_EQ(reflection_tester.MapSize(*dynamic_message, "map_int32_int32"), 2);
-  // The map field is marked as CLEAN, ByteSize() will use map which do not
+  // The map field is marked as CLEAN, ByteSizeLong() will use map which do not
   // have duplicate keys to calculate.
-  int size = dynamic_message->ByteSize();
+  int size = dynamic_message->ByteSizeLong();
   EXPECT_EQ(expected_size, size);
 
   // Protobuf used to have a bug for serialize when map it marked CLEAN. It used
-  // repeated field to calculate ByteSize but use map to serialize the real
-  // data, thus the ByteSize may bigger than real serialized size. A crash might
-  // be happen at SerializeToString(). Or an "unexpect end group" warning was
-  // raised at parse back if user use SerializeWithCachedSizes() which avoids
-  // size check at serialize.
+  // repeated field to calculate ByteSizeLong but use map to serialize the real
+  // data, thus the ByteSizeLong may bigger than real serialized size. A crash
+  // might be happen at SerializeToString(). Or an "unexpect end group" warning
+  // was raised at parse back if user use SerializeWithCachedSizes() which
+  // avoids size check at serialize.
   std::string serialized_data;
   dynamic_message->SerializeToString(&serialized_data);
   EXPECT_EQ(serialized_data, expected_serialized_data);
@@ -3118,7 +3118,7 @@ TEST(WireFormatForMapFieldTest, MapParseHelpers) {
 template <typename T>
 static std::string DeterministicSerializationWithSerializePartialToCodedStream(
     const T& t) {
-  const int size = t.ByteSize();
+  const size_t size = t.ByteSizeLong();
   std::string result(size, '\0');
   io::ArrayOutputStream array_stream(::google::protobuf::string_as_array(&result), size);
   io::CodedOutputStream output_stream(&array_stream);
@@ -3132,7 +3132,7 @@ static std::string DeterministicSerializationWithSerializePartialToCodedStream(
 template <typename T>
 static std::string DeterministicSerializationWithSerializeToCodedStream(
     const T& t) {
-  const int size = t.ByteSize();
+  const size_t size = t.ByteSizeLong();
   std::string result(size, '\0');
   io::ArrayOutputStream array_stream(::google::protobuf::string_as_array(&result), size);
   io::CodedOutputStream output_stream(&array_stream);
@@ -3145,7 +3145,7 @@ static std::string DeterministicSerializationWithSerializeToCodedStream(
 
 template <typename T>
 static std::string DeterministicSerialization(const T& t) {
-  const int size = t.ByteSize();
+  const size_t size = t.ByteSizeLong();
   std::string result(size, '\0');
   io::ArrayOutputStream array_stream(::google::protobuf::string_as_array(&result), size);
   {

src/google/protobuf/port_def.inc

@@ -547,14 +547,26 @@ PROTOBUF_EXPORT_TEMPLATE_TEST(DEFAULT, __declspec(dllimport));
 // Windows declares several inconvenient macro names.  We #undef them and then
 // restore them in port_undef.inc.
 #ifdef _MSC_VER
+#pragma push_macro("CREATE_NEW")
+#undef CREATE_NEW
+#pragma push_macro("DOUBLE_CLICK")
+#undef DOUBLE_CLICK
 #pragma push_macro("ERROR")
 #undef ERROR
+#pragma push_macro("ERROR_BUSY")
+#undef ERROR_BUSY
+#pragma push_macro("ERROR_NOT_FOUND")
+#undef ERROR_NOT_FOUND
 #pragma push_macro("GetMessage")
 #undef GetMessage
 #pragma push_macro("IGNORE")
 #undef IGNORE
 #pragma push_macro("IN")
 #undef IN
+#pragma push_macro("INPUT_KEYBOARD")
+#undef INPUT_KEYBOARD
+#pragma push_macro("NO_ERROR")
+#undef NO_ERROR
 #pragma push_macro("OUT")
 #undef OUT
 #pragma push_macro("OPTIONAL")
@@ -563,6 +575,14 @@ PROTOBUF_EXPORT_TEMPLATE_TEST(DEFAULT, __declspec(dllimport));
 #undef min
 #pragma push_macro("max")
 #undef max
+#pragma push_macro("REASON_UNKNOWN")
+#undef REASON_UNKNOWN
+#pragma push_macro("SERVICE_DISABLED")
+#undef SERVICE_DISABLED
+#pragma push_macro("SEVERITY_ERROR")
+#undef SEVERITY_ERROR
+#pragma push_macro("STRICT")
+#undef STRICT
 #endif  // _MSC_VER
 
 #if defined(__clang__) || defined(__GNUC__) || defined(_MSC_VER)

src/google/protobuf/port_undef.inc

@@ -87,14 +87,24 @@
 
 // Restore macro that may have been #undef'd in port_def.inc.
 #ifdef _MSC_VER
+#pragma pop_macro("CREATE_NEW")
+#pragma pop_macro("DOUBLE_CLICK")
 #pragma pop_macro("ERROR")
+#pragma pop_macro("ERROR_BUSY")
+#pragma pop_macro("ERROR_NOT_FOUND")
 #pragma pop_macro("GetMessage")
 #pragma pop_macro("IGNORE")
 #pragma pop_macro("IN")
+#pragma pop_macro("INPUT_KEYBOARD")
 #pragma pop_macro("OUT")
 #pragma pop_macro("OPTIONAL")
 #pragma pop_macro("min")
 #pragma pop_macro("max")
+#pragma pop_macro("NO_ERROR")
+#pragma pop_macro("REASON_UNKNOWN")
+#pragma pop_macro("SERVICE_DISABLED")
+#pragma pop_macro("SEVERITY_ERROR")
+#pragma pop_macro("STRICT")
 #endif
 
 #if defined(__clang__) || defined(__GNUC__) || defined(_MSC_VER)

src/google/protobuf/proto3_arena_unittest.cc

@@ -40,6 +40,7 @@
 #include <google/protobuf/text_format.h>
 #include <google/protobuf/testing/googletest.h>
 #include <gtest/gtest.h>
+#include <google/protobuf/stubs/strutil.h>
 
 using proto3_arena_unittest::TestAllTypes;
 
@@ -217,9 +218,15 @@ TEST(Proto3OptionalTest, OptionalFieldDescriptor) {
 
   for (int i = 0; i < d->field_count(); i++) {
     const FieldDescriptor* f = d->field(i);
-    EXPECT_TRUE(f->has_optional_keyword()) << f->full_name();
-    EXPECT_TRUE(f->has_presence()) << f->full_name();
-    EXPECT_TRUE(f->containing_oneof()) << f->full_name();
+    if (HasPrefixString(f->name(), "singular")) {
+      EXPECT_FALSE(f->has_optional_keyword()) << f->full_name();
+      EXPECT_FALSE(f->has_presence()) << f->full_name();
+      EXPECT_FALSE(f->containing_oneof()) << f->full_name();
+    } else {
+      EXPECT_TRUE(f->has_optional_keyword()) << f->full_name();
+      EXPECT_TRUE(f->has_presence()) << f->full_name();
+      EXPECT_TRUE(f->containing_oneof()) << f->full_name();
+    }
   }
 }
 

src/google/protobuf/stubs/strutil.h

@@ -194,6 +194,8 @@ inline void UpperString(string * s) {
   }
 }
 
+inline void ToUpper(string* s) { UpperString(s); }
+
 inline string ToUpper(const string& s) {
   string out = s;
   UpperString(&out);

src/google/protobuf/unittest_proto3_optional.proto

@@ -72,4 +72,8 @@ message TestProto3Optional {
   optional NestedMessage optional_nested_message = 18;
   optional NestedMessage lazy_nested_message = 19 [lazy = true];
   optional NestedEnum optional_nested_enum = 21;
+
+  // Add some non-optional fields to verify we can mix them.
+  int32 singular_int32 = 22;
+  int64 singular_int64 = 23;
 }

src/google/protobuf/util/internal/default_value_objectwriter_test.cc

@@ -29,6 +29,7 @@
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <google/protobuf/util/internal/default_value_objectwriter.h>
+
 #include <google/protobuf/util/internal/expecting_objectwriter.h>
 #include <google/protobuf/util/internal/testdata/default_value_test.pb.h>
 #include <google/protobuf/util/internal/type_info_test_helper.h>

src/google/protobuf/util/internal/protostream_objectwriter_test.cc

@@ -172,7 +172,7 @@ class BaseProtoStreamObjectWriterTest
 MATCHER_P(HasObjectLocation, expected,
           "Verifies the expected object location") {
   std::string actual = get<0>(arg).ToString();
-  if (actual.compare(expected) == 0) return true;
+  if (actual == expected) return true;
   *result_listener << "actual location is: " << actual;
   return false;
 }