Merge remote-tracking branch 'upstream/master' into protobuf_buffer_serialization

CHANGES.txt

@@ -3,15 +3,24 @@ Unreleased Changes
   C++:
   * Removed deprecated unsafe arena string accessors
   * Enabled heterogeneous lookup for std::string keys in maps.
-  * Improved the randomness of map ordering.
   * Removed implicit conversion from StringPiece to std::string
   * Fix use-after-destroy bug when the Map is allocated in the arena.
-  * Improved the randomness of proto map ordering
+  * Improved the randomness of map ordering
+  * Added stack overflow protection for text format with unknown fields
+  * Use std::hash for proto maps to help with portability.
+  * Added more Windows macros to proto whitelist.
+  * Arena constructors for map entry messages are now marked "explicit"
+    (for regular messages they were already explicit).
 
   Python:
   * Reject lowercase t for Timestamp json format. Fixes a conformance test.
   * Improved the error message when AttributeError is returned from __getattr__
     in EnumTypeWrapper.
+  * Json format will print full_name directly for extensions.
+
+  Java:
+  * Fixed a bug where setting optional proto3 enums with setFooValue() would
+    not mark the value as present.
 
   C#:
   * Dropped support for netstandard1.0 (replaced by support for netstandard1.1).
@@ -24,6 +33,11 @@ Unreleased Changes
     from .NET framework and old versions of mono) that do not support
     ref structs. (#7490)
 
+2020-06-01 version 3.12.3 (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript)
+
+  Objective-C
+  * Tweak the union used for Extensions to support old generated code. #7573
+
 2020-05-26 version 3.12.2 (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript)
 
   C++

Makefile.am

@@ -917,6 +917,7 @@ php_EXTRA_DIST=                                                       \
   php/tests/descriptors_test.php                                      \
   php/tests/encode_decode_test.php                                    \
   php/tests/gdb_test.sh                                               \
+  php/tests/generate_protos.sh                                        \
   php/tests/generated_class_test.php                                  \
   php/tests/generated_phpdoc_test.php                                 \
   php/tests/generated_service_test.php                                \

Protobuf-C++.podspec

@@ -1,6 +1,6 @@
 Pod::Spec.new do |s|
   s.name     = 'Protobuf-C++'
-  s.version  = '3.12.2'
+  s.version  = '3.12.3'
   s.summary  = 'Protocol Buffers v3 runtime library for C++.'
   s.homepage = 'https://github.com/google/protobuf'
   s.license  = '3-Clause BSD License'

Protobuf.podspec

@@ -5,7 +5,7 @@
 # dependent projects use the :git notation to refer to the library.
 Pod::Spec.new do |s|
   s.name     = 'Protobuf'
-  s.version  = '3.12.2'
+  s.version  = '3.12.3'
   s.summary  = 'Protocol Buffers v.3 runtime library for Objective-C.'
   s.homepage = 'https://github.com/protocolbuffers/protobuf'
   s.license  = '3-Clause BSD License'

configure.ac

@@ -17,7 +17,7 @@ AC_PREREQ(2.59)
 # In the SVN trunk, the version should always be the next anticipated release
 # version with the "-pre" suffix.  (We used to use "-SNAPSHOT" but this pushed
 # the size of one file name in the dist tarfile over the 99-char limit.)
-AC_INIT([Protocol Buffers],[3.12.2],[protobuf@googlegroups.com],[protobuf])
+AC_INIT([Protocol Buffers],[3.12.3],[protobuf@googlegroups.com],[protobuf])
 
 AM_MAINTAINER_MODE([enable])
 

conformance/failure_list_csharp.txt

@@ -1,4 +1,3 @@
 Recommended.Proto3.JsonInput.BytesFieldBase64Url.JsonOutput
 Recommended.Proto3.JsonInput.BytesFieldBase64Url.ProtobufOutput
-Required.Proto2.JsonInput.StoresDefaultPrimitive.Validator
 Recommended.Proto2.JsonInput.FieldNameExtension.Validator

csharp/Google.Protobuf.Tools.nuspec

@@ -5,7 +5,7 @@
     <title>Google Protocol Buffers tools</title>
     <summary>Tools for Protocol Buffers - Google's data interchange format.</summary>
     <description>See project site for more info.</description>
-    <version>3.12.2</version>
+    <version>3.12.3</version>
     <authors>Google Inc.</authors>
     <owners>protobuf-packages</owners>
     <licenseUrl>https://github.com/protocolbuffers/protobuf/blob/master/LICENSE</licenseUrl>

csharp/src/Google.Protobuf.Benchmarks/ParseMessagesBenchmark.cs

@@ -37,6 +37,7 @@ using System.IO;
 using System.Linq;
 using System.Buffers;
 using Google.Protobuf.WellKnownTypes;
+using Benchmarks.Proto3;
 
 namespace Google.Protobuf.Benchmarks
 {
@@ -50,6 +51,7 @@ namespace Google.Protobuf.Benchmarks
 
         SubTest manyWrapperFieldsTest = new SubTest(CreateManyWrapperFieldsMessage(), ManyWrapperFieldsMessage.Parser, () => new ManyWrapperFieldsMessage(), MaxMessages);
         SubTest manyPrimitiveFieldsTest = new SubTest(CreateManyPrimitiveFieldsMessage(), ManyPrimitiveFieldsMessage.Parser, () => new ManyPrimitiveFieldsMessage(), MaxMessages);
+        SubTest repeatedFieldTest = new SubTest(CreateRepeatedFieldMessage(), GoogleMessage1.Parser, () => new GoogleMessage1(), MaxMessages);
         SubTest emptyMessageTest = new SubTest(new Empty(), Empty.Parser, () => new Empty(), MaxMessages);
 
         public IEnumerable<int> MessageCountValues => new[] { 10, 100 };
@@ -83,6 +85,18 @@ namespace Google.Protobuf.Benchmarks
             return manyPrimitiveFieldsTest.ParseFromReadOnlySequence();
         }
 
+        [Benchmark]
+        public IMessage RepeatedFieldMessage_ParseFromByteArray()
+        {
+            return repeatedFieldTest.ParseFromByteArray();
+        }
+
+        [Benchmark]
+        public IMessage RepeatedFieldMessage_ParseFromReadOnlySequence()
+        {
+            return repeatedFieldTest.ParseFromReadOnlySequence();
+        }
+
         [Benchmark]
         public IMessage EmptyMessage_ParseFromByteArray()
         {
@@ -123,6 +137,20 @@ namespace Google.Protobuf.Benchmarks
             manyPrimitiveFieldsTest.ParseDelimitedMessagesFromReadOnlySequence(messageCount);
         }
 
+        [Benchmark]
+        [ArgumentsSource(nameof(MessageCountValues))]
+        public void RepeatedFieldMessage_ParseDelimitedMessagesFromByteArray(int messageCount)
+        {
+            repeatedFieldTest.ParseDelimitedMessagesFromByteArray(messageCount);
+        }
+
+        [Benchmark]
+        [ArgumentsSource(nameof(MessageCountValues))]
+        public void RepeatedFieldMessage_ParseDelimitedMessagesFromReadOnlySequence(int messageCount)
+        {
+            repeatedFieldTest.ParseDelimitedMessagesFromReadOnlySequence(messageCount);
+        }
+
         public static ManyWrapperFieldsMessage CreateManyWrapperFieldsMessage()
         {
             // Example data match data of an internal benchmarks
@@ -157,6 +185,17 @@ namespace Google.Protobuf.Benchmarks
             };
         }
 
+        public static GoogleMessage1 CreateRepeatedFieldMessage()
+        {
+            // Message with a repeated fixed length item collection
+            var message = new GoogleMessage1();
+            for (ulong i = 0; i < 1000; i++)
+            {
+                message.Field5.Add(i);
+            }
+            return message;
+        }
+
         private class SubTest
         {
             private readonly IMessage message;

csharp/src/Google.Protobuf.Conformance/Program.cs

@@ -83,44 +83,52 @@ namespace Google.Protobuf.Conformance
 
         private static ConformanceResponse PerformRequest(ConformanceRequest request, TypeRegistry typeRegistry)
         {
+            ExtensionRegistry proto2ExtensionRegistry = new ExtensionRegistry
+            {
+                ProtobufTestMessages.Proto2.TestMessagesProto2Extensions.ExtensionInt32,
+                ProtobufTestMessages.Proto2.TestAllTypesProto2.Types.MessageSetCorrectExtension1.Extensions.MessageSetExtension,
+                ProtobufTestMessages.Proto2.TestAllTypesProto2.Types.MessageSetCorrectExtension2.Extensions.MessageSetExtension
+            };
             IMessage message;
             try
             {
                 switch (request.PayloadCase)
                 {
                     case ConformanceRequest.PayloadOneofCase.JsonPayload:
-                        if (request.TestCategory == global::Conformance.TestCategory.JsonIgnoreUnknownParsingTest) {
+                        if (request.TestCategory == global::Conformance.TestCategory.JsonIgnoreUnknownParsingTest)
+                        {
                             return new ConformanceResponse { Skipped = "CSharp doesn't support skipping unknown fields in json parsing." };
                         }
                         var parser = new JsonParser(new JsonParser.Settings(20, typeRegistry));
-                        message = parser.Parse<ProtobufTestMessages.Proto3.TestAllTypesProto3>(request.JsonPayload);
-                        break;
-                    case ConformanceRequest.PayloadOneofCase.ProtobufPayload:
-                    {
-                        if (request.MessageType.Equals("protobuf_test_messages.proto3.TestAllTypesProto3"))
+                        switch (request.MessageType)
                         {
-                            message = ProtobufTestMessages.Proto3.TestAllTypesProto3.Parser.ParseFrom(request.ProtobufPayload);
+                            case "protobuf_test_messages.proto3.TestAllTypesProto3":
+                                message = parser.Parse<ProtobufTestMessages.Proto3.TestAllTypesProto3>(request.JsonPayload);
+                                break;
+                            case "protobuf_test_messages.proto2.TestAllTypesProto2":
+                                message = parser.Parse<ProtobufTestMessages.Proto2.TestAllTypesProto2>(request.JsonPayload);
+                                break;
+                            default:
+                                throw new Exception($" Protobuf request doesn't have specific payload type ({request.MessageType})");
                         }
-                        else if (request.MessageType.Equals("protobuf_test_messages.proto2.TestAllTypesProto2"))
-                        {
-                            ExtensionRegistry registry = new ExtensionRegistry()
-                            {
-                                ProtobufTestMessages.Proto2.TestMessagesProto2Extensions.ExtensionInt32,
-                                ProtobufTestMessages.Proto2.TestAllTypesProto2.Types.MessageSetCorrectExtension1.Extensions.MessageSetExtension,
-                                ProtobufTestMessages.Proto2.TestAllTypesProto2.Types.MessageSetCorrectExtension2.Extensions.MessageSetExtension
-                            };
-                            message = ProtobufTestMessages.Proto2.TestAllTypesProto2.Parser.WithExtensionRegistry(registry).ParseFrom(request.ProtobufPayload);
-                        }
-                        else
+                        break;
+                    case ConformanceRequest.PayloadOneofCase.ProtobufPayload:
+                        switch (request.MessageType)
                         {
-                            throw new Exception(" Protobuf request doesn't have specific payload type");
+                            case "protobuf_test_messages.proto3.TestAllTypesProto3":
+                                message = ProtobufTestMessages.Proto3.TestAllTypesProto3.Parser.ParseFrom(request.ProtobufPayload);
+                                break;
+                            case "protobuf_test_messages.proto2.TestAllTypesProto2":
+                                message = ProtobufTestMessages.Proto2.TestAllTypesProto2.Parser
+                                    .WithExtensionRegistry(proto2ExtensionRegistry)
+                                    .ParseFrom(request.ProtobufPayload);
+                                break;
+                            default:
+                                throw new Exception($" Protobuf request doesn't have specific payload type ({request.MessageType})");
                         }
                         break;
-                    }
 					case ConformanceRequest.PayloadOneofCase.TextPayload:
-					{
 						return new ConformanceResponse { Skipped = "CSharp doesn't support text format" };
-					}
                     default:
                         throw new Exception("Unsupported request payload: " + request.PayloadCase);
                 }

csharp/src/Google.Protobuf.Test/Collections/RepeatedFieldTest.cs

@@ -595,6 +595,95 @@ namespace Google.Protobuf.Collections
             Assert.AreEqual(((SampleEnum)(-5)), values[5]);
         }
 
+        [Test]
+        public void TestPackedRepeatedFieldCollectionNonDivisibleLength()
+        {
+            uint tag = WireFormat.MakeTag(10, WireFormat.WireType.LengthDelimited);
+            var codec = FieldCodec.ForFixed32(tag);
+            var stream = new MemoryStream();
+            var output = new CodedOutputStream(stream);
+            output.WriteTag(tag);
+            output.WriteString("A long string");
+            output.WriteTag(codec.Tag);
+            output.WriteRawVarint32((uint)codec.FixedSize - 1); // Length not divisible by FixedSize
+            output.WriteFixed32(uint.MaxValue);
+            output.Flush();
+            stream.Position = 0;
+
+            var input = new CodedInputStream(stream);
+            input.ReadTag();
+            input.ReadString();
+            input.ReadTag();
+            var field = new RepeatedField<uint>();
+            Assert.Throws<InvalidProtocolBufferException>(() => field.AddEntriesFrom(input, codec));
+
+            // Collection was not pre-initialized
+            Assert.AreEqual(0, field.Count);
+        }
+
+        [Test]
+        public void TestPackedRepeatedFieldCollectionNotAllocatedWhenLengthExceedsBuffer()
+        {
+            uint tag = WireFormat.MakeTag(10, WireFormat.WireType.LengthDelimited);
+            var codec = FieldCodec.ForFixed32(tag);
+            var stream = new MemoryStream();
+            var output = new CodedOutputStream(stream);
+            output.WriteTag(tag);
+            output.WriteString("A long string");
+            output.WriteTag(codec.Tag);
+            output.WriteRawVarint32((uint)codec.FixedSize);
+            // Note that there is no content for the packed field.
+            // The field length exceeds the remaining length of content.
+            output.Flush();
+            stream.Position = 0;
+
+            var input = new CodedInputStream(stream);
+            input.ReadTag();
+            input.ReadString();
+            input.ReadTag();
+            var field = new RepeatedField<uint>();
+            Assert.Throws<InvalidProtocolBufferException>(() => field.AddEntriesFrom(input, codec));
+
+            // Collection was not pre-initialized
+            Assert.AreEqual(0, field.Count);
+        }
+
+        [Test]
+        public void TestPackedRepeatedFieldCollectionNotAllocatedWhenLengthExceedsRemainingData()
+        {
+            uint tag = WireFormat.MakeTag(10, WireFormat.WireType.LengthDelimited);
+            var codec = FieldCodec.ForFixed32(tag);
+            var stream = new MemoryStream();
+            var output = new CodedOutputStream(stream);
+            output.WriteTag(tag);
+            output.WriteString("A long string");
+            output.WriteTag(codec.Tag);
+            output.WriteRawVarint32((uint)codec.FixedSize);
+            // Note that there is no content for the packed field.
+            // The field length exceeds the remaining length of the buffer.
+            output.Flush();
+            stream.Position = 0;
+
+            var sequence = ReadOnlySequenceFactory.CreateWithContent(stream.ToArray());
+            ParseContext.Initialize(sequence, out ParseContext ctx);
+
+            ctx.ReadTag();
+            ctx.ReadString();
+            ctx.ReadTag();
+            var field = new RepeatedField<uint>();
+            try
+            {
+                field.AddEntriesFrom(ref ctx, codec);
+                Assert.Fail();
+            }
+            catch (InvalidProtocolBufferException)
+            {
+            }
+
+            // Collection was not pre-initialized
+            Assert.AreEqual(0, field.Count);
+        }
+
         // Fairly perfunctory tests for the non-generic IList implementation
         [Test]
         public void IList_Indexer()

csharp/src/Google.Protobuf.Test/JsonFormatterTest.cs

@@ -40,6 +40,7 @@ using Google.Protobuf.Reflection;
 using static Google.Protobuf.JsonParserTest; // For WrapInQuotes
 using System.IO;
 using Google.Protobuf.Collections;
+using ProtobufUnittest;
 
 namespace Google.Protobuf
 {
@@ -153,6 +154,48 @@ namespace Google.Protobuf
             AssertJson(expectedText, actualText);
         }
 
+        [Test]
+        public void WithFormatDefaultValues_DoesNotAffectMessageFields()
+        {
+            var message = new TestAllTypes();
+            var formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithFormatDefaultValues(true));
+            var json = formatter.Format(message);
+            Assert.IsFalse(json.Contains("\"singleNestedMessage\""));
+            Assert.IsFalse(json.Contains("\"singleForeignMessage\""));
+            Assert.IsFalse(json.Contains("\"singleImportMessage\""));
+        }
+
+        [Test]
+        public void WithFormatDefaultValues_DoesNotAffectProto3OptionalFields()
+        {
+            var message = new TestProto3Optional();
+            message.OptionalInt32 = 0;
+            var formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithFormatDefaultValues(true));
+            var json = formatter.Format(message);
+            // The non-optional proto3 fields are formatted, as is the optional-but-specified field.
+            AssertJson("{ 'optionalInt32': 0, 'singularInt32': 0, 'singularInt64': '0' }", json);
+        }
+
+        [Test]
+        public void WithFormatDefaultValues_DoesNotAffectProto2Fields()
+        {
+            var message = new TestProtos.Proto2.ForeignMessage();
+            message.C = 0;
+            var formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithFormatDefaultValues(true));
+            var json = formatter.Format(message);
+            // The specified field is formatted, but the non-specified field (d) is not.
+            AssertJson("{ 'c': 0 }", json);
+        }
+
+        [Test]
+        public void WithFormatDefaultValues_DoesNotAffectOneofFields()
+        {
+            var message = new TestOneof();
+            var formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithFormatDefaultValues(true));
+            var json = formatter.Format(message);
+            AssertJson("{ }", json);
+        }
+
         [Test]
         public void RepeatedField()
         {
@@ -313,14 +356,16 @@ namespace Google.Protobuf
         }
 
         [Test]
-        public void WrapperFormatting_IncludeNull()
+        public void WrapperFormatting_FormatDefaultValuesDoesNotFormatNull()
         {
             // The actual JSON here is very large because there are lots of fields. Just test a couple of them.
             var message = new TestWellKnownTypes { Int32Field = 10 };
             var formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithFormatDefaultValues(true));
             var actualJson = formatter.Format(message);
-            Assert.IsTrue(actualJson.Contains("\"int64Field\": null"));
-            Assert.IsFalse(actualJson.Contains("\"int32Field\": null"));
+            // This *used* to include "int64Field": null, but that was a bug.
+            // WithDefaultValues should not affect message fields, including wrapper types.
+            Assert.IsFalse(actualJson.Contains("\"int64Field\": null"));
+            Assert.IsTrue(actualJson.Contains("\"int32Field\": 10"));
         }
 
         [Test]
@@ -602,6 +647,13 @@ namespace Google.Protobuf
             AssertWriteValue(value, "[ 1, 2, 3 ]");
         }
 
+        [Test]
+        public void Proto2_DefaultValuesWritten()
+        {
+            var value = new ProtobufTestMessages.Proto2.TestAllTypesProto2() { FieldName13 = 0 };
+            AssertWriteValue(value, "{ 'FieldName13': 0 }");
+        }
+
         private static void AssertWriteValue(object value, string expectedJson)
         {
             var writer = new StringWriter();

csharp/src/Google.Protobuf.Test/JsonParserTest.cs

@@ -34,6 +34,7 @@ using Google.Protobuf.Reflection;
 using Google.Protobuf.TestProtos;
 using Google.Protobuf.WellKnownTypes;
 using NUnit.Framework;
+using ProtobufTestMessages.Proto2;
 using System;
 
 namespace Google.Protobuf
@@ -949,6 +950,16 @@ namespace Google.Protobuf
             Assert.Throws<InvalidProtocolBufferException>(() => TestAllTypes.Parser.ParseJson(json));
         }
 
+        [Test]
+        public void Proto2_DefaultValuesPreserved()
+        {
+            string json = "{ \"FieldName13\": 0 }";
+            var parsed = TestAllTypesProto2.Parser.ParseJson(json);
+            Assert.False(parsed.HasFieldName10);
+            Assert.True(parsed.HasFieldName13);
+            Assert.AreEqual(0, parsed.FieldName13);
+        }
+
         [Test]
         [TestCase("5")]
         [TestCase("\"text\"")]

csharp/src/Google.Protobuf.Test/ReadOnlySequenceFactory.cs

@@ -50,9 +50,9 @@ namespace Google.Protobuf
             while (currentIndex < data.Length)
             {
                 var segment = new List<byte>();
-                for (; currentIndex < Math.Min(currentIndex + segmentSize, data.Length); currentIndex++)
+                while (segment.Count < segmentSize && currentIndex < data.Length)
                 {
-                    segment.Add(data[currentIndex]);
+                    segment.Add(data[currentIndex++]);
                 }
                 segments.Add(segment.ToArray());
                 segments.Add(new byte[0]);

csharp/src/Google.Protobuf/Collections/RepeatedField.cs

@@ -35,6 +35,7 @@ using System.Collections;
 using System.Collections.Generic;
 using System.IO;
 using System.Security;
+using System.Threading;
 
 namespace Google.Protobuf.Collections
 {
@@ -126,9 +127,31 @@ namespace Google.Protobuf.Collections
                 if (length > 0)
                 {
                     int oldLimit = SegmentedBufferHelper.PushLimit(ref ctx.state, length);
-                    while (!SegmentedBufferHelper.IsReachedLimit(ref ctx.state))
+
+                    // If the content is fixed size then we can calculate the length
+                    // of the repeated field and pre-initialize the underlying collection.
+                    //
+                    // Check that the supplied length doesn't exceed the underlying buffer.
+                    // That prevents a malicious length from initializing a very large collection.
+                    if (codec.FixedSize > 0 && length % codec.FixedSize == 0 && IsDataAvailable(ref ctx, length))
+                    {
+                        EnsureSize(count + (length / codec.FixedSize));
+
+                        while (!SegmentedBufferHelper.IsReachedLimit(ref ctx.state))
+                        {
+                            // Only FieldCodecs with a fixed size can reach here, and they are all known
+                            // types that don't allow the user to specify a custom reader action.
+                            // reader action will never return null.
+                            array[count++] = reader(ref ctx);
+                        }
+                    }
+                    else
                     {
-                        Add(reader(ref ctx));
+                        // Content is variable size so add until we reach the limit.
+                        while (!SegmentedBufferHelper.IsReachedLimit(ref ctx.state))
+                        {
+                            Add(reader(ref ctx));
+                        }
                     }
                     SegmentedBufferHelper.PopLimit(ref ctx.state, oldLimit);
                 }
@@ -144,6 +167,24 @@ namespace Google.Protobuf.Collections
             }
         }
 
+        private bool IsDataAvailable(ref ParseContext ctx, int size)
+        {
+            // Data fits in remaining buffer
+            if (size <= ctx.state.bufferSize - ctx.state.bufferPos)
+            {
+                return true;
+            }
+
+            // Data fits in remaining source data.
+            // Note that this will never be true when reading from a stream as the total length is unknown.
+            if (size < ctx.state.segmentedBufferHelper.TotalLength - ctx.state.totalBytesRetired - ctx.state.bufferPos)
+            {
+                return true;
+            }
+
+            return false;
+        }
+
         /// <summary>
         /// Calculates the size of this collection based on the given codec.
         /// </summary>

csharp/src/Google.Protobuf/Google.Protobuf.csproj

@@ -5,6 +5,7 @@
     <Copyright>Copyright 2015, Google Inc.</Copyright>
     <AssemblyTitle>Google Protocol Buffers</AssemblyTitle>
     <VersionPrefix>3.12.2</VersionPrefix>
+    <VersionPrefix>3.12.3</VersionPrefix>
     <!-- C# 7.2 is required for Span/BufferWriter/ReadOnlySequence -->
     <LangVersion>7.2</LangVersion>
     <Authors>Google Inc.</Authors>

csharp/src/Google.Protobuf/JsonFormatter.cs

@@ -221,19 +221,12 @@ namespace Google.Protobuf
             foreach (var field in fields.InFieldNumberOrder())
             {
                 var accessor = field.Accessor;
-                if (field.ContainingOneof != null && field.ContainingOneof.Accessor.GetCaseFieldDescriptor(message) != field)
-                {
-                    continue;
-                }
-                // Omit default values unless we're asked to format them, or they're oneofs (where the default
-                // value is still formatted regardless, because that's how we preserve the oneof case).
-                object value = accessor.GetValue(message);
-                if (field.ContainingOneof == null && !settings.FormatDefaultValues && IsDefaultValue(accessor, value))
+                var value = accessor.GetValue(message);
+                if (!ShouldFormatFieldValue(message, field, value))
                 {
                     continue;
                 }
 
-                // Okay, all tests complete: let's write the field value...
                 if (!first)
                 {
                     writer.Write(PropertySeparator);
@@ -248,6 +241,18 @@ namespace Google.Protobuf
             return !first;
         }
 
+        /// <summary>
+        /// Determines whether or not a field value should be serialized according to the field,
+        /// its value in the message, and the settings of this formatter.
+        /// </summary>
+        private bool ShouldFormatFieldValue(IMessage message, FieldDescriptor field, object value) =>
+            field.HasPresence
+            // Fields that support presence *just* use that
+            ? field.Accessor.HasValue(message)
+            // Otherwise, format if either we've been asked to format default values, or if it's
+            // not a default value anyway.
+            : settings.FormatDefaultValues || !IsDefaultValue(field, value);
+
         // Converted from java/core/src/main/java/com/google/protobuf/Descriptors.java
         internal static string ToJsonName(string name)
         {
@@ -295,19 +300,19 @@ namespace Google.Protobuf
             writer.Write("null");
         }
 
-        private static bool IsDefaultValue(IFieldAccessor accessor, object value)
+        private static bool IsDefaultValue(FieldDescriptor descriptor, object value)
         {
-            if (accessor.Descriptor.IsMap)
+            if (descriptor.IsMap)
             {
                 IDictionary dictionary = (IDictionary) value;
                 return dictionary.Count == 0;
             }
-            if (accessor.Descriptor.IsRepeated)
+            if (descriptor.IsRepeated)
             {
                 IList list = (IList) value;
                 return list.Count == 0;
             }
-            switch (accessor.Descriptor.FieldType)
+            switch (descriptor.FieldType)
             {
                 case FieldType.Bool:
                     return (bool) value == false;
@@ -793,8 +798,10 @@ namespace Google.Protobuf
             }
 
             /// <summary>
-            /// Whether fields whose values are the default for the field type (e.g. 0 for integers)
-            /// should be formatted (true) or omitted (false).
+            /// Whether fields which would otherwise not be included in the formatted data
+            /// should be formatted even when the value is not present, or has the default value.
+            /// This option only affects fields which don't support "presence" (e.g.
+            /// singular non-optional proto3 primitive fields).
             /// </summary>
             public bool FormatDefaultValues { get; }
 

docs/third_party.md

@@ -18,6 +18,7 @@ These are projects we know about implementing Protocol Buffers for other program
 * C: https://github.com/squidfunk/protobluff
 * C: https://github.com/eerimoq/pbtools
 * C++: https://github.com/google/protobuf (Google-official implementation)
+* C++: https://EmbeddedProto.com
 * C/C++: http://spbc.sf.net/
 * C#: http://code.google.com/p/protobuf-csharp-port
 * C#: https://silentorbit.com/protobuf/

generate_descriptor_proto.sh

@@ -62,7 +62,7 @@ do
     PROTOC=$BOOTSTRAP_PROTOC
     BOOTSTRAP_PROTOC=""
   else
-    make $@ protoc
+    make -j$(nproc) $@ protoc
     if test $? -ne 0; then
       echo "Failed to build protoc."
       exit 1

java/bom/pom.xml

@@ -4,7 +4,7 @@
 
   <groupId>com.google.protobuf</groupId>
   <artifactId>protobuf-bom</artifactId>
-  <version>3.12.2</version>
+  <version>3.12.3</version>
   <packaging>pom</packaging>
 
   <name>Protocol Buffers [BOM]</name>

java/core/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>com.google.protobuf</groupId>
     <artifactId>protobuf-parent</artifactId>
-    <version>3.12.2</version>
+    <version>3.12.3</version>
   </parent>
 
   <artifactId>protobuf-java</artifactId>

java/lite/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>com.google.protobuf</groupId>
     <artifactId>protobuf-parent</artifactId>
-    <version>3.12.2</version>
+    <version>3.12.3</version>
   </parent>
 
   <artifactId>protobuf-javalite</artifactId>

java/pom.xml

@@ -4,7 +4,7 @@
 
   <groupId>com.google.protobuf</groupId>
   <artifactId>protobuf-parent</artifactId>
-  <version>3.12.2</version>
+  <version>3.12.3</version>
   <packaging>pom</packaging>
 
   <name>Protocol Buffers [Parent]</name>

java/util/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>com.google.protobuf</groupId>
     <artifactId>protobuf-parent</artifactId>
-    <version>3.12.2</version>
+    <version>3.12.3</version>
   </parent>
 
   <artifactId>protobuf-java-util</artifactId>

js/gulpfile.js

@@ -72,7 +72,7 @@ gulp.task('genproto_group1_closure', function (cb) {
 });
 
 gulp.task('genproto_group2_closure', function (cb) {
-  exec(protoc + ' --js_out=library=testproto_libs2,binary:.  -I ../src -I . -I commonjs ' + group2Protos.join(' '),
+  exec(protoc + ' --experimental_allow_proto3_optional --js_out=library=testproto_libs2,binary:.  -I ../src -I . -I commonjs ' + group2Protos.join(' '),
        function (err, stdout, stderr) {
     console.log(stdout);
     console.log(stderr);
@@ -99,7 +99,7 @@ gulp.task('genproto_group1_commonjs', function (cb) {
 });
 
 gulp.task('genproto_group2_commonjs', function (cb) {
-  exec('mkdir -p commonjs_out && ' + protoc + ' --js_out=import_style=commonjs,binary:commonjs_out -I ../src -I commonjs -I . ' + group2Protos.join(' '),
+  exec('mkdir -p commonjs_out && ' + protoc + ' --experimental_allow_proto3_optional --js_out=import_style=commonjs,binary:commonjs_out -I ../src -I commonjs -I . ' + group2Protos.join(' '),
        function (err, stdout, stderr) {
     console.log(stdout);
     console.log(stderr);

js/package.json

@@ -1,6 +1,6 @@
 {
   "name": "google-protobuf",
-  "version": "3.12.2",
+  "version": "3.12.3",
   "description": "Protocol Buffers for JavaScript",
   "main": "google-protobuf.js",
   "files": [

js/proto3_test.js

@@ -77,7 +77,7 @@ describe('proto3Test', function() {
   it('testEqualsProto3', function() {
     var msg1 = new proto.jspb.test.TestProto3();
     var msg2 = new proto.jspb.test.TestProto3();
-    msg2.setOptionalString('');
+    msg2.setSingularString('');
 
     assertTrue(jspb.Message.equals(msg1, msg2));
   });
@@ -90,12 +90,12 @@ describe('proto3Test', function() {
     var msg = new proto.jspb.test.TestProto3();
 
     // Setting should work normally.
-    msg.setOptionalString('optionalString');
-    assertEquals(msg.getOptionalString(), 'optionalString');
+    msg.setSingularString('optionalString');
+    assertEquals(msg.getSingularString(), 'optionalString');
 
     // Clearing should work too ...
-    msg.setOptionalString('');
-    assertEquals(msg.getOptionalString(), '');
+    msg.setSingularString('');
+    assertEquals(msg.getSingularString(), '');
 
     // ... and shouldn't affect the equality with a brand new message.
     assertTrue(jspb.Message.equals(msg, new proto.jspb.test.TestProto3()));
@@ -107,6 +107,58 @@ describe('proto3Test', function() {
   it('testProto3FieldDefaults', function() {
     var msg = new proto.jspb.test.TestProto3();
 
+    assertEquals(msg.getSingularInt32(), 0);
+    assertEquals(msg.getSingularInt64(), 0);
+    assertEquals(msg.getSingularUint32(), 0);
+    assertEquals(msg.getSingularUint64(), 0);
+    assertEquals(msg.getSingularSint32(), 0);
+    assertEquals(msg.getSingularSint64(), 0);
+    assertEquals(msg.getSingularFixed32(), 0);
+    assertEquals(msg.getSingularFixed64(), 0);
+    assertEquals(msg.getSingularSfixed32(), 0);
+    assertEquals(msg.getSingularSfixed64(), 0);
+    assertEquals(msg.getSingularFloat(), 0);
+    assertEquals(msg.getSingularDouble(), 0);
+    assertEquals(msg.getSingularString(), '');
+
+    // TODO(b/26173701): when we change bytes fields default getter to return
+    // Uint8Array, we'll want to switch this assertion to match the u8 case.
+    assertEquals(typeof msg.getSingularBytes(), 'string');
+    assertEquals(msg.getSingularBytes_asU8() instanceof Uint8Array, true);
+    assertEquals(typeof msg.getSingularBytes_asB64(), 'string');
+    assertEquals(msg.getSingularBytes().length, 0);
+    assertEquals(msg.getSingularBytes_asU8().length, 0);
+    assertEquals(msg.getSingularBytes_asB64(), '');
+
+    assertEquals(msg.getSingularForeignEnum(),
+                 proto.jspb.test.Proto3Enum.PROTO3_FOO);
+    assertEquals(msg.getSingularForeignMessage(), undefined);
+    assertEquals(msg.getSingularForeignMessage(), undefined);
+
+    assertEquals(msg.getRepeatedInt32List().length, 0);
+    assertEquals(msg.getRepeatedInt64List().length, 0);
+    assertEquals(msg.getRepeatedUint32List().length, 0);
+    assertEquals(msg.getRepeatedUint64List().length, 0);
+    assertEquals(msg.getRepeatedSint32List().length, 0);
+    assertEquals(msg.getRepeatedSint64List().length, 0);
+    assertEquals(msg.getRepeatedFixed32List().length, 0);
+    assertEquals(msg.getRepeatedFixed64List().length, 0);
+    assertEquals(msg.getRepeatedSfixed32List().length, 0);
+    assertEquals(msg.getRepeatedSfixed64List().length, 0);
+    assertEquals(msg.getRepeatedFloatList().length, 0);
+    assertEquals(msg.getRepeatedDoubleList().length, 0);
+    assertEquals(msg.getRepeatedStringList().length, 0);
+    assertEquals(msg.getRepeatedBytesList().length, 0);
+    assertEquals(msg.getRepeatedForeignEnumList().length, 0);
+    assertEquals(msg.getRepeatedForeignMessageList().length, 0);
+  });
+
+  /**
+   * Test presence for proto3 optional fields.
+   */
+  it('testProto3Optional', function() {
+    var msg = new proto.jspb.test.TestProto3();
+
     assertEquals(msg.getOptionalInt32(), 0);
     assertEquals(msg.getOptionalInt64(), 0);
     assertEquals(msg.getOptionalUint32(), 0);
@@ -135,51 +187,65 @@ describe('proto3Test', function() {
     assertEquals(msg.getOptionalForeignMessage(), undefined);
     assertEquals(msg.getOptionalForeignMessage(), undefined);
 
-    assertEquals(msg.getRepeatedInt32List().length, 0);
-    assertEquals(msg.getRepeatedInt64List().length, 0);
-    assertEquals(msg.getRepeatedUint32List().length, 0);
-    assertEquals(msg.getRepeatedUint64List().length, 0);
-    assertEquals(msg.getRepeatedSint32List().length, 0);
-    assertEquals(msg.getRepeatedSint64List().length, 0);
-    assertEquals(msg.getRepeatedFixed32List().length, 0);
-    assertEquals(msg.getRepeatedFixed64List().length, 0);
-    assertEquals(msg.getRepeatedSfixed32List().length, 0);
-    assertEquals(msg.getRepeatedSfixed64List().length, 0);
-    assertEquals(msg.getRepeatedFloatList().length, 0);
-    assertEquals(msg.getRepeatedDoubleList().length, 0);
-    assertEquals(msg.getRepeatedStringList().length, 0);
-    assertEquals(msg.getRepeatedBytesList().length, 0);
-    assertEquals(msg.getRepeatedForeignEnumList().length, 0);
-    assertEquals(msg.getRepeatedForeignMessageList().length, 0);
+    // Serializing an empty proto yields the empty string.
+    assertEquals(msg.serializeBinary().length, 0);
 
-  });
+    // Values start as unset, but can be explicitly set even to default values
+    // like 0.
+    assertFalse(msg.hasOptionalInt32());
+    msg.setOptionalInt32(0);
+    assertTrue(msg.hasOptionalInt32());
+
+    assertFalse(msg.hasOptionalInt64());
+    msg.setOptionalInt64(0);
+    assertTrue(msg.hasOptionalInt64());
+
+    assertFalse(msg.hasOptionalString());
+    msg.setOptionalString("");
+    assertTrue(msg.hasOptionalString());
+
+    // Now the proto will have a non-zero size, even though its values are 0.
+    var serialized = msg.serializeBinary();
+    assertNotEquals(serialized.length, 0);
 
+    var msg2 = proto.jspb.test.TestProto3.deserializeBinary(serialized);
+    assertTrue(msg2.hasOptionalInt32());
+    assertTrue(msg2.hasOptionalInt64());
+    assertTrue(msg2.hasOptionalString());
+
+    // We can clear fields to go back to empty.
+    msg2.clearOptionalInt32();
+    assertFalse(msg2.hasOptionalInt32());
+
+    msg2.clearOptionalString();
+    assertFalse(msg2.hasOptionalString());
+  });
 
   /**
-   * Test that all fields can be set and read via a serialization roundtrip.
+   * Test that all fields can be set ,and read via a serialization roundtrip.
    */
-  it('testProto3FieldSetGet', function() {
+  it('testProto3FieldSetGet', function () {
     var msg = new proto.jspb.test.TestProto3();
 
-    msg.setOptionalInt32(-42);
-    msg.setOptionalInt64(-0x7fffffff00000000);
-    msg.setOptionalUint32(0x80000000);
-    msg.setOptionalUint64(0xf000000000000000);
-    msg.setOptionalSint32(-100);
-    msg.setOptionalSint64(-0x8000000000000000);
-    msg.setOptionalFixed32(1234);
-    msg.setOptionalFixed64(0x1234567800000000);
-    msg.setOptionalSfixed32(-1234);
-    msg.setOptionalSfixed64(-0x1234567800000000);
-    msg.setOptionalFloat(1.5);
-    msg.setOptionalDouble(-1.5);
-    msg.setOptionalBool(true);
-    msg.setOptionalString('hello world');
-    msg.setOptionalBytes(BYTES);
+    msg.setSingularInt32(-42);
+    msg.setSingularInt64(-0x7fffffff00000000);
+    msg.setSingularUint32(0x80000000);
+    msg.setSingularUint64(0xf000000000000000);
+    msg.setSingularSint32(-100);
+    msg.setSingularSint64(-0x8000000000000000);
+    msg.setSingularFixed32(1234);
+    msg.setSingularFixed64(0x1234567800000000);
+    msg.setSingularSfixed32(-1234);
+    msg.setSingularSfixed64(-0x1234567800000000);
+    msg.setSingularFloat(1.5);
+    msg.setSingularDouble(-1.5);
+    msg.setSingularBool(true);
+    msg.setSingularString('hello world');
+    msg.setSingularBytes(BYTES);
     var submsg = new proto.jspb.test.ForeignMessage();
     submsg.setC(16);
-    msg.setOptionalForeignMessage(submsg);
-    msg.setOptionalForeignEnum(proto.jspb.test.Proto3Enum.PROTO3_BAR);
+    msg.setSingularForeignMessage(submsg);
+    msg.setSingularForeignEnum(proto.jspb.test.Proto3Enum.PROTO3_BAR);
 
     msg.setRepeatedInt32List([-42]);
     msg.setRepeatedInt64List([-0x7fffffff00000000]);
@@ -206,23 +272,23 @@ describe('proto3Test', function() {
     var serialized = msg.serializeBinary();
     msg = proto.jspb.test.TestProto3.deserializeBinary(serialized);
 
-    assertEquals(msg.getOptionalInt32(), -42);
-    assertEquals(msg.getOptionalInt64(), -0x7fffffff00000000);
-    assertEquals(msg.getOptionalUint32(), 0x80000000);
-    assertEquals(msg.getOptionalUint64(), 0xf000000000000000);
-    assertEquals(msg.getOptionalSint32(), -100);
-    assertEquals(msg.getOptionalSint64(), -0x8000000000000000);
-    assertEquals(msg.getOptionalFixed32(), 1234);
-    assertEquals(msg.getOptionalFixed64(), 0x1234567800000000);
-    assertEquals(msg.getOptionalSfixed32(), -1234);
-    assertEquals(msg.getOptionalSfixed64(), -0x1234567800000000);
-    assertEquals(msg.getOptionalFloat(), 1.5);
-    assertEquals(msg.getOptionalDouble(), -1.5);
-    assertEquals(msg.getOptionalBool(), true);
-    assertEquals(msg.getOptionalString(), 'hello world');
-    assertEquals(true, bytesCompare(msg.getOptionalBytes(), BYTES));
-    assertEquals(msg.getOptionalForeignMessage().getC(), 16);
-    assertEquals(msg.getOptionalForeignEnum(),
+    assertEquals(msg.getSingularInt32(), -42);
+    assertEquals(msg.getSingularInt64(), -0x7fffffff00000000);
+    assertEquals(msg.getSingularUint32(), 0x80000000);
+    assertEquals(msg.getSingularUint64(), 0xf000000000000000);
+    assertEquals(msg.getSingularSint32(), -100);
+    assertEquals(msg.getSingularSint64(), -0x8000000000000000);
+    assertEquals(msg.getSingularFixed32(), 1234);
+    assertEquals(msg.getSingularFixed64(), 0x1234567800000000);
+    assertEquals(msg.getSingularSfixed32(), -1234);
+    assertEquals(msg.getSingularSfixed64(), -0x1234567800000000);
+    assertEquals(msg.getSingularFloat(), 1.5);
+    assertEquals(msg.getSingularDouble(), -1.5);
+    assertEquals(msg.getSingularBool(), true);
+    assertEquals(msg.getSingularString(), 'hello world');
+    assertEquals(true, bytesCompare(msg.getSingularBytes(), BYTES));
+    assertEquals(msg.getSingularForeignMessage().getC(), 16);
+    assertEquals(msg.getSingularForeignEnum(),
         proto.jspb.test.Proto3Enum.PROTO3_BAR);
 
     assertElementsEquals(msg.getRepeatedInt32List(), [-42]);
@@ -327,20 +393,20 @@ describe('proto3Test', function() {
     // Set each primitive to a non-default value, then back to its default, to
     // ensure that the serialization is actually checking the value and not just
     // whether it has ever been set.
-    msg.setOptionalInt32(42);
-    msg.setOptionalInt32(0);
-    msg.setOptionalDouble(3.14);
-    msg.setOptionalDouble(0.0);
-    msg.setOptionalBool(true);
-    msg.setOptionalBool(false);
-    msg.setOptionalString('hello world');
-    msg.setOptionalString('');
-    msg.setOptionalBytes(goog.crypt.base64.encodeString('\u00FF\u00FF'));
-    msg.setOptionalBytes('');
-    msg.setOptionalForeignMessage(new proto.jspb.test.ForeignMessage());
-    msg.setOptionalForeignMessage(null);
-    msg.setOptionalForeignEnum(proto.jspb.test.Proto3Enum.PROTO3_BAR);
-    msg.setOptionalForeignEnum(proto.jspb.test.Proto3Enum.PROTO3_FOO);
+    msg.setSingularInt32(42);
+    msg.setSingularInt32(0);
+    msg.setSingularDouble(3.14);
+    msg.setSingularDouble(0.0);
+    msg.setSingularBool(true);
+    msg.setSingularBool(false);
+    msg.setSingularString('hello world');
+    msg.setSingularString('');
+    msg.setSingularBytes(goog.crypt.base64.encodeString('\u00FF\u00FF'));
+    msg.setSingularBytes('');
+    msg.setSingularForeignMessage(new proto.jspb.test.ForeignMessage());
+    msg.setSingularForeignMessage(null);
+    msg.setSingularForeignEnum(proto.jspb.test.Proto3Enum.PROTO3_BAR);
+    msg.setSingularForeignEnum(proto.jspb.test.Proto3Enum.PROTO3_FOO);
     msg.setOneofUint32(32);
     msg.clearOneofUint32();
 
@@ -355,27 +421,25 @@ describe('proto3Test', function() {
   it('testBytesFieldsInterop', function() {
     var msg = new proto.jspb.test.TestProto3();
     // Set as a base64 string and check all the getters work.
-    msg.setOptionalBytes(BYTES_B64);
-    assertTrue(bytesCompare(msg.getOptionalBytes_asU8(), BYTES));
-    assertTrue(bytesCompare(msg.getOptionalBytes_asB64(), BYTES));
-    assertTrue(bytesCompare(msg.getOptionalBytes(), BYTES));
+    msg.setSingularBytes(BYTES_B64);
+    assertTrue(bytesCompare(msg.getSingularBytes_asU8(), BYTES));
+    assertTrue(bytesCompare(msg.getSingularBytes_asB64(), BYTES));
+    assertTrue(bytesCompare(msg.getSingularBytes(), BYTES));
 
     // Test binary serialize round trip doesn't break it.
     msg = proto.jspb.test.TestProto3.deserializeBinary(msg.serializeBinary());
-    assertTrue(bytesCompare(msg.getOptionalBytes_asU8(), BYTES));
-    assertTrue(bytesCompare(msg.getOptionalBytes_asB64(), BYTES));
-    assertTrue(bytesCompare(msg.getOptionalBytes(), BYTES));
+    assertTrue(bytesCompare(msg.getSingularBytes_asU8(), BYTES));
+    assertTrue(bytesCompare(msg.getSingularBytes_asB64(), BYTES));
+    assertTrue(bytesCompare(msg.getSingularBytes(), BYTES));
 
     msg = new proto.jspb.test.TestProto3();
     // Set as a Uint8Array and check all the getters work.
-    msg.setOptionalBytes(BYTES);
-    assertTrue(bytesCompare(msg.getOptionalBytes_asU8(), BYTES));
-    assertTrue(bytesCompare(msg.getOptionalBytes_asB64(), BYTES));
-    assertTrue(bytesCompare(msg.getOptionalBytes(), BYTES));
-
+    msg.setSingularBytes(BYTES);
+    assertTrue(bytesCompare(msg.getSingularBytes_asU8(), BYTES));
+    assertTrue(bytesCompare(msg.getSingularBytes_asB64(), BYTES));
+    assertTrue(bytesCompare(msg.getSingularBytes(), BYTES));
   });
 
-
   it('testTimestampWellKnownType', function() {
     var msg = new proto.google.protobuf.Timestamp();
     msg.fromDate(new Date(123456789));

js/proto3_test.proto

@@ -35,24 +35,43 @@ import "testbinary.proto";
 package jspb.test;
 
 message TestProto3 {
-  int32 optional_int32 = 1;
-  int64 optional_int64 = 2;
-  uint32 optional_uint32 = 3;
-  uint64 optional_uint64 = 4;
-  sint32 optional_sint32 = 5;
-  sint64 optional_sint64 = 6;
-  fixed32 optional_fixed32 = 7;
-  fixed64 optional_fixed64 = 8;
-  sfixed32 optional_sfixed32 = 9;
-  sfixed64 optional_sfixed64 = 10;
-  float optional_float = 11;
-  double optional_double = 12;
-  bool optional_bool = 13;
-  string optional_string = 14;
-  bytes optional_bytes = 15;
+  int32 singular_int32 = 1;
+  int64 singular_int64 = 2;
+  uint32 singular_uint32 = 3;
+  uint64 singular_uint64 = 4;
+  sint32 singular_sint32 = 5;
+  sint64 singular_sint64 = 6;
+  fixed32 singular_fixed32 = 7;
+  fixed64 singular_fixed64 = 8;
+  sfixed32 singular_sfixed32 = 9;
+  sfixed64 singular_sfixed64 = 10;
+  float singular_float = 11;
+  double singular_double = 12;
+  bool singular_bool = 13;
+  string singular_string = 14;
+  bytes singular_bytes = 15;
 
-  ForeignMessage optional_foreign_message = 19;
-  Proto3Enum optional_foreign_enum = 22;
+  ForeignMessage singular_foreign_message = 19;
+  Proto3Enum singular_foreign_enum = 22;
+
+  optional int32 optional_int32 = 121;
+  optional int64 optional_int64 = 122;
+  optional uint32 optional_uint32 = 123;
+  optional uint64 optional_uint64 = 124;
+  optional sint32 optional_sint32 = 125;
+  optional sint64 optional_sint64 = 126;
+  optional fixed32 optional_fixed32 = 127;
+  optional fixed64 optional_fixed64 = 128;
+  optional sfixed32 optional_sfixed32 = 129;
+  optional sfixed64 optional_sfixed64 = 130;
+  optional float optional_float = 131;
+  optional double optional_double = 132;
+  optional bool optional_bool = 133;
+  optional string optional_string = 134;
+  optional bytes optional_bytes = 135;
+
+  optional ForeignMessage optional_foreign_message = 136;
+  optional Proto3Enum optional_foreign_enum =137;
 
   repeated int32 repeated_int32 = 31;
   repeated int64 repeated_int64 = 32;

kokoro/macos/objectivec_cocoapods_integration/build.sh

@@ -6,6 +6,7 @@
 cd $(dirname $0)/../../..
 
 # Prepare worker environment to run tests
+KOKORO_INSTALL_COCOAPODS=yes
 source kokoro/macos/prepare_build_macos_rc
 
 ./tests.sh objectivec_cocoapods_integration

kokoro/macos/prepare_build_macos_rc

@@ -2,6 +2,8 @@
 #
 # This script sets up a Kokoro MacOS worker for running Protobuf tests
 
+set -eux
+
 ##
 # Select Xcode version
 
@@ -18,24 +20,61 @@ export CC=gcc
 export CXX=g++
 
 ##
-# Install Brew and core softwares
+# Brew: update, then upgrade the installed tools to current version and install
+# some needed ones not in the Kokoro base image. This ensure current versions
+# of CMake, autotools, etc.
+
+# But first...
+#
+# The transitive deps of the installed tools need protobuf, but Kokoro manually
+# installed it outside of brew so it needs to be removed so brew can install the
+# tools (and a newer version of protobuf). g/kokoro-users/7FRvQMUdN40 about why
+# it is a manual install vs. a brew install in the first place.
+sudo rm -rf \
+    /usr/local/include/google/protobuf \
+    /usr/local/bin/protoc
+# Likewise, updating python can have issues because of some existing binaries.
+sudo rm -rf \
+    /usr/local/bin/2to3* \
+    /usr/local/bin/idle3* \
+    /usr/local/bin/pydoc3* \
+    /usr/local/bin/python3* \
+    /usr/local/bin/pyvenv*
+
+brew update
+brew upgrade
+
+##
+# Install Ruby
 
-ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
-source $HOME/.rvm/scripts/rvm
-brew uninstall node icu4c cmake wget
-brew prune
-brew install gflags gpg gpg2 node openssl pcre ruby cmake wget
-sudo chown -R $(whoami) /usr/local
-brew postinstall node
+if [[ "${KOKORO_INSTALL_RUBY:-}" == "yes" ]] ; then
+  brew install ruby
+fi
+
+##
+# Install Cocoapods
+
+if [[ "${KOKORO_INSTALL_COCOAPODS:-}" == "yes" ]] ; then
+  # The existing cocoapods was installed via gem, but that doesn't work well
+  # with the overlap in deps with things managed by brew (errors around ruby
+  # versions, etc.); so remove it and install in via brew instead.
+  gem uninstall -a "$(gem list | grep cocoapods | cut -d ' ' -f 1)"
+  brew install cocoapods
+fi
 
 ##
 # Install Tox
 
-sudo pip install tox==2.4.1
+if [[ "${KOKORO_INSTALL_TOX:-}" == "yes" ]] ; then
+  sudo pip install tox==2.4.1
+fi
 
 ##
 # Install RVM
 
-gpg2 --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3
-command curl -sSL https://rvm.io/mpapis.asc | gpg2 --import -
-curl -sSL https://get.rvm.io | bash -s stable --ruby
+if [[ "${KOKORO_INSTALL_RVM:-}" == "yes" ]] ; then
+  curl -sSL https://rvm.io/mpapis.asc | gpg --import -
+  curl -sSL https://rvm.io/pkuczynski.asc | gpg --import -
+
+  curl -sSL https://get.rvm.io | bash -s stable --ruby
+fi

kokoro/macos/python/build.sh

@@ -6,6 +6,7 @@
 cd $(dirname $0)/../../..
 
 # Prepare worker environment to run tests
+KOKORO_INSTALL_TOX=yes
 source kokoro/macos/prepare_build_macos_rc
 
 ./tests.sh python

kokoro/macos/python_cpp/build.sh

@@ -6,6 +6,7 @@
 cd $(dirname $0)/../../..
 
 # Prepare worker environment to run tests
+KOKORO_INSTALL_TOX=yes
 source kokoro/macos/prepare_build_macos_rc
 g++ --version
 

kokoro/macos/ruby23/build.sh

@@ -6,6 +6,8 @@
 cd $(dirname $0)/../../..
 
 # Prepare worker environment to run tests
+KOKORO_INSTALL_RUBY=yes
+KOKORO_INSTALL_RVM=yes
 source kokoro/macos/prepare_build_macos_rc
 
 ./tests.sh ruby23

kokoro/macos/ruby24/build.sh

@@ -6,6 +6,8 @@
 cd $(dirname $0)/../../..
 
 # Prepare worker environment to run tests
+KOKORO_INSTALL_RUBY=yes
+KOKORO_INSTALL_RVM=yes
 source kokoro/macos/prepare_build_macos_rc
 
 ./tests.sh ruby24

kokoro/macos/ruby25/build.sh

@@ -6,6 +6,8 @@
 cd $(dirname $0)/../../..
 
 # Prepare worker environment to run tests
+KOKORO_INSTALL_RUBY=yes
+KOKORO_INSTALL_RVM=yes
 source kokoro/macos/prepare_build_macos_rc
 
 ./tests.sh ruby25

kokoro/macos/ruby26/build.sh

@@ -6,6 +6,8 @@
 cd $(dirname $0)/../../..
 
 # Prepare worker environment to run tests
+KOKORO_INSTALL_RUBY=yes
+KOKORO_INSTALL_RVM=yes
 source kokoro/macos/prepare_build_macos_rc
 
 ./tests.sh ruby26

kokoro/macos/ruby27/build.sh

@@ -6,6 +6,8 @@
 cd $(dirname $0)/../../..
 
 # Prepare worker environment to run tests
+KOKORO_INSTALL_RUBY=yes
+KOKORO_INSTALL_RVM=yes
 source kokoro/macos/prepare_build_macos_rc
 
 ./tests.sh ruby27

objectivec/GPBDescriptor_PackagePrivate.h

@@ -131,14 +131,29 @@ typedef NS_OPTIONS(uint8_t, GPBExtensionOptions) {
 typedef struct GPBExtensionDescription {
   GPBGenericValue defaultValue;
   const char *singletonName;
+  // Before 3.12, `extendedClass` was just a `const char *`. Thanks to nested
+  // initialization (https://en.cppreference.com/w/c/language/struct_initialization#Nested_initialization)
+  // old generated code with `.extendedClass = GPBStringifySymbol(Something)`
+  // still works; and the current generator can use `extendedClass.clazz`, to
+  // pass a Class reference.
   union {
     const char *name;
     Class clazz;
   } extendedClass;
+  // Before 3.12, this was `const char *messageOrGroupClassName`. In the
+  // initial 3.12 release, we moved the `union messageOrGroupClass`, and failed
+  // to realize that would break existing source code for extensions. So to
+  // keep existing source code working, we added an unnamed union (C11) to
+  // provide both the old field name and the new union. This keeps both older
+  // and newer code working.
+  // Background: https://github.com/protocolbuffers/protobuf/issues/7555
   union {
-    const char *name;
-    Class clazz;
-  } messageOrGroupClass;
+    const char *messageOrGroupClassName;
+    union {
+     const char *name;
+     Class clazz;
+   } messageOrGroupClass;
+  };
   GPBEnumDescriptorFunc enumDescriptorFunc;
   int32_t fieldNumber;
   GPBDataType dataType;

objectivec/README.md

@@ -172,6 +172,12 @@ supported keys are:
     lot of proto files included in it; and having multiple lines makes things
     easier to read.
 
+  * `runtime_import_prefix`: The `value` used for this key to be used as a
+    prefix on `#import`s of runtime provided headers in the generated files.
+    When integrating ObjC protos into a build system, this can be used to avoid
+    having to add the runtime directory to the header search path since the
+    generate `#import` will be more complete.
+
 Contributing
 ------------
 

php/composer.json

@@ -23,7 +23,7 @@
     }
   },
   "scripts": {
-    "test": "(cd tests && rm -rf generated && mkdir -p generated && ../../src/protoc --php_out=generated -I../../src -I. proto/empty/echo.proto proto/test.proto proto/test_include.proto proto/test_no_namespace.proto proto/test_prefix.proto proto/test_php_namespace.proto proto/test_empty_php_namespace.proto proto/test_reserved_enum_lower.proto proto/test_reserved_enum_upper.proto proto/test_reserved_enum_value_lower.proto proto/test_reserved_enum_value_upper.proto proto/test_reserved_message_lower.proto proto/test_reserved_message_upper.proto proto/test_service.proto proto/test_service_namespace.proto proto/test_wrapper_type_setters.proto proto/test_descriptors.proto) && (cd ../src && ./protoc --php_out=../php/tests/generated -I../php/tests -I. ../php/tests/proto/test_import_descriptor_proto.proto) && vendor/bin/phpunit",
-    "aggregate_metadata_test": "(cd tests && rm -rf generated && mkdir -p generated && ../../src/protoc --php_out=aggregate_metadata=foo#bar:generated -I../../src -I. proto/test.proto proto/test_include.proto && ../../src/protoc --php_out=generated -I../../src -I. proto/empty/echo.proto proto/test_no_namespace.proto proto/test_empty_php_namespace.proto proto/test_prefix.proto proto/test_php_namespace.proto proto/test_reserved_enum_lower.proto proto/test_reserved_enum_upper.proto proto/test_reserved_enum_value_lower.proto proto/test_reserved_enum_value_upper.proto proto/test_reserved_message_lower.proto proto/test_reserved_message_upper.proto proto/test_service.proto proto/test_service_namespace.proto proto/test_wrapper_type_setters.proto proto/test_descriptors.proto) && (cd ../src && ./protoc --php_out=aggregate_metadata=foo:../php/tests/generated -I../php/tests -I. ../php/tests/proto/test_import_descriptor_proto.proto) && vendor/bin/phpunit"
+    "test": "tests/generate_protos.sh && vendor/bin/phpunit",
+    "aggregate_metadata_test": "tests/generate_protos.sh --aggregate_metadata && vendor/bin/phpunit"
   }
 }

php/ext/google/protobuf/message.c

@@ -1726,7 +1726,7 @@ PHP_PROTO_INIT_SUBMSGCLASS_START("Google\\Protobuf\\Timestamp",
                              0 ,ZEND_ACC_PRIVATE TSRMLS_CC);
 PHP_PROTO_INIT_SUBMSGCLASS_END
 
-PHP_METHOD(Timestamp, __construct) {
+static PHP_METHOD(Timestamp, __construct) {
   init_file_timestamp(TSRMLS_C);
   INIT_MESSAGE_WITH_ARRAY;
 }

php/ext/google/protobuf/package.xml

@@ -10,11 +10,11 @@
   <email>protobuf-opensource@google.com</email>
   <active>yes</active>
  </lead>
- <date>2020-05-26</date>
- <time>13:57:10</time>
+ <date>2020-06-01</date>
+ <time>01:14:10</time>
  <version>
-  <release>3.12.2</release>
-  <api>3.12.2</api>
+  <release>3.12.3</release>
+  <api>3.12.3</api>
  </version>
  <stability>
   <release>stable</release>
@@ -599,5 +599,19 @@ G  A release.
    <license uri="https://opensource.org/licenses/BSD-3-Clause">3-Clause BSD License</license>
    <notes>GA release.</notes>
   </release>
+  <release>
+   <version>
+    <release>3.12.3</release>
+    <api>3.12.3</api>
+   </version>
+   <stability>
+    <release>stable</release>
+    <api>stable</api>
+   </stability>
+   <date>2020-06-01</date>
+   <time>01:09:39</time>
+   <license uri="https://opensource.org/licenses/BSD-3-Clause">3-Clause BSD License</license>
+   <notes>GA release.</notes>
+  </release>
  </changelog>
 </package>

php/ext/google/protobuf/protobuf.h

@@ -37,7 +37,7 @@
 #include "upb.h"
 
 #define PHP_PROTOBUF_EXTNAME "protobuf"
-#define PHP_PROTOBUF_VERSION "3.12.2"
+#define PHP_PROTOBUF_VERSION "3.12.3"
 
 #define MAX_LENGTH_OF_INT64 20
 #define SIZEOF_INT64 8
@@ -1292,7 +1292,7 @@ PHP_METHOD(Duration, setSeconds);
 PHP_METHOD(Duration, getNanos);
 PHP_METHOD(Duration, setNanos);
 
-PHP_METHOD(Timestamp, __construct);
+static PHP_METHOD(Timestamp, __construct);
 PHP_METHOD(Timestamp, fromDateTime);
 PHP_METHOD(Timestamp, toDateTime);
 PHP_METHOD(Timestamp, getSeconds);

php/tests/compile_extension.sh

@@ -1,8 +1,10 @@
 #!/bin/bash
 
+set -ex
+
 cd $(dirname $0)
 
-if [ "$1" = "--release"]; then
+if [ "$1" = "--release" ]; then
   CFLAGS="-Wall"
 else
   # To get debugging symbols in PHP itself, build PHP with:
@@ -12,6 +14,5 @@ fi
 
 pushd  ../ext/google/protobuf
 make clean || true
-set -e
 phpize && ./configure --with-php-config=$(which php-config) CFLAGS="$CFLAGS" && make
 popd

php/tests/generate_protos.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -ex
+
+cd `dirname $0`
+
+rm -rf generated
+mkdir -p generated
+
+find proto -type f -name "*.proto"| xargs ../../src/protoc --php_out=generated -I../../src -I.
+
+if [ "$1" = "--aggregate_metadata" ]; then
+  # Overwrite some of the files to use aggregation.
+  AGGREGATED_FILES="proto/test.proto proto/test_include.proto proto/test_import_descriptor_proto.proto"
+  ../../src/protoc --php_out=aggregate_metadata=foo#bar:generated -I../../src -I. $AGGREGATED_FILES
+fi

php/tests/generated_class_test.php

@@ -1527,6 +1527,21 @@ class GeneratedClassTest extends TestBase
     public function testNoExceptionWithVarDump()
     {
         $m = new Sub(['a' => 1]);
-        var_dump($m);
+        /*
+         * This line currently segfaults on macOS with:
+         *
+         *    frame #0: 0x00000001029936cc xdebug.so`xdebug_zend_hash_is_recursive + 4
+         *    frame #1: 0x00000001029a6736 xdebug.so`xdebug_var_export_text_ansi + 1006
+         *    frame #2: 0x00000001029a715d xdebug.so`xdebug_get_zval_value_text_ansi + 273
+         *    frame #3: 0x000000010298a441 xdebug.so`zif_xdebug_var_dump + 297
+         *    frame #4: 0x000000010298d558 xdebug.so`xdebug_execute_internal + 640
+         *    frame #5: 0x000000010046d47f php`ZEND_DO_FCALL_SPEC_RETVAL_UNUSED_HANDLER + 364
+         *    frame #6: 0x000000010043cabc php`execute_ex + 44
+         *    frame #7: 0x000000010298d151 xdebug.so`xdebug_execute_ex + 1662
+         *    frame #8: 0x000000010046d865 php`ZEND_DO_FCALL_SPEC_RETVAL_USED_HANDLER + 426
+         *
+         * The value we are passing to var_dump() appears to be corrupt somehow.
+         */
+        /* var_dump($m); */
     }
 }

php/tests/test.sh

@@ -1,9 +1,23 @@
 #!/bin/bash
 
+set -ex
+
 cd $(dirname $0)
 
+./generate_protos.sh
 ./compile_extension.sh
 
+PHP_VERSION=$(php -r "echo PHP_VERSION;")
+
+# Each version of PHPUnit supports a fairly narrow range of PHP versions.
+case "$PHP_VERSION" in
+  5.6.*) PHPUNIT=phpunit-5.6.8.phar;;
+  7.0.*) PHPUNIT=phpunit-5.6.0.phar;;  # Oddly older than for 5.6. Not sure the reason.
+  7.3.*) PHPUNIT=phpunit-8.phar;;
+esac
+
+[ -f $PHPUNIT ] || wget https://phar.phpunit.de/$PHPUNIT
+
 tests=( array_test.php encode_decode_test.php generated_class_test.php map_field_test.php well_known_test.php descriptors_test.php wrapper_type_setters_test.php)
 
 for t in "${tests[@]}"
@@ -11,7 +25,7 @@ do
   echo "****************************"
   echo "* $t"
   echo "****************************"
-  php -dextension=../ext/google/protobuf/modules/protobuf.so `which phpunit` --bootstrap autoload.php $t
+  php -dextension=../ext/google/protobuf/modules/protobuf.so $PHPUNIT --bootstrap autoload.php $t
   echo ""
 done
 
@@ -20,7 +34,7 @@ do
   echo "****************************"
   echo "* $t persistent"
   echo "****************************"
-  php -d protobuf.keep_descriptor_pool_after_request=1 -dextension=../ext/google/protobuf/modules/protobuf.so `which phpunit` --bootstrap autoload.php $t
+  php -d protobuf.keep_descriptor_pool_after_request=1 -dextension=../ext/google/protobuf/modules/protobuf.so $PHPUNIT --bootstrap autoload.php $t
   echo ""
 done
 
@@ -40,6 +54,6 @@ valgrind --leak-check=yes php -d protobuf.keep_descriptor_pool_after_request=1 -
 #   echo "****************************"
 #   echo "* $t (memory leak)"
 #   echo "****************************"
-#   valgrind --leak-check=yes php -dextension=../ext/google/protobuf/modules/protobuf.so `which phpunit` --bootstrap autoload.php $t
+#   valgrind --leak-check=yes php -dextension=../ext/google/protobuf/modules/protobuf.so $PHPUNIT --bootstrap autoload.php $t
 #   echo ""
 # done

protoc-artifacts/pom.xml

@@ -8,7 +8,7 @@
   </parent>
   <groupId>com.google.protobuf</groupId>
   <artifactId>protoc</artifactId>
-  <version>3.12.2</version>
+  <version>3.12.3</version>
   <packaging>pom</packaging>
   <name>Protobuf Compiler</name>
   <description>

python/google/protobuf/__init__.py

@@ -30,7 +30,7 @@
 
 # Copyright 2007 Google Inc. All Rights Reserved.
 
-__version__ = '3.12.2'
+__version__ = '3.12.3'
 
 if __name__ != '__main__':
   try:

python/release.sh

@@ -80,14 +80,16 @@ python setup.py build
 python setup.py test
 
 # Deploy source package to testing PyPI
-python setup.py sdist upload -r https://test.pypi.org/legacy/
+python setup.py sdist
+twine upload --skip-existing -r testpypi -u protobuf-wheel-test dist/*
 
 # Test locally with different python versions.
 run_install_test ${TESTING_VERSION} python2.7 https://test.pypi.org/simple
 run_install_test ${TESTING_VERSION} python3 https://test.pypi.org/simple
 
 # Deploy egg/wheel packages to testing PyPI and test again.
-python setup.py bdist_egg bdist_wheel upload -r https://test.pypi.org/legacy/
+python setup.py clean build bdist_wheel
+twine upload --skip-existing -r testpypi -u protobuf-wheel-test dist/*
 
 run_install_test ${TESTING_VERSION} python2.7 https://test.pypi.org/simple
 run_install_test ${TESTING_VERSION} python3 https://test.pypi.org/simple
@@ -103,13 +105,15 @@ if [ $TESTING_ONLY -eq 0 ]; then
   echo "Publishing to PyPI..."
   # Be sure to run build before sdist, because otherwise sdist will not include
   # well-known types.
-  python setup.py clean build sdist upload
+  python setup.py clean build sdist
+  twine upload --skip-existing -u protobuf-packages dist/*
   # Be sure to run clean before bdist_xxx, because otherwise bdist_xxx will
   # include files you may not want in the package. E.g., if you have built
   # and tested with --cpp_implemenation, bdist_xxx will include the _message.so
   # file even when you no longer pass the --cpp_implemenation flag. See:
   #   https://github.com/protocolbuffers/protobuf/issues/3042
-  python setup.py clean build bdist_egg bdist_wheel upload
+  python setup.py clean build bdist_wheel
+  twine upload --skip-existing -u protobuf-packages dist/*
 else
   # Set the version number back (i.e., remove dev suffix).
   sed -i -r "s/__version__ = '.*'/__version__ = '${VERSION}'/" google/protobuf/__init__.py

ruby/google-protobuf.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name        = "google-protobuf"
-  s.version     = "3.12.2"
+  s.version     = "3.12.3"
   git_tag       = "v#{s.version.to_s.sub('.rc.', '-rc')}" # Converts X.Y.Z.rc.N to vX.Y.Z-rcN, used for the git tag
   s.licenses    = ["BSD-3-Clause"]
   s.summary     = "Protocol Buffers"

src/Makefile.am

@@ -18,7 +18,7 @@ else
 PTHREAD_DEF =
 endif
 
-PROTOBUF_VERSION = 23:2:0
+PROTOBUF_VERSION = 23:3:0
 
 if GCC
 # Turn on all warnings except for sign comparison (we ignore sign comparison

src/google/protobuf/any.pb.h

@@ -13,7 +13,7 @@
 #error incompatible with your Protocol Buffer headers. Please update
 #error your headers.
 #endif
-#if 3012002 < PROTOBUF_MIN_PROTOC_VERSION
+#if 3012003 < PROTOBUF_MIN_PROTOC_VERSION
 #error This file was generated by an older version of protoc which is
 #error incompatible with your Protocol Buffer headers. Please
 #error regenerate this file with a newer version of protoc.

src/google/protobuf/api.pb.h

@@ -13,7 +13,7 @@
 #error incompatible with your Protocol Buffer headers. Please update
 #error your headers.
 #endif
-#if 3012002 < PROTOBUF_MIN_PROTOC_VERSION
+#if 3012003 < PROTOBUF_MIN_PROTOC_VERSION
 #error This file was generated by an older version of protoc which is
 #error incompatible with your Protocol Buffer headers. Please
 #error regenerate this file with a newer version of protoc.

src/google/protobuf/compiler/csharp/csharp_generator.cc

@@ -51,7 +51,7 @@ namespace csharp {
 Generator::Generator() {}
 Generator::~Generator() {}
 
-uint64 Generator::GetSupportedFeatures() const {
+uint64_t Generator::GetSupportedFeatures() const {
   return CodeGenerator::Feature::FEATURE_PROTO3_OPTIONAL;
 }
 

src/google/protobuf/compiler/csharp/csharp_generator.h

@@ -57,7 +57,7 @@ class PROTOC_EXPORT Generator : public CodeGenerator {
     const string& parameter,
     GeneratorContext* generator_context,
     string* error) const override;
-  uint64 GetSupportedFeatures() const override;
+  uint64_t GetSupportedFeatures() const override;
 };
 
 }  // namespace csharp

src/google/protobuf/compiler/js/js_generator.cc

@@ -467,6 +467,7 @@ bool IgnoreMessage(const Descriptor* d) { return d->options().map_entry(); }
 
 // Does JSPB ignore this entire oneof? True only if all fields are ignored.
 bool IgnoreOneof(const OneofDescriptor* oneof) {
+  if (oneof->is_synthetic()) return true;
   for (int i = 0; i < oneof->field_count(); i++) {
     if (!IgnoreField(oneof->field(i))) {
       return false;
@@ -576,6 +577,7 @@ std::string JSOneofIndex(const OneofDescriptor* oneof) {
   int index = -1;
   for (int i = 0; i < oneof->containing_type()->oneof_decl_count(); i++) {
     const OneofDescriptor* o = oneof->containing_type()->oneof_decl(i);
+    if (o->is_synthetic()) continue;
     // If at least one field in this oneof is not JSPB-ignored, count the oneof.
     for (int j = 0; j < o->field_count(); j++) {
       const FieldDescriptor* f = o->field(j);
@@ -794,6 +796,11 @@ std::string DoubleToString(double value) {
   return PostProcessFloat(result);
 }
 
+bool InRealOneof(const FieldDescriptor* field) {
+  return field->containing_oneof() &&
+         !field->containing_oneof()->is_synthetic();
+}
+
 // Return true if this is an integral field that should be represented as string
 // in JS.
 bool IsIntegralFieldWithStringJSType(const FieldDescriptor* field) {
@@ -1173,7 +1180,7 @@ std::string RepeatedFieldsArrayName(const GeneratorOptions& options,
 
 bool HasOneofFields(const Descriptor* desc) {
   for (int i = 0; i < desc->field_count(); i++) {
-    if (desc->field(i)->containing_oneof()) {
+    if (InRealOneof(desc->field(i))) {
       return true;
     }
   }
@@ -1411,15 +1418,9 @@ std::string GetPivot(const Descriptor* desc) {
 // generate extra methods (clearFoo() and hasFoo()) for this field.
 bool HasFieldPresence(const GeneratorOptions& options,
                       const FieldDescriptor* field) {
-  if (field->is_repeated() || field->is_map()) {
-    // We say repeated fields and maps don't have presence, but we still do
-    // generate clearFoo() methods for them through a special case elsewhere.
-    return false;
-  }
-
-  return field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE ||
-         field->containing_oneof() != NULL ||
-         field->file()->syntax() == FileDescriptor::SYNTAX_PROTO2;
+  // This returns false for repeated fields and maps, but we still do
+  // generate clearFoo() methods for these through a special case elsewhere.
+  return field->has_presence();
 }
 
 // We use this to implement the semantics that same file can be generated
@@ -2676,7 +2677,7 @@ void Generator::GenerateClassField(const GeneratorOptions& options,
                               /* singular_if_not_packed = */ false),
         "class", GetMessagePath(options, field->containing_type()),
         "settername", "set" + JSGetterName(options, field), "oneoftag",
-        (field->containing_oneof() ? "Oneof" : ""), "repeatedtag",
+        (InRealOneof(field) ? "Oneof" : ""), "repeatedtag",
         (field->is_repeated() ? "Repeated" : ""));
     printer->Annotate("settername", field);
 
@@ -2686,7 +2687,7 @@ void Generator::GenerateClassField(const GeneratorOptions& options,
         "\n"
         "\n",
         "index", JSFieldIndex(field), "oneofgroup",
-        (field->containing_oneof() ? (", " + JSOneofArray(options, field))
+        (InRealOneof(field) ? (", " + JSOneofArray(options, field))
                                    : ""));
 
     if (field->is_repeated()) {
@@ -2811,8 +2812,7 @@ void Generator::GenerateClassField(const GeneratorOptions& options,
           "  return jspb.Message.set$oneoftag$Field(this, $index$",
           "class", GetMessagePath(options, field->containing_type()),
           "settername", "set" + JSGetterName(options, field), "oneoftag",
-          (field->containing_oneof() ? "Oneof" : ""), "index",
-          JSFieldIndex(field));
+          (InRealOneof(field) ? "Oneof" : ""), "index", JSFieldIndex(field));
       printer->Annotate("settername", field);
       printer->Print(
           "$oneofgroup$, $type$value$rptvalueinit$$typeclose$);\n"
@@ -2822,8 +2822,7 @@ void Generator::GenerateClassField(const GeneratorOptions& options,
           "type",
           untyped ? "/** @type{string|number|boolean|Array|undefined} */(" : "",
           "typeclose", untyped ? ")" : "", "oneofgroup",
-          (field->containing_oneof() ? (", " + JSOneofArray(options, field))
-                                     : ""),
+          (InRealOneof(field) ? (", " + JSOneofArray(options, field)) : ""),
           "rptvalueinit", (field->is_repeated() ? " || []" : ""));
     }
 
@@ -2899,8 +2898,8 @@ void Generator::GenerateClassField(const GeneratorOptions& options,
             "$index$$maybeoneofgroup$, ",
         "class", GetMessagePath(options, field->containing_type()),
         "clearername", "clear" + JSGetterName(options, field),
-        "maybeoneof", (field->containing_oneof() ? "Oneof" : ""),
-        "maybeoneofgroup", (field->containing_oneof()
+        "maybeoneof", (InRealOneof(field) ? "Oneof" : ""),
+        "maybeoneofgroup", (InRealOneof(field)
                             ? (", " + JSOneofArray(options, field))
                             : ""),
         "index", JSFieldIndex(field));
@@ -2965,7 +2964,7 @@ void Generator::GenerateRepeatedPrimitiveHelperMethods(
       "\n",
       "type", untyped ? "/** @type{string|number|boolean|!Uint8Array} */(" : "",
       "typeclose", untyped ? ")" : "", "oneofgroup",
-      (field->containing_oneof() ? (", " + JSOneofArray(options, field)) : ""),
+      (InRealOneof(field) ? (", " + JSOneofArray(options, field)) : ""),
       "rptvalueinit", "");
   // clang-format on
 }
@@ -2994,7 +2993,7 @@ void Generator::GenerateRepeatedMessageHelperMethods(
       "\n"
       "\n",
       "index", JSFieldIndex(field), "oneofgroup",
-      (field->containing_oneof() ? (", " + JSOneofArray(options, field)) : ""),
+      (InRealOneof(field) ? (", " + JSOneofArray(options, field)) : ""),
       "ctor", GetMessagePath(options, field->message_type()));
 }
 

src/google/protobuf/compiler/js/js_generator.h

@@ -142,18 +142,21 @@ class PROTOC_EXPORT Generator : public CodeGenerator {
   Generator() {}
   virtual ~Generator() {}
 
-  virtual bool Generate(const FileDescriptor* file,
-                        const std::string& parameter, GeneratorContext* context,
-                        std::string* error) const {
+  bool Generate(const FileDescriptor* file, const std::string& parameter,
+                GeneratorContext* context, std::string* error) const override {
     *error = "Unimplemented Generate() method. Call GenerateAll() instead.";
     return false;
   }
 
-  virtual bool HasGenerateAll() const { return true; }
+  bool HasGenerateAll() const override { return true; }
 
-  virtual bool GenerateAll(const std::vector<const FileDescriptor*>& files,
-                           const std::string& parameter,
-                           GeneratorContext* context, std::string* error) const;
+  bool GenerateAll(const std::vector<const FileDescriptor*>& files,
+                   const std::string& parameter, GeneratorContext* context,
+                   std::string* error) const override;
+
+  uint64 GetSupportedFeatures() const override {
+    return FEATURE_PROTO3_OPTIONAL;
+  }
 
  private:
   void GenerateHeader(const GeneratorOptions& options,

src/google/protobuf/compiler/objectivec/objectivec_file.cc

@@ -209,15 +209,15 @@ FileGenerator::FileGenerator(const FileDescriptor *file, const Options& options)
 FileGenerator::~FileGenerator() {}
 
 void FileGenerator::GenerateHeader(io::Printer *printer) {
-  std::set<string> headers;
+  std::vector<string> headers;
   // Generated files bundled with the library get minimal imports, everything
   // else gets the wrapper so everything is usable.
   if (is_bundled_proto_) {
-    headers.insert("GPBRootObject.h");
-    headers.insert("GPBMessage.h");
-    headers.insert("GPBDescriptor.h");
+    headers.push_back("GPBDescriptor.h");
+    headers.push_back("GPBMessage.h");
+    headers.push_back("GPBRootObject.h");
   } else {
-    headers.insert("GPBProtocolBuffers.h");
+    headers.push_back("GPBProtocolBuffers.h");
   }
   PrintFileRuntimePreamble(printer, headers);
 
@@ -242,6 +242,7 @@ void FileGenerator::GenerateHeader(io::Printer *printer) {
     ImportWriter import_writer(
         options_.generate_for_named_framework,
         options_.named_framework_to_proto_path_mappings_path,
+        options_.runtime_import_prefix,
         is_bundled_proto_);
     const string header_extension(kHeaderExtension);
     for (int i = 0; i < file_->public_dependency_count(); i++) {
@@ -337,8 +338,8 @@ void FileGenerator::GenerateHeader(io::Printer *printer) {
 
 void FileGenerator::GenerateSource(io::Printer *printer) {
   // #import the runtime support.
-  std::set<string> headers;
-  headers.insert("GPBProtocolBuffers_RuntimeSupport.h");
+  std::vector<string> headers;
+  headers.push_back("GPBProtocolBuffers_RuntimeSupport.h");
   PrintFileRuntimePreamble(printer, headers);
 
   // Enums use atomic in the generated code, so add the system import as needed.
@@ -355,6 +356,7 @@ void FileGenerator::GenerateSource(io::Printer *printer) {
     ImportWriter import_writer(
         options_.generate_for_named_framework,
         options_.named_framework_to_proto_path_mappings_path,
+        options_.runtime_import_prefix,
         is_bundled_proto_);
     const string header_extension(kHeaderExtension);
 
@@ -590,48 +592,15 @@ void FileGenerator::GenerateSource(io::Printer *printer) {
 // files. This currently only supports the runtime coming from a framework
 // as defined by the official CocoaPod.
 void FileGenerator::PrintFileRuntimePreamble(
-    io::Printer* printer, const std::set<string>& headers_to_import) const {
+    io::Printer* printer, const std::vector<string>& headers_to_import) const {
   printer->Print(
       "// Generated by the protocol buffer compiler.  DO NOT EDIT!\n"
       "// source: $filename$\n"
       "\n",
       "filename", file_->name());
-
-  const string framework_name(ProtobufLibraryFrameworkName);
-  const string cpp_symbol(ProtobufFrameworkImportSymbol(framework_name));
-
-  printer->Print(
-      "// This CPP symbol can be defined to use imports that match up to the framework\n"
-      "// imports needed when using CocoaPods.\n"
-      "#if !defined($cpp_symbol$)\n"
-      " #define $cpp_symbol$ 0\n"
-      "#endif\n"
-      "\n"
-      "#if $cpp_symbol$\n",
-      "cpp_symbol", cpp_symbol);
-
-
-  for (std::set<string>::const_iterator iter = headers_to_import.begin();
-       iter != headers_to_import.end(); ++iter) {
-    printer->Print(
-        " #import <$framework_name$/$header$>\n",
-        "header", *iter,
-        "framework_name", framework_name);
-  }
-
-  printer->Print(
-      "#else\n");
-
-  for (std::set<string>::const_iterator iter = headers_to_import.begin();
-       iter != headers_to_import.end(); ++iter) {
-    printer->Print(
-        " #import \"$header$\"\n",
-        "header", *iter);
-  }
-
-  printer->Print(
-      "#endif\n"
-      "\n");
+  ImportWriter::PrintRuntimeImports(
+      printer, headers_to_import, options_.runtime_import_prefix, true);
+  printer->Print("\n");
 }
 
 }  // namespace objectivec

src/google/protobuf/compiler/objectivec/objectivec_file.h

@@ -72,7 +72,7 @@ class FileGenerator {
   const Options options_;
 
   void PrintFileRuntimePreamble(
-      io::Printer* printer, const std::set<string>& headers_to_import) const;
+      io::Printer* printer, const std::vector<string>& headers_to_import) const;
 };
 
 }  // namespace objectivec

src/google/protobuf/compiler/objectivec/objectivec_generator.cc

@@ -127,6 +127,13 @@ bool ObjectiveCGenerator::GenerateAll(const std::vector<const FileDescriptor*>&
       // with generate_for_named_framework, or the relative path to it's include
       // path otherwise.
       generation_options.named_framework_to_proto_path_mappings_path = options[i].second;
+    } else if (options[i].first == "runtime_import_prefix") {
+      // Path to use as a prefix on #imports of runtime provided headers in the
+      // generated files. When integrating ObjC protos into a build system,
+      // this can be used to avoid having to add the runtime directory to the
+      // header search path since the generate #import will be more complete.
+      generation_options.runtime_import_prefix =
+          StripSuffixString(options[i].second, "/");
     } else {
       *error = "error: Unknown generator option: " + options[i].first;
       return false;

src/google/protobuf/compiler/objectivec/objectivec_generator.h

@@ -67,7 +67,7 @@ class PROTOC_EXPORT ObjectiveCGenerator : public CodeGenerator {
                    GeneratorContext* context,
                    string* error) const override;
 
-  uint64 GetSupportedFeatures() const override {
+  uint64_t GetSupportedFeatures() const override {
     return FEATURE_PROTO3_OPTIONAL;
   }
 };

src/google/protobuf/compiler/objectivec/objectivec_helpers.cc

@@ -1566,10 +1566,12 @@ bool ParseSimpleFile(
 ImportWriter::ImportWriter(
   const string& generate_for_named_framework,
   const string& named_framework_to_proto_path_mappings_path,
+  const string& runtime_import_prefix,
   bool include_wkt_imports)
     : generate_for_named_framework_(generate_for_named_framework),
       named_framework_to_proto_path_mappings_path_(
           named_framework_to_proto_path_mappings_path),
+      runtime_import_prefix_(runtime_import_prefix),
       include_wkt_imports_(include_wkt_imports),
       need_to_parse_mapping_file_(true) {
 }
@@ -1585,8 +1587,7 @@ void ImportWriter::AddFile(const FileDescriptor* file,
     if (include_wkt_imports_) {
       const string header_name =
         "GPB" + FilePathBasename(file) + header_extension;
-      protobuf_framework_imports_.push_back(header_name);
-      protobuf_non_framework_imports_.push_back(header_name);
+      protobuf_imports_.push_back(header_name);
     }
     return;
   }
@@ -1616,36 +1617,10 @@ void ImportWriter::AddFile(const FileDescriptor* file,
 }
 
 void ImportWriter::Print(io::Printer* printer) const {
-  assert(protobuf_non_framework_imports_.size() ==
-         protobuf_framework_imports_.size());
-
   bool add_blank_line = false;
 
-  if (!protobuf_framework_imports_.empty()) {
-    const string framework_name(ProtobufLibraryFrameworkName);
-    const string cpp_symbol(ProtobufFrameworkImportSymbol(framework_name));
-
-    printer->Print(
-        "#if $cpp_symbol$\n",
-        "cpp_symbol", cpp_symbol);
-    for (std::vector<string>::const_iterator iter = protobuf_framework_imports_.begin();
-         iter != protobuf_framework_imports_.end(); ++iter) {
-      printer->Print(
-          " #import <$framework_name$/$header$>\n",
-          "framework_name", framework_name,
-          "header", *iter);
-    }
-    printer->Print(
-        "#else\n");
-    for (std::vector<string>::const_iterator iter = protobuf_non_framework_imports_.begin();
-         iter != protobuf_non_framework_imports_.end(); ++iter) {
-      printer->Print(
-          " #import \"$header$\"\n",
-          "header", *iter);
-    }
-    printer->Print(
-        "#endif\n");
-
+  if (!protobuf_imports_.empty()) {
+    PrintRuntimeImports(printer, protobuf_imports_, runtime_import_prefix_);
     add_blank_line = true;
   }
 
@@ -1678,6 +1653,57 @@ void ImportWriter::Print(io::Printer* printer) const {
   }
 }
 
+void ImportWriter::PrintRuntimeImports(
+    io::Printer* printer,
+    const std::vector<string>& header_to_import,
+    const string& runtime_import_prefix,
+    bool default_cpp_symbol) {
+
+  // Given an override, use that.
+  if (!runtime_import_prefix.empty()) {
+    for (const auto& header : header_to_import) {
+      printer->Print(
+          " #import \"$import_prefix$/$header$\"\n",
+          "import_prefix", runtime_import_prefix,
+          "header", header);
+    }
+    return;
+  }
+
+  const string framework_name(ProtobufLibraryFrameworkName);
+  const string cpp_symbol(ProtobufFrameworkImportSymbol(framework_name));
+
+  if (default_cpp_symbol) {
+    printer->Print(
+        "// This CPP symbol can be defined to use imports that match up to the framework\n"
+        "// imports needed when using CocoaPods.\n"
+        "#if !defined($cpp_symbol$)\n"
+        " #define $cpp_symbol$ 0\n"
+        "#endif\n"
+        "\n",
+        "cpp_symbol", cpp_symbol);
+  }
+
+  printer->Print(
+      "#if $cpp_symbol$\n",
+      "cpp_symbol", cpp_symbol);
+  for (const auto& header : header_to_import) {
+    printer->Print(
+        " #import <$framework_name$/$header$>\n",
+        "framework_name", framework_name,
+        "header", header);
+  }
+  printer->Print(
+      "#else\n");
+  for (const auto& header : header_to_import) {
+    printer->Print(
+        " #import \"$header$\"\n",
+        "header", header);
+  }
+  printer->Print(
+      "#endif\n");
+}
+
 void ImportWriter::ParseFrameworkMappings() {
   need_to_parse_mapping_file_ = false;
   if (named_framework_to_proto_path_mappings_path_.empty()) {

src/google/protobuf/compiler/objectivec/objectivec_helpers.h

@@ -53,6 +53,7 @@ struct Options {
   std::vector<string> expected_prefixes_suppressions;
   string generate_for_named_framework;
   string named_framework_to_proto_path_mappings_path;
+  string runtime_import_prefix;
 };
 
 // Escape C++ trigraphs by escaping question marks to "\?".
@@ -279,12 +280,18 @@ class PROTOC_EXPORT ImportWriter {
  public:
   ImportWriter(const string& generate_for_named_framework,
                const string& named_framework_to_proto_path_mappings_path,
+               const string& runtime_import_prefix,
                bool include_wkt_imports);
   ~ImportWriter();
 
   void AddFile(const FileDescriptor* file, const string& header_extension);
   void Print(io::Printer *printer) const;
 
+  static void PrintRuntimeImports(io::Printer *printer,
+                                  const std::vector<string>& header_to_import,
+                                  const string& runtime_import_prefix,
+                                  bool default_cpp_symbol = false);
+
  private:
   class ProtoFrameworkCollector : public LineConsumer {
    public:
@@ -301,12 +308,12 @@ class PROTOC_EXPORT ImportWriter {
 
   const string generate_for_named_framework_;
   const string named_framework_to_proto_path_mappings_path_;
+  const string runtime_import_prefix_;
   const bool include_wkt_imports_;
   std::map<string, string> proto_file_to_framework_name_;
   bool need_to_parse_mapping_file_;
 
-  std::vector<string> protobuf_framework_imports_;
-  std::vector<string> protobuf_non_framework_imports_;
+  std::vector<string> protobuf_imports_;
   std::vector<string> other_framework_imports_;
   std::vector<string> other_imports_;
 };

src/google/protobuf/compiler/plugin.pb.h

@@ -13,7 +13,7 @@
 #error incompatible with your Protocol Buffer headers. Please update
 #error your headers.
 #endif
-#if 3012002 < PROTOBUF_MIN_PROTOC_VERSION
+#if 3012003 < PROTOBUF_MIN_PROTOC_VERSION
 #error This file was generated by an older version of protoc which is
 #error incompatible with your Protocol Buffer headers. Please
 #error regenerate this file with a newer version of protoc.

src/google/protobuf/compiler/python/python_generator.cc

@@ -183,7 +183,7 @@ void PrintTopBoilerplate(io::Printer* printer, const FileDescriptor* file,
       "# -*- coding: utf-8 -*-\n"
       "# Generated by the protocol buffer compiler.  DO NOT EDIT!\n"
       "# source: $filename$\n"
-      "\n",
+      "\"\"\"Generated protocol buffer code.\"\"\"\n",
       "filename", file->name());
   if (HasTopLevelEnums(file)) {
     printer->Print(

src/google/protobuf/compiler/ruby/ruby_generator.h

@@ -52,7 +52,7 @@ class PROTOC_EXPORT Generator : public CodeGenerator {
   bool Generate(const FileDescriptor* file, const string& parameter,
                 GeneratorContext* generator_context,
                 string* error) const override;
-  uint64 GetSupportedFeatures() const override {
+  uint64_t GetSupportedFeatures() const override {
     return FEATURE_PROTO3_OPTIONAL;
   }
 };

src/google/protobuf/descriptor.pb.h

@@ -13,7 +13,7 @@
 #error incompatible with your Protocol Buffer headers. Please update
 #error your headers.
 #endif
-#if 3012002 < PROTOBUF_MIN_PROTOC_VERSION
+#if 3012003 < PROTOBUF_MIN_PROTOC_VERSION
 #error This file was generated by an older version of protoc which is
 #error incompatible with your Protocol Buffer headers. Please
 #error regenerate this file with a newer version of protoc.

src/google/protobuf/duration.pb.h

@@ -13,7 +13,7 @@
 #error incompatible with your Protocol Buffer headers. Please update
 #error your headers.
 #endif
-#if 3012002 < PROTOBUF_MIN_PROTOC_VERSION
+#if 3012003 < PROTOBUF_MIN_PROTOC_VERSION
 #error This file was generated by an older version of protoc which is
 #error incompatible with your Protocol Buffer headers. Please
 #error regenerate this file with a newer version of protoc.

src/google/protobuf/empty.pb.h

@@ -13,7 +13,7 @@
 #error incompatible with your Protocol Buffer headers. Please update
 #error your headers.
 #endif
-#if 3012002 < PROTOBUF_MIN_PROTOC_VERSION
+#if 3012003 < PROTOBUF_MIN_PROTOC_VERSION
 #error This file was generated by an older version of protoc which is
 #error incompatible with your Protocol Buffer headers. Please
 #error regenerate this file with a newer version of protoc.

src/google/protobuf/field_mask.pb.h

@@ -13,7 +13,7 @@
 #error incompatible with your Protocol Buffer headers. Please update
 #error your headers.
 #endif
-#if 3012002 < PROTOBUF_MIN_PROTOC_VERSION
+#if 3012003 < PROTOBUF_MIN_PROTOC_VERSION
 #error This file was generated by an older version of protoc which is
 #error incompatible with your Protocol Buffer headers. Please
 #error regenerate this file with a newer version of protoc.

src/google/protobuf/port_def.inc

@@ -300,7 +300,7 @@
 
 // Shared google3/opensource definitions. //////////////////////////////////////
 
-#define PROTOBUF_VERSION 3012002
+#define PROTOBUF_VERSION 3012003
 #define PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC 3012000
 #define PROTOBUF_MIN_PROTOC_VERSION 3012000
 #define PROTOBUF_VERSION_SUFFIX ""

src/google/protobuf/source_context.pb.h

@@ -13,7 +13,7 @@
 #error incompatible with your Protocol Buffer headers. Please update
 #error your headers.
 #endif
-#if 3012002 < PROTOBUF_MIN_PROTOC_VERSION
+#if 3012003 < PROTOBUF_MIN_PROTOC_VERSION
 #error This file was generated by an older version of protoc which is
 #error incompatible with your Protocol Buffer headers. Please
 #error regenerate this file with a newer version of protoc.

src/google/protobuf/struct.pb.h

@@ -13,7 +13,7 @@
 #error incompatible with your Protocol Buffer headers. Please update
 #error your headers.
 #endif
-#if 3012002 < PROTOBUF_MIN_PROTOC_VERSION
+#if 3012003 < PROTOBUF_MIN_PROTOC_VERSION
 #error This file was generated by an older version of protoc which is
 #error incompatible with your Protocol Buffer headers. Please
 #error regenerate this file with a newer version of protoc.

src/google/protobuf/stubs/common.h

@@ -82,7 +82,7 @@ namespace internal {
 
 // The current version, represented as a single integer to make comparison
 // easier:  major * 10^6 + minor * 10^3 + micro
-#define GOOGLE_PROTOBUF_VERSION 3012002
+#define GOOGLE_PROTOBUF_VERSION 3012003
 
 // A suffix string for alpha, beta or rc releases. Empty for stable releases.
 #define GOOGLE_PROTOBUF_VERSION_SUFFIX ""

src/google/protobuf/timestamp.pb.h

@@ -13,7 +13,7 @@
 #error incompatible with your Protocol Buffer headers. Please update
 #error your headers.
 #endif
-#if 3012002 < PROTOBUF_MIN_PROTOC_VERSION
+#if 3012003 < PROTOBUF_MIN_PROTOC_VERSION
 #error This file was generated by an older version of protoc which is
 #error incompatible with your Protocol Buffer headers. Please
 #error regenerate this file with a newer version of protoc.

src/google/protobuf/type.pb.h

@@ -13,7 +13,7 @@
 #error incompatible with your Protocol Buffer headers. Please update
 #error your headers.
 #endif
-#if 3012002 < PROTOBUF_MIN_PROTOC_VERSION
+#if 3012003 < PROTOBUF_MIN_PROTOC_VERSION
 #error This file was generated by an older version of protoc which is
 #error incompatible with your Protocol Buffer headers. Please
 #error regenerate this file with a newer version of protoc.

src/google/protobuf/wrappers.pb.h

@@ -13,7 +13,7 @@
 #error incompatible with your Protocol Buffer headers. Please update
 #error your headers.
 #endif
-#if 3012002 < PROTOBUF_MIN_PROTOC_VERSION
+#if 3012003 < PROTOBUF_MIN_PROTOC_VERSION
 #error This file was generated by an older version of protoc which is
 #error incompatible with your Protocol Buffer headers. Please
 #error regenerate this file with a newer version of protoc.

tests.sh

@@ -305,8 +305,6 @@ build_objectivec_tvos_release() {
 }
 
 build_objectivec_cocoapods_integration() {
-  # Update pod to the latest version.
-  gem install cocoapods --no_document
   objectivec/Tests/CocoaPods/run_tests.sh
 }
 
@@ -457,48 +455,16 @@ build_javascript() {
   cd conformance && make test_nodejs && cd ..
 }
 
-generate_php_test_proto() {
-  internal_build_cpp
-  pushd php/tests
-  # Generate test file
-  rm -rf generated
-  mkdir generated
-  ../../src/protoc --php_out=generated         \
-    -I../../src -I.                            \
-    proto/empty/echo.proto                     \
-    proto/test.proto                           \
-    proto/test_include.proto                   \
-    proto/test_no_namespace.proto              \
-    proto/test_prefix.proto                    \
-    proto/test_php_namespace.proto             \
-    proto/test_empty_php_namespace.proto       \
-    proto/test_reserved_enum_lower.proto       \
-    proto/test_reserved_enum_upper.proto       \
-    proto/test_reserved_enum_value_lower.proto \
-    proto/test_reserved_enum_value_upper.proto \
-    proto/test_reserved_message_lower.proto    \
-    proto/test_reserved_message_upper.proto    \
-    proto/test_service.proto                   \
-    proto/test_service_namespace.proto         \
-    proto/test_wrapper_type_setters.proto      \
-    proto/test_descriptors.proto
-  pushd ../../src
-  ./protoc --php_out=../php/tests/generated -I../php/tests -I. \
-    ../php/tests/proto/test_import_descriptor_proto.proto
-  popd
-  popd
-}
-
 use_php() {
   VERSION=$1
   export PATH=/usr/local/php-${VERSION}/bin:$PATH
-  generate_php_test_proto
+  internal_build_cpp
 }
 
 use_php_zts() {
   VERSION=$1
   export PATH=/usr/local/php-${VERSION}-zts/bin:$PATH
-  generate_php_test_proto
+  internal_build_cpp
 }
 
 build_php5.5() {
@@ -507,11 +473,9 @@ build_php5.5() {
   pushd php
   rm -rf vendor
   composer update
-  ./vendor/bin/phpunit
-  popd
-  pushd conformance
-  make test_php
+  composer test
   popd
+  (cd conformance && make test_php)
 }
 
 build_php5.5_c() {
@@ -534,6 +498,7 @@ build_php5.5_mixed() {
   rm -rf vendor
   composer update
   tests/compile_extension.sh
+  tests/generate_protos.sh
   php -dextension=./ext/google/protobuf/modules/protobuf.so ./vendor/bin/phpunit
   popd
 }
@@ -557,11 +522,9 @@ build_php5.6() {
   pushd php
   rm -rf vendor
   composer update
-  ./vendor/bin/phpunit
-  popd
-  pushd conformance
-  make test_php
+  composer test
   popd
+  (cd conformance && make test_php)
 }
 
 build_php5.6_c() {
@@ -584,6 +547,7 @@ build_php5.6_mixed() {
   rm -rf vendor
   composer update
   tests/compile_extension.sh
+  tests/generate_protos.sh
   php -dextension=./ext/google/protobuf/modules/protobuf.so ./vendor/bin/phpunit
   popd
 }
@@ -603,18 +567,13 @@ build_php5.6_zts_c() {
 }
 
 build_php5.6_mac() {
-  generate_php_test_proto
+  internal_build_cpp
   # Install PHP
   curl -s https://php-osx.liip.ch/install.sh | bash -s 5.6
   PHP_FOLDER=`find /usr/local -type d -name "php5-5.6*"`  # The folder name may change upon time
   test ! -z "$PHP_FOLDER"
   export PATH="$PHP_FOLDER/bin:$PATH"
 
-  # Install phpunit
-  curl https://phar.phpunit.de/phpunit-5.6.8.phar -L -o phpunit.phar
-  chmod +x phpunit.phar
-  sudo mv phpunit.phar /usr/local/bin/phpunit
-
   # Install valgrind
   echo "#! /bin/bash" > valgrind
   chmod ug+x valgrind
@@ -630,7 +589,7 @@ build_php7.0() {
   pushd php
   rm -rf vendor
   composer update
-  ./vendor/bin/phpunit
+  composer test
   popd
   (cd conformance && make test_php)
 }
@@ -655,6 +614,7 @@ build_php7.0_mixed() {
   rm -rf vendor
   composer update
   tests/compile_extension.sh
+  tests/generate_protos.sh
   php -dextension=./ext/google/protobuf/modules/protobuf.so ./vendor/bin/phpunit
   popd
 }
@@ -674,18 +634,13 @@ build_php7.0_zts_c() {
 }
 
 build_php7.0_mac() {
-  generate_php_test_proto
+  internal_build_cpp
   # Install PHP
   curl -s https://php-osx.liip.ch/install.sh | bash -s 7.0
   PHP_FOLDER=`find /usr/local -type d -name "php5-7.0*"`  # The folder name may change upon time
   test ! -z "$PHP_FOLDER"
   export PATH="$PHP_FOLDER/bin:$PATH"
 
-  # Install phpunit
-  curl https://phar.phpunit.de/phpunit-5.6.0.phar -L -o phpunit.phar
-  chmod +x phpunit.phar
-  sudo mv phpunit.phar /usr/local/bin/phpunit
-
   # Install valgrind
   echo "#! /bin/bash" > valgrind
   chmod ug+x valgrind
@@ -697,7 +652,7 @@ build_php7.0_mac() {
 }
 
 build_php7.3_mac() {
-  generate_php_test_proto
+  internal_build_cpp
   # Install PHP
   # We can't test PHP 7.4 with these binaries yet:
   #   https://github.com/liip/php-osx/issues/276
@@ -706,11 +661,6 @@ build_php7.3_mac() {
   test ! -z "$PHP_FOLDER"
   export PATH="$PHP_FOLDER/bin:$PATH"
 
-  # Install phpunit
-  curl https://phar.phpunit.de/phpunit-8.phar -L -o phpunit.phar
-  chmod +x phpunit.phar
-  sudo mv phpunit.phar /usr/local/bin/phpunit
-
   # Install valgrind
   echo "#! /bin/bash" > valgrind
   chmod ug+x valgrind
@@ -736,7 +686,7 @@ build_php7.1() {
   pushd php
   rm -rf vendor
   composer update
-  ./vendor/bin/phpunit
+  composer test
   popd
   (cd conformance && make test_php)
 }
@@ -761,6 +711,7 @@ build_php7.1_mixed() {
   rm -rf vendor
   composer update
   tests/compile_extension.sh
+  tests/generate_protos.sh
   php -dextension=./ext/google/protobuf/modules/protobuf.so ./vendor/bin/phpunit
   popd
 }
@@ -784,7 +735,7 @@ build_php7.4() {
   pushd php
   rm -rf vendor
   composer update
-  ./vendor/bin/phpunit
+  composer test
   popd
   (cd conformance && make test_php)
 }
@@ -810,6 +761,7 @@ build_php7.4_mixed() {
   rm -rf vendor
   composer update
   tests/compile_extension.sh
+  tests/generate_protos.sh
   php -dextension=./ext/google/protobuf/modules/protobuf.so ./vendor/bin/phpunit
   popd
   (cd php/ext/google/protobuf && phpize --clean)