Browse Source

Merge pull request #7485 from haberman/sync-stage

Integrate from Piper for C++, Java, and Python
Joshua Haberman 5 years ago
parent
commit
acbdca9a20

+ 10 - 0
conformance/binary_json_conformance_suite.cc

@@ -1865,6 +1865,16 @@ void BinaryAndJsonConformanceSuite::RunJsonTestsForFieldNameConvention() {
       })",
       })",
       [](const Json::Value& value) { return !value.isMember("FieldName13"); },
       [](const Json::Value& value) { return !value.isMember("FieldName13"); },
       true);
       true);
+  RunValidJsonTestWithValidator(
+      "FieldNameExtension", RECOMMENDED,
+      R"({
+        "[protobuf_test_messages.proto2.extension_int32]": 1
+      })",
+      [](const Json::Value& value) {
+        return value.isMember(
+            "[protobuf_test_messages.proto2.extension_int32]");
+      },
+      false);
 }
 }
 
 
 void BinaryAndJsonConformanceSuite::RunJsonTestsForNonRepeatedTypes() {
 void BinaryAndJsonConformanceSuite::RunJsonTestsForNonRepeatedTypes() {

+ 1 - 0
conformance/failure_list_cpp.txt

@@ -34,3 +34,4 @@ Recommended.Proto3.JsonInput.TrailingCommaInAnObject
 Recommended.Proto3.JsonInput.TrailingCommaInAnObjectWithNewlines
 Recommended.Proto3.JsonInput.TrailingCommaInAnObjectWithNewlines
 Recommended.Proto3.JsonInput.TrailingCommaInAnObjectWithSpace
 Recommended.Proto3.JsonInput.TrailingCommaInAnObjectWithSpace
 Recommended.Proto3.JsonInput.TrailingCommaInAnObjectWithSpaceCommaSpace
 Recommended.Proto3.JsonInput.TrailingCommaInAnObjectWithSpaceCommaSpace
+Recommended.Proto2.JsonInput.FieldNameExtension.Validator

+ 1 - 0
conformance/failure_list_csharp.txt

@@ -1,3 +1,4 @@
 Recommended.Proto3.JsonInput.BytesFieldBase64Url.JsonOutput
 Recommended.Proto3.JsonInput.BytesFieldBase64Url.JsonOutput
 Recommended.Proto3.JsonInput.BytesFieldBase64Url.ProtobufOutput
 Recommended.Proto3.JsonInput.BytesFieldBase64Url.ProtobufOutput
 Required.Proto2.JsonInput.StoresDefaultPrimitive.Validator
 Required.Proto2.JsonInput.StoresDefaultPrimitive.Validator
+Recommended.Proto2.JsonInput.FieldNameExtension.Validator

+ 1 - 0
conformance/failure_list_java.txt

@@ -34,6 +34,7 @@ Recommended.Proto3.JsonInput.StringFieldUnpairedHighSurrogate
 Recommended.Proto3.JsonInput.StringFieldUnpairedLowSurrogate
 Recommended.Proto3.JsonInput.StringFieldUnpairedLowSurrogate
 Recommended.Proto3.JsonInput.Uint32MapFieldKeyNotQuoted
 Recommended.Proto3.JsonInput.Uint32MapFieldKeyNotQuoted
 Recommended.Proto3.JsonInput.Uint64MapFieldKeyNotQuoted
 Recommended.Proto3.JsonInput.Uint64MapFieldKeyNotQuoted
+Recommended.Proto2.JsonInput.FieldNameExtension.Validator
 Required.Proto3.JsonInput.EnumFieldNotQuoted
 Required.Proto3.JsonInput.EnumFieldNotQuoted
 Required.Proto3.JsonInput.Int32FieldLeadingZero
 Required.Proto3.JsonInput.Int32FieldLeadingZero
 Required.Proto3.JsonInput.Int32FieldNegativeWithLeadingZero
 Required.Proto3.JsonInput.Int32FieldNegativeWithLeadingZero

+ 1 - 0
conformance/failure_list_php.txt

@@ -1,6 +1,7 @@
 Recommended.FieldMaskNumbersDontRoundTrip.JsonOutput
 Recommended.FieldMaskNumbersDontRoundTrip.JsonOutput
 Recommended.FieldMaskPathsDontRoundTrip.JsonOutput
 Recommended.FieldMaskPathsDontRoundTrip.JsonOutput
 Recommended.FieldMaskTooManyUnderscore.JsonOutput
 Recommended.FieldMaskTooManyUnderscore.JsonOutput
+Recommended.Proto2.JsonInput.FieldNameExtension.Validator
 Recommended.Proto3.JsonInput.BytesFieldBase64Url.JsonOutput
 Recommended.Proto3.JsonInput.BytesFieldBase64Url.JsonOutput
 Recommended.Proto3.JsonInput.BytesFieldBase64Url.ProtobufOutput
 Recommended.Proto3.JsonInput.BytesFieldBase64Url.ProtobufOutput
 Recommended.Proto3.JsonInput.FieldMaskInvalidCharacter
 Recommended.Proto3.JsonInput.FieldMaskInvalidCharacter

+ 1 - 0
conformance/failure_list_php_c.txt

@@ -1,6 +1,7 @@
 Recommended.FieldMaskNumbersDontRoundTrip.JsonOutput
 Recommended.FieldMaskNumbersDontRoundTrip.JsonOutput
 Recommended.FieldMaskPathsDontRoundTrip.JsonOutput
 Recommended.FieldMaskPathsDontRoundTrip.JsonOutput
 Recommended.FieldMaskTooManyUnderscore.JsonOutput
 Recommended.FieldMaskTooManyUnderscore.JsonOutput
+Recommended.Proto2.JsonInput.FieldNameExtension.Validator
 Recommended.Proto3.JsonInput.BytesFieldBase64Url.JsonOutput
 Recommended.Proto3.JsonInput.BytesFieldBase64Url.JsonOutput
 Recommended.Proto3.JsonInput.BytesFieldBase64Url.ProtobufOutput
 Recommended.Proto3.JsonInput.BytesFieldBase64Url.ProtobufOutput
 Recommended.Proto3.JsonInput.DurationHas3FractionalDigits.Validator
 Recommended.Proto3.JsonInput.DurationHas3FractionalDigits.Validator

+ 1 - 0
conformance/failure_list_php_c_32.txt

@@ -1,6 +1,7 @@
 Recommended.FieldMaskNumbersDontRoundTrip.JsonOutput
 Recommended.FieldMaskNumbersDontRoundTrip.JsonOutput
 Recommended.FieldMaskPathsDontRoundTrip.JsonOutput
 Recommended.FieldMaskPathsDontRoundTrip.JsonOutput
 Recommended.FieldMaskTooManyUnderscore.JsonOutput
 Recommended.FieldMaskTooManyUnderscore.JsonOutput
+Recommended.Proto2.JsonInput.FieldNameExtension.Validator
 Recommended.Proto3.JsonInput.BytesFieldBase64Url.JsonOutput
 Recommended.Proto3.JsonInput.BytesFieldBase64Url.JsonOutput
 Recommended.Proto3.JsonInput.BytesFieldBase64Url.ProtobufOutput
 Recommended.Proto3.JsonInput.BytesFieldBase64Url.ProtobufOutput
 Recommended.Proto3.JsonInput.DurationHas3FractionalDigits.Validator
 Recommended.Proto3.JsonInput.DurationHas3FractionalDigits.Validator

+ 1 - 0
conformance/failure_list_ruby.txt

@@ -1,6 +1,7 @@
 Recommended.FieldMaskNumbersDontRoundTrip.JsonOutput
 Recommended.FieldMaskNumbersDontRoundTrip.JsonOutput
 Recommended.FieldMaskPathsDontRoundTrip.JsonOutput
 Recommended.FieldMaskPathsDontRoundTrip.JsonOutput
 Recommended.FieldMaskTooManyUnderscore.JsonOutput
 Recommended.FieldMaskTooManyUnderscore.JsonOutput
+Recommended.Proto2.JsonInput.FieldNameExtension.Validator
 Recommended.Proto3.JsonInput.BytesFieldBase64Url.JsonOutput
 Recommended.Proto3.JsonInput.BytesFieldBase64Url.JsonOutput
 Recommended.Proto3.JsonInput.BytesFieldBase64Url.ProtobufOutput
 Recommended.Proto3.JsonInput.BytesFieldBase64Url.ProtobufOutput
 Recommended.Proto3.JsonInput.DurationHas3FractionalDigits.Validator
 Recommended.Proto3.JsonInput.DurationHas3FractionalDigits.Validator

+ 1 - 0
java/core/generate-test-sources-build.xml

@@ -17,6 +17,7 @@
         <arg value="${protobuf.source.dir}/google/protobuf/unittest_no_generic_services.proto"/>
         <arg value="${protobuf.source.dir}/google/protobuf/unittest_no_generic_services.proto"/>
         <arg value="${protobuf.source.dir}/google/protobuf/unittest_optimize_for.proto"/>
         <arg value="${protobuf.source.dir}/google/protobuf/unittest_optimize_for.proto"/>
         <arg value="${protobuf.source.dir}/google/protobuf/unittest_proto3.proto"/>
         <arg value="${protobuf.source.dir}/google/protobuf/unittest_proto3.proto"/>
+        <arg value="${protobuf.source.dir}/google/protobuf/unittest_proto3_optional.proto"/>
         <arg value="${protobuf.source.dir}/google/protobuf/unittest_well_known_types.proto"/>
         <arg value="${protobuf.source.dir}/google/protobuf/unittest_well_known_types.proto"/>
         <arg value="${test.proto.dir}/com/google/protobuf/any_test.proto"/>
         <arg value="${test.proto.dir}/com/google/protobuf/any_test.proto"/>
         <arg value="${test.proto.dir}/com/google/protobuf/cached_field_size_test.proto"/>
         <arg value="${test.proto.dir}/com/google/protobuf/cached_field_size_test.proto"/>

+ 108 - 0
java/core/src/test/java/com/google/protobuf/FieldPresenceTest.java

@@ -38,6 +38,7 @@ import com.google.protobuf.Descriptors.OneofDescriptor;
 import com.google.protobuf.FieldPresenceTestProto.TestAllTypes;
 import com.google.protobuf.FieldPresenceTestProto.TestAllTypes;
 import com.google.protobuf.FieldPresenceTestProto.TestOptionalFieldsOnly;
 import com.google.protobuf.FieldPresenceTestProto.TestOptionalFieldsOnly;
 import com.google.protobuf.FieldPresenceTestProto.TestRepeatedFieldsOnly;
 import com.google.protobuf.FieldPresenceTestProto.TestRepeatedFieldsOnly;
+import com.google.protobuf.testing.proto.TestProto3Optional;
 import protobuf_unittest.UnittestProto;
 import protobuf_unittest.UnittestProto;
 import junit.framework.TestCase;
 import junit.framework.TestCase;
 
 
@@ -101,6 +102,113 @@ public class FieldPresenceTest extends TestCase {
         UnittestProto.TestAllTypes.Builder.class, TestAllTypes.Builder.class, "OneofBytes");
         UnittestProto.TestAllTypes.Builder.class, TestAllTypes.Builder.class, "OneofBytes");
   }
   }
 
 
+  public void testHasMethodForProto3Optional() throws Exception {
+    assertFalse(TestProto3Optional.getDefaultInstance().hasOptionalInt32());
+    assertFalse(TestProto3Optional.getDefaultInstance().hasOptionalInt64());
+    assertFalse(TestProto3Optional.getDefaultInstance().hasOptionalUint32());
+    assertFalse(TestProto3Optional.getDefaultInstance().hasOptionalUint64());
+    assertFalse(TestProto3Optional.getDefaultInstance().hasOptionalSint32());
+    assertFalse(TestProto3Optional.getDefaultInstance().hasOptionalSint64());
+    assertFalse(TestProto3Optional.getDefaultInstance().hasOptionalFixed32());
+    assertFalse(TestProto3Optional.getDefaultInstance().hasOptionalFixed64());
+    assertFalse(TestProto3Optional.getDefaultInstance().hasOptionalFloat());
+    assertFalse(TestProto3Optional.getDefaultInstance().hasOptionalDouble());
+    assertFalse(TestProto3Optional.getDefaultInstance().hasOptionalBool());
+    assertFalse(TestProto3Optional.getDefaultInstance().hasOptionalString());
+    assertFalse(TestProto3Optional.getDefaultInstance().hasOptionalBytes());
+
+    TestProto3Optional.Builder builder = TestProto3Optional.newBuilder().setOptionalInt32(0);
+    assertTrue(builder.hasOptionalInt32());
+    assertTrue(builder.build().hasOptionalInt32());
+
+    TestProto3Optional.Builder otherBuilder = TestProto3Optional.newBuilder().setOptionalInt32(1);
+    otherBuilder.mergeFrom(builder.build());
+    assertTrue(otherBuilder.hasOptionalInt32());
+    assertEquals(0, otherBuilder.getOptionalInt32());
+
+    TestProto3Optional.Builder builder3 =
+        TestProto3Optional.newBuilder().setOptionalNestedEnumValue(5);
+    assertTrue(builder3.hasOptionalNestedEnum());
+
+    TestProto3Optional.Builder builder4 =
+        TestProto3Optional.newBuilder().setOptionalNestedEnum(TestProto3Optional.NestedEnum.FOO);
+    assertTrue(builder4.hasOptionalNestedEnum());
+
+    TestProto3Optional proto = TestProto3Optional.parseFrom(builder.build().toByteArray());
+    assertTrue(proto.hasOptionalInt32());
+    assertTrue(proto.toBuilder().hasOptionalInt32());
+  }
+
+  private static void assertProto3OptionalReflection(String name) throws Exception {
+    FieldDescriptor fieldDescriptor = TestProto3Optional.getDescriptor().findFieldByName(name);
+    OneofDescriptor oneofDescriptor = fieldDescriptor.getContainingOneof();
+    assertNotNull(fieldDescriptor.getContainingOneof());
+    assertTrue(fieldDescriptor.hasOptionalKeyword());
+    assertTrue(fieldDescriptor.hasPresence());
+
+    assertFalse(TestProto3Optional.getDefaultInstance().hasOneof(oneofDescriptor));
+    assertNull(TestProto3Optional.getDefaultInstance().getOneofFieldDescriptor(oneofDescriptor));
+
+    TestProto3Optional.Builder builder = TestProto3Optional.newBuilder();
+    builder.setField(fieldDescriptor, fieldDescriptor.getDefaultValue());
+    assertTrue(builder.hasField(fieldDescriptor));
+    assertEquals(fieldDescriptor.getDefaultValue(), builder.getField(fieldDescriptor));
+    assertTrue(builder.build().hasField(fieldDescriptor));
+    assertEquals(fieldDescriptor.getDefaultValue(), builder.build().getField(fieldDescriptor));
+    assertTrue(builder.hasOneof(oneofDescriptor));
+    assertEquals(fieldDescriptor, builder.getOneofFieldDescriptor(oneofDescriptor));
+    assertTrue(builder.build().hasOneof(oneofDescriptor));
+    assertEquals(fieldDescriptor, builder.build().getOneofFieldDescriptor(oneofDescriptor));
+
+    TestProto3Optional.Builder otherBuilder = TestProto3Optional.newBuilder();
+    otherBuilder.mergeFrom(builder.build());
+    assertTrue(otherBuilder.hasField(fieldDescriptor));
+    assertEquals(fieldDescriptor.getDefaultValue(), otherBuilder.getField(fieldDescriptor));
+
+    TestProto3Optional proto = TestProto3Optional.parseFrom(builder.build().toByteArray());
+    assertTrue(proto.hasField(fieldDescriptor));
+    assertTrue(proto.toBuilder().hasField(fieldDescriptor));
+
+    DynamicMessage.Builder dynamicBuilder =
+        DynamicMessage.newBuilder(TestProto3Optional.getDescriptor());
+    dynamicBuilder.setField(fieldDescriptor, fieldDescriptor.getDefaultValue());
+    assertTrue(dynamicBuilder.hasField(fieldDescriptor));
+    assertEquals(fieldDescriptor.getDefaultValue(), dynamicBuilder.getField(fieldDescriptor));
+    assertTrue(dynamicBuilder.build().hasField(fieldDescriptor));
+    assertEquals(
+        fieldDescriptor.getDefaultValue(), dynamicBuilder.build().getField(fieldDescriptor));
+    assertTrue(dynamicBuilder.hasOneof(oneofDescriptor));
+    assertEquals(fieldDescriptor, dynamicBuilder.getOneofFieldDescriptor(oneofDescriptor));
+    assertTrue(dynamicBuilder.build().hasOneof(oneofDescriptor));
+    assertEquals(fieldDescriptor, dynamicBuilder.build().getOneofFieldDescriptor(oneofDescriptor));
+
+    DynamicMessage.Builder otherDynamicBuilder =
+        DynamicMessage.newBuilder(TestProto3Optional.getDescriptor());
+    otherDynamicBuilder.mergeFrom(dynamicBuilder.build());
+    assertTrue(otherDynamicBuilder.hasField(fieldDescriptor));
+    assertEquals(fieldDescriptor.getDefaultValue(), otherDynamicBuilder.getField(fieldDescriptor));
+
+    DynamicMessage dynamicProto =
+        DynamicMessage.parseFrom(TestProto3Optional.getDescriptor(), builder.build().toByteArray());
+    assertTrue(dynamicProto.hasField(fieldDescriptor));
+    assertTrue(dynamicProto.toBuilder().hasField(fieldDescriptor));
+  }
+
+  public void testProto3Optional_reflection() throws Exception {
+    assertProto3OptionalReflection("optional_int32");
+    assertProto3OptionalReflection("optional_int64");
+    assertProto3OptionalReflection("optional_uint32");
+    assertProto3OptionalReflection("optional_uint64");
+    assertProto3OptionalReflection("optional_sint32");
+    assertProto3OptionalReflection("optional_sint64");
+    assertProto3OptionalReflection("optional_fixed32");
+    assertProto3OptionalReflection("optional_fixed64");
+    assertProto3OptionalReflection("optional_float");
+    assertProto3OptionalReflection("optional_double");
+    assertProto3OptionalReflection("optional_bool");
+    assertProto3OptionalReflection("optional_string");
+    assertProto3OptionalReflection("optional_bytes");
+  }
 
 
   public void testOneofEquals() throws Exception {
   public void testOneofEquals() throws Exception {
     TestAllTypes.Builder builder = TestAllTypes.newBuilder();
     TestAllTypes.Builder builder = TestAllTypes.newBuilder();

+ 20 - 0
java/core/src/test/java/com/google/protobuf/TextFormatTest.java

@@ -40,6 +40,8 @@ import com.google.protobuf.Descriptors.Descriptor;
 import com.google.protobuf.Descriptors.FieldDescriptor;
 import com.google.protobuf.Descriptors.FieldDescriptor;
 import com.google.protobuf.Descriptors.FileDescriptor;
 import com.google.protobuf.Descriptors.FileDescriptor;
 import com.google.protobuf.TextFormat.Parser.SingularOverwritePolicy;
 import com.google.protobuf.TextFormat.Parser.SingularOverwritePolicy;
+import com.google.protobuf.testing.proto.TestProto3Optional;
+import com.google.protobuf.testing.proto.TestProto3Optional.NestedEnum;
 import any_test.AnyTestProto.TestAny;
 import any_test.AnyTestProto.TestAny;
 import map_test.MapTestProto.TestMap;
 import map_test.MapTestProto.TestMap;
 import protobuf_unittest.UnittestMset.TestMessageSetExtension1;
 import protobuf_unittest.UnittestMset.TestMessageSetExtension1;
@@ -319,6 +321,24 @@ public class TextFormatTest extends TestCase {
     assertEquals(canonicalExoticText, message.toString());
     assertEquals(canonicalExoticText, message.toString());
   }
   }
 
 
+  public void testRoundtripProto3Optional() throws Exception {
+    Message message =
+        TestProto3Optional.newBuilder()
+            .setOptionalInt32(1)
+            .setOptionalInt64(2)
+            .setOptionalNestedEnum(NestedEnum.BAZ)
+            .build();
+    TestProto3Optional.Builder message2 = TestProto3Optional.newBuilder();
+    TextFormat.merge(message.toString(), message2);
+
+    assertTrue(message2.hasOptionalInt32());
+    assertTrue(message2.hasOptionalInt64());
+    assertTrue(message2.hasOptionalNestedEnum());
+    assertEquals(1, message2.getOptionalInt32());
+    assertEquals(2, message2.getOptionalInt64());
+    assertEquals(NestedEnum.BAZ, message2.getOptionalNestedEnum());
+  }
+
   public void testPrintMessageSet() throws Exception {
   public void testPrintMessageSet() throws Exception {
     TestMessageSet messageSet =
     TestMessageSet messageSet =
         TestMessageSet.newBuilder()
         TestMessageSet.newBuilder()

+ 1 - 0
java/lite/generate-test-sources-build.xml

@@ -15,6 +15,7 @@
         <arg value="${protobuf.source.dir}/google/protobuf/unittest_no_generic_services.proto"/>
         <arg value="${protobuf.source.dir}/google/protobuf/unittest_no_generic_services.proto"/>
         <arg value="${protobuf.source.dir}/google/protobuf/unittest_optimize_for.proto"/>
         <arg value="${protobuf.source.dir}/google/protobuf/unittest_optimize_for.proto"/>
         <arg value="${protobuf.source.dir}/google/protobuf/unittest_proto3.proto"/>
         <arg value="${protobuf.source.dir}/google/protobuf/unittest_proto3.proto"/>
+        <arg value="${protobuf.source.dir}/google/protobuf/unittest_proto3_optional.proto"/>
         <arg value="${protobuf.source.dir}/google/protobuf/unittest_well_known_types.proto"/>
         <arg value="${protobuf.source.dir}/google/protobuf/unittest_well_known_types.proto"/>
         <arg value="${protobuf.basedir}/java/core/${test.proto.dir}/com/google/protobuf/any_test.proto"/>
         <arg value="${protobuf.basedir}/java/core/${test.proto.dir}/com/google/protobuf/any_test.proto"/>
         <arg value="${protobuf.basedir}/java/core/${test.proto.dir}/com/google/protobuf/cached_field_size_test.proto"/>
         <arg value="${protobuf.basedir}/java/core/${test.proto.dir}/com/google/protobuf/cached_field_size_test.proto"/>

+ 1 - 4
python/google/protobuf/internal/test_proto3_optional.proto

@@ -30,10 +30,7 @@
 
 
 syntax = "proto3";
 syntax = "proto3";
 
 
-package protobuf_unittest;
-
-option java_multiple_files = true;
-option java_package = "com.google.protobuf.testing.proto";
+package google.protobuf.python.internal;
 
 
 message TestProto3Optional {
 message TestProto3Optional {
   message NestedMessage {
   message NestedMessage {

+ 3 - 1
src/google/protobuf/api.pb.h

@@ -1063,7 +1063,9 @@ inline void Api::unsafe_arena_set_allocated_source_context(
   // @@protoc_insertion_point(field_unsafe_arena_set_allocated:google.protobuf.Api.source_context)
   // @@protoc_insertion_point(field_unsafe_arena_set_allocated:google.protobuf.Api.source_context)
 }
 }
 inline PROTOBUF_NAMESPACE_ID::SourceContext* Api::release_source_context() {
 inline PROTOBUF_NAMESPACE_ID::SourceContext* Api::release_source_context() {
-  auto temp = unsafe_arena_release_source_context();
+  
+  PROTOBUF_NAMESPACE_ID::SourceContext* temp = source_context_;
+  source_context_ = nullptr;
   if (GetArena() != nullptr) {
   if (GetArena() != nullptr) {
     temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp);
     temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp);
   }
   }

+ 0 - 1
src/google/protobuf/compiler/cpp/cpp_message.cc

@@ -614,7 +614,6 @@ MessageGenerator::MessageGenerator(
     }
     }
   }
   }
 
 
-
   if (!has_bit_indices_.empty()) {
   if (!has_bit_indices_.empty()) {
     field_generators_.SetHasBitIndices(has_bit_indices_);
     field_generators_.SetHasBitIndices(has_bit_indices_);
   }
   }

+ 4 - 1
src/google/protobuf/compiler/cpp/cpp_message_field.cc

@@ -199,7 +199,10 @@ void MessageFieldGenerator::GenerateInlineAccessorDefinitions(
         "}\n");
         "}\n");
     format(
     format(
         "inline $type$* $classname$::$release_name$() {\n"
         "inline $type$* $classname$::$release_name$() {\n"
-        "  auto temp = unsafe_arena_release_$name$();\n"
+        "$type_reference_function$"
+        "  $clear_hasbit$\n"
+        "  $type$* temp = $casted_member$;\n"
+        "  $name$_ = nullptr;\n"
         "  if (GetArena() != nullptr) {\n"
         "  if (GetArena() != nullptr) {\n"
         "    temp = ::$proto_ns$::internal::DuplicateIfNonNull(temp);\n"
         "    temp = ::$proto_ns$::internal::DuplicateIfNonNull(temp);\n"
         "  }\n"
         "  }\n"

+ 1 - 0
src/google/protobuf/compiler/java/java_enum_field.cc

@@ -225,6 +225,7 @@ void ImmutableEnumFieldGenerator::GenerateBuilderMembers(
     printer->Print(variables_,
     printer->Print(variables_,
                    "$deprecation$public Builder "
                    "$deprecation$public Builder "
                    "${$set$capitalized_name$Value$}$(int value) {\n"
                    "${$set$capitalized_name$Value$}$(int value) {\n"
+                   "  $set_has_field_bit_builder$\n"
                    "  $name$_ = value;\n"
                    "  $name$_ = value;\n"
                    "  $on_changed$\n"
                    "  $on_changed$\n"
                    "  return this;\n"
                    "  return this;\n"

+ 3 - 1
src/google/protobuf/compiler/plugin.pb.h

@@ -1441,7 +1441,9 @@ inline void CodeGeneratorRequest::unsafe_arena_set_allocated_compiler_version(
   // @@protoc_insertion_point(field_unsafe_arena_set_allocated:google.protobuf.compiler.CodeGeneratorRequest.compiler_version)
   // @@protoc_insertion_point(field_unsafe_arena_set_allocated:google.protobuf.compiler.CodeGeneratorRequest.compiler_version)
 }
 }
 inline PROTOBUF_NAMESPACE_ID::compiler::Version* CodeGeneratorRequest::release_compiler_version() {
 inline PROTOBUF_NAMESPACE_ID::compiler::Version* CodeGeneratorRequest::release_compiler_version() {
-  auto temp = unsafe_arena_release_compiler_version();
+  _has_bits_[0] &= ~0x00000002u;
+  PROTOBUF_NAMESPACE_ID::compiler::Version* temp = compiler_version_;
+  compiler_version_ = nullptr;
   if (GetArena() != nullptr) {
   if (GetArena() != nullptr) {
     temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp);
     temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp);
   }
   }

+ 30 - 10
src/google/protobuf/descriptor.pb.h

@@ -7484,7 +7484,9 @@ inline void FileDescriptorProto::unsafe_arena_set_allocated_options(
   // @@protoc_insertion_point(field_unsafe_arena_set_allocated:google.protobuf.FileDescriptorProto.options)
   // @@protoc_insertion_point(field_unsafe_arena_set_allocated:google.protobuf.FileDescriptorProto.options)
 }
 }
 inline PROTOBUF_NAMESPACE_ID::FileOptions* FileDescriptorProto::release_options() {
 inline PROTOBUF_NAMESPACE_ID::FileOptions* FileDescriptorProto::release_options() {
-  auto temp = unsafe_arena_release_options();
+  _has_bits_[0] &= ~0x00000008u;
+  PROTOBUF_NAMESPACE_ID::FileOptions* temp = options_;
+  options_ = nullptr;
   if (GetArena() != nullptr) {
   if (GetArena() != nullptr) {
     temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp);
     temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp);
   }
   }
@@ -7565,7 +7567,9 @@ inline void FileDescriptorProto::unsafe_arena_set_allocated_source_code_info(
   // @@protoc_insertion_point(field_unsafe_arena_set_allocated:google.protobuf.FileDescriptorProto.source_code_info)
   // @@protoc_insertion_point(field_unsafe_arena_set_allocated:google.protobuf.FileDescriptorProto.source_code_info)
 }
 }
 inline PROTOBUF_NAMESPACE_ID::SourceCodeInfo* FileDescriptorProto::release_source_code_info() {
 inline PROTOBUF_NAMESPACE_ID::SourceCodeInfo* FileDescriptorProto::release_source_code_info() {
-  auto temp = unsafe_arena_release_source_code_info();
+  _has_bits_[0] &= ~0x00000010u;
+  PROTOBUF_NAMESPACE_ID::SourceCodeInfo* temp = source_code_info_;
+  source_code_info_ = nullptr;
   if (GetArena() != nullptr) {
   if (GetArena() != nullptr) {
     temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp);
     temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp);
   }
   }
@@ -7799,7 +7803,9 @@ inline void DescriptorProto_ExtensionRange::unsafe_arena_set_allocated_options(
   // @@protoc_insertion_point(field_unsafe_arena_set_allocated:google.protobuf.DescriptorProto.ExtensionRange.options)
   // @@protoc_insertion_point(field_unsafe_arena_set_allocated:google.protobuf.DescriptorProto.ExtensionRange.options)
 }
 }
 inline PROTOBUF_NAMESPACE_ID::ExtensionRangeOptions* DescriptorProto_ExtensionRange::release_options() {
 inline PROTOBUF_NAMESPACE_ID::ExtensionRangeOptions* DescriptorProto_ExtensionRange::release_options() {
-  auto temp = unsafe_arena_release_options();
+  _has_bits_[0] &= ~0x00000001u;
+  PROTOBUF_NAMESPACE_ID::ExtensionRangeOptions* temp = options_;
+  options_ = nullptr;
   if (GetArena() != nullptr) {
   if (GetArena() != nullptr) {
     temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp);
     temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp);
   }
   }
@@ -8271,7 +8277,9 @@ inline void DescriptorProto::unsafe_arena_set_allocated_options(
   // @@protoc_insertion_point(field_unsafe_arena_set_allocated:google.protobuf.DescriptorProto.options)
   // @@protoc_insertion_point(field_unsafe_arena_set_allocated:google.protobuf.DescriptorProto.options)
 }
 }
 inline PROTOBUF_NAMESPACE_ID::MessageOptions* DescriptorProto::release_options() {
 inline PROTOBUF_NAMESPACE_ID::MessageOptions* DescriptorProto::release_options() {
-  auto temp = unsafe_arena_release_options();
+  _has_bits_[0] &= ~0x00000002u;
+  PROTOBUF_NAMESPACE_ID::MessageOptions* temp = options_;
+  options_ = nullptr;
   if (GetArena() != nullptr) {
   if (GetArena() != nullptr) {
     temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp);
     temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp);
   }
   }
@@ -9091,7 +9099,9 @@ inline void FieldDescriptorProto::unsafe_arena_set_allocated_options(
   // @@protoc_insertion_point(field_unsafe_arena_set_allocated:google.protobuf.FieldDescriptorProto.options)
   // @@protoc_insertion_point(field_unsafe_arena_set_allocated:google.protobuf.FieldDescriptorProto.options)
 }
 }
 inline PROTOBUF_NAMESPACE_ID::FieldOptions* FieldDescriptorProto::release_options() {
 inline PROTOBUF_NAMESPACE_ID::FieldOptions* FieldDescriptorProto::release_options() {
-  auto temp = unsafe_arena_release_options();
+  _has_bits_[0] &= ~0x00000020u;
+  PROTOBUF_NAMESPACE_ID::FieldOptions* temp = options_;
+  options_ = nullptr;
   if (GetArena() != nullptr) {
   if (GetArena() != nullptr) {
     temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp);
     temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp);
   }
   }
@@ -9297,7 +9307,9 @@ inline void OneofDescriptorProto::unsafe_arena_set_allocated_options(
   // @@protoc_insertion_point(field_unsafe_arena_set_allocated:google.protobuf.OneofDescriptorProto.options)
   // @@protoc_insertion_point(field_unsafe_arena_set_allocated:google.protobuf.OneofDescriptorProto.options)
 }
 }
 inline PROTOBUF_NAMESPACE_ID::OneofOptions* OneofDescriptorProto::release_options() {
 inline PROTOBUF_NAMESPACE_ID::OneofOptions* OneofDescriptorProto::release_options() {
-  auto temp = unsafe_arena_release_options();
+  _has_bits_[0] &= ~0x00000002u;
+  PROTOBUF_NAMESPACE_ID::OneofOptions* temp = options_;
+  options_ = nullptr;
   if (GetArena() != nullptr) {
   if (GetArena() != nullptr) {
     temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp);
     temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp);
   }
   }
@@ -9574,7 +9586,9 @@ inline void EnumDescriptorProto::unsafe_arena_set_allocated_options(
   // @@protoc_insertion_point(field_unsafe_arena_set_allocated:google.protobuf.EnumDescriptorProto.options)
   // @@protoc_insertion_point(field_unsafe_arena_set_allocated:google.protobuf.EnumDescriptorProto.options)
 }
 }
 inline PROTOBUF_NAMESPACE_ID::EnumOptions* EnumDescriptorProto::release_options() {
 inline PROTOBUF_NAMESPACE_ID::EnumOptions* EnumDescriptorProto::release_options() {
-  auto temp = unsafe_arena_release_options();
+  _has_bits_[0] &= ~0x00000002u;
+  PROTOBUF_NAMESPACE_ID::EnumOptions* temp = options_;
+  options_ = nullptr;
   if (GetArena() != nullptr) {
   if (GetArena() != nullptr) {
     temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp);
     temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp);
   }
   }
@@ -9893,7 +9907,9 @@ inline void EnumValueDescriptorProto::unsafe_arena_set_allocated_options(
   // @@protoc_insertion_point(field_unsafe_arena_set_allocated:google.protobuf.EnumValueDescriptorProto.options)
   // @@protoc_insertion_point(field_unsafe_arena_set_allocated:google.protobuf.EnumValueDescriptorProto.options)
 }
 }
 inline PROTOBUF_NAMESPACE_ID::EnumValueOptions* EnumValueDescriptorProto::release_options() {
 inline PROTOBUF_NAMESPACE_ID::EnumValueOptions* EnumValueDescriptorProto::release_options() {
-  auto temp = unsafe_arena_release_options();
+  _has_bits_[0] &= ~0x00000002u;
+  PROTOBUF_NAMESPACE_ID::EnumValueOptions* temp = options_;
+  options_ = nullptr;
   if (GetArena() != nullptr) {
   if (GetArena() != nullptr) {
     temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp);
     temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp);
   }
   }
@@ -10110,7 +10126,9 @@ inline void ServiceDescriptorProto::unsafe_arena_set_allocated_options(
   // @@protoc_insertion_point(field_unsafe_arena_set_allocated:google.protobuf.ServiceDescriptorProto.options)
   // @@protoc_insertion_point(field_unsafe_arena_set_allocated:google.protobuf.ServiceDescriptorProto.options)
 }
 }
 inline PROTOBUF_NAMESPACE_ID::ServiceOptions* ServiceDescriptorProto::release_options() {
 inline PROTOBUF_NAMESPACE_ID::ServiceOptions* ServiceDescriptorProto::release_options() {
-  auto temp = unsafe_arena_release_options();
+  _has_bits_[0] &= ~0x00000002u;
+  PROTOBUF_NAMESPACE_ID::ServiceOptions* temp = options_;
+  options_ = nullptr;
   if (GetArena() != nullptr) {
   if (GetArena() != nullptr) {
     temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp);
     temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp);
   }
   }
@@ -10474,7 +10492,9 @@ inline void MethodDescriptorProto::unsafe_arena_set_allocated_options(
   // @@protoc_insertion_point(field_unsafe_arena_set_allocated:google.protobuf.MethodDescriptorProto.options)
   // @@protoc_insertion_point(field_unsafe_arena_set_allocated:google.protobuf.MethodDescriptorProto.options)
 }
 }
 inline PROTOBUF_NAMESPACE_ID::MethodOptions* MethodDescriptorProto::release_options() {
 inline PROTOBUF_NAMESPACE_ID::MethodOptions* MethodDescriptorProto::release_options() {
-  auto temp = unsafe_arena_release_options();
+  _has_bits_[0] &= ~0x00000008u;
+  PROTOBUF_NAMESPACE_ID::MethodOptions* temp = options_;
+  options_ = nullptr;
   if (GetArena() != nullptr) {
   if (GetArena() != nullptr) {
     temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp);
     temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp);
   }
   }

+ 368 - 52
src/google/protobuf/descriptor_database.cc

@@ -146,6 +146,53 @@ bool SimpleDescriptorDatabase::DescriptorIndex<Value>::AddFile(
   return true;
   return true;
 }
 }
 
 
+namespace {
+
+// Returns true if and only if all characters in the name are alphanumerics,
+// underscores, or periods.
+bool ValidateSymbolName(StringPiece name) {
+  for (char c : name) {
+    // I don't trust ctype.h due to locales.  :(
+    if (c != '.' && c != '_' && (c < '0' || c > '9') && (c < 'A' || c > 'Z') &&
+        (c < 'a' || c > 'z')) {
+      return false;
+    }
+  }
+  return true;
+}
+
+// Find the last key in the container which sorts less than or equal to the
+// symbol name.  Since upper_bound() returns the *first* key that sorts
+// *greater* than the input, we want the element immediately before that.
+template <typename Container, typename Key>
+typename Container::const_iterator FindLastLessOrEqual(Container* container,
+                                                       const Key& key) {
+  auto iter = container->upper_bound(key);
+  if (iter != container->begin()) --iter;
+  return iter;
+}
+
+// As above, but using std::upper_bound instead.
+template <typename Container, typename Key, typename Cmp>
+typename Container::const_iterator FindLastLessOrEqual(Container* container,
+                                                       const Key& key,
+                                                       const Cmp& cmp) {
+  auto iter = std::upper_bound(container->begin(), container->end(), key, cmp);
+  if (iter != container->begin()) --iter;
+  return iter;
+}
+
+// True if either the arguments are equal or super_symbol identifies a
+// parent symbol of sub_symbol (e.g. "foo.bar" is a parent of
+// "foo.bar.baz", but not a parent of "foo.barbaz").
+bool IsSubSymbol(StringPiece sub_symbol, StringPiece super_symbol) {
+  return sub_symbol == super_symbol ||
+         (HasPrefixString(super_symbol, sub_symbol) &&
+          super_symbol[sub_symbol.size()] == '.');
+}
+
+}  // namespace
+
 template <typename Value>
 template <typename Value>
 bool SimpleDescriptorDatabase::DescriptorIndex<Value>::AddSymbol(
 bool SimpleDescriptorDatabase::DescriptorIndex<Value>::AddSymbol(
     const std::string& name, Value value) {
     const std::string& name, Value value) {
@@ -161,8 +208,7 @@ bool SimpleDescriptorDatabase::DescriptorIndex<Value>::AddSymbol(
 
 
   // Try to look up the symbol to make sure a super-symbol doesn't already
   // Try to look up the symbol to make sure a super-symbol doesn't already
   // exist.
   // exist.
-  typename std::map<std::string, Value>::iterator iter =
-      FindLastLessOrEqual(name);
+  auto iter = FindLastLessOrEqual(&by_symbol_, name);
 
 
   if (iter == by_symbol_.end()) {
   if (iter == by_symbol_.end()) {
     // Apparently the map is currently empty.  Just insert and be done with it.
     // Apparently the map is currently empty.  Just insert and be done with it.
@@ -252,8 +298,7 @@ Value SimpleDescriptorDatabase::DescriptorIndex<Value>::FindFile(
 template <typename Value>
 template <typename Value>
 Value SimpleDescriptorDatabase::DescriptorIndex<Value>::FindSymbol(
 Value SimpleDescriptorDatabase::DescriptorIndex<Value>::FindSymbol(
     const std::string& name) {
     const std::string& name) {
-  typename std::map<std::string, Value>::iterator iter =
-      FindLastLessOrEqual(name);
+  auto iter = FindLastLessOrEqual(&by_symbol_, name);
 
 
   return (iter != by_symbol_.end() && IsSubSymbol(iter->first, name))
   return (iter != by_symbol_.end() && IsSubSymbol(iter->first, name))
              ? iter->second
              ? iter->second
@@ -294,40 +339,6 @@ void SimpleDescriptorDatabase::DescriptorIndex<Value>::FindAllFileNames(
   }
   }
 }
 }
 
 
-template <typename Value>
-typename std::map<std::string, Value>::iterator
-SimpleDescriptorDatabase::DescriptorIndex<Value>::FindLastLessOrEqual(
-    const std::string& name) {
-  // Find the last key in the map which sorts less than or equal to the
-  // symbol name.  Since upper_bound() returns the *first* key that sorts
-  // *greater* than the input, we want the element immediately before that.
-  typename std::map<std::string, Value>::iterator iter =
-      by_symbol_.upper_bound(name);
-  if (iter != by_symbol_.begin()) --iter;
-  return iter;
-}
-
-template <typename Value>
-bool SimpleDescriptorDatabase::DescriptorIndex<Value>::IsSubSymbol(
-    const std::string& sub_symbol, const std::string& super_symbol) {
-  return sub_symbol == super_symbol ||
-         (HasPrefixString(super_symbol, sub_symbol) &&
-          super_symbol[sub_symbol.size()] == '.');
-}
-
-template <typename Value>
-bool SimpleDescriptorDatabase::DescriptorIndex<Value>::ValidateSymbolName(
-    const std::string& name) {
-  for (int i = 0; i < name.size(); i++) {
-    // I don't trust ctype.h due to locales.  :(
-    if (name[i] != '.' && name[i] != '_' && (name[i] < '0' || name[i] > '9') &&
-        (name[i] < 'A' || name[i] > 'Z') && (name[i] < 'a' || name[i] > 'z')) {
-      return false;
-    }
-  }
-  return true;
-}
-
 // -------------------------------------------------------------------
 // -------------------------------------------------------------------
 
 
 bool SimpleDescriptorDatabase::Add(const FileDescriptorProto& file) {
 bool SimpleDescriptorDatabase::Add(const FileDescriptorProto& file) {
@@ -378,19 +389,88 @@ bool SimpleDescriptorDatabase::MaybeCopy(const FileDescriptorProto* file,
 
 
 // -------------------------------------------------------------------
 // -------------------------------------------------------------------
 
 
-EncodedDescriptorDatabase::EncodedDescriptorDatabase() {}
-EncodedDescriptorDatabase::~EncodedDescriptorDatabase() {
-  for (int i = 0; i < files_to_delete_.size(); i++) {
-    operator delete(files_to_delete_[i]);
-  }
-}
+class EncodedDescriptorDatabase::DescriptorIndex {
+ public:
+  using Value = std::pair<const void*, int>;
+  // Helpers to recursively add particular descriptors and all their contents
+  // to the index.
+  bool AddFile(const FileDescriptorProto& file, Value value);
+
+  Value FindFile(StringPiece filename);
+  Value FindSymbol(StringPiece name);
+  Value FindSymbolOnlyFlat(StringPiece name) const;
+  Value FindExtension(StringPiece containing_type, int field_number);
+  bool FindAllExtensionNumbers(StringPiece containing_type,
+                               std::vector<int>* output);
+  void FindAllFileNames(std::vector<std::string>* output) const;
+
+ private:
+  friend class EncodedDescriptorDatabase;
+
+  bool AddSymbol(StringPiece name, Value value);
+  bool AddNestedExtensions(StringPiece filename,
+                           const DescriptorProto& message_type, Value value);
+  bool AddExtension(StringPiece filename,
+                    const FieldDescriptorProto& field, Value value);
+
+  // All the maps below have two representations:
+  //  - a std::set<> where we insert initially.
+  //  - a std::vector<> where we flatten the structure on demand.
+  // The initial tree helps avoid O(N) behavior of inserting into a sorted
+  // vector, while the vector reduces the heap requirements of the data
+  // structure.
+
+  void EnsureFlat();
+
+  struct Entry {
+    std::string name;
+    Value data;
+  };
+  struct Compare {
+    bool operator()(const Entry& a, const Entry& b) const {
+      return a.name < b.name;
+    }
+    bool operator()(const Entry& a, StringPiece b) const {
+      return a.name < b;
+    }
+    bool operator()(StringPiece a, const Entry& b) const {
+      return a < b.name;
+    }
+  };
+  std::set<Entry, Compare> by_name_;
+  std::vector<Entry> by_name_flat_;
+  std::set<Entry, Compare> by_symbol_;
+  std::vector<Entry> by_symbol_flat_;
+  struct ExtensionEntry {
+    std::string extendee;
+    int extension_number;
+    Value data;
+  };
+  struct ExtensionCompare {
+    bool operator()(const ExtensionEntry& a, const ExtensionEntry& b) const {
+      return std::tie(a.extendee, a.extension_number) <
+             std::tie(b.extendee, b.extension_number);
+    }
+    bool operator()(const ExtensionEntry& a,
+                    std::tuple<StringPiece, int> b) const {
+      return std::tie(a.extendee, a.extension_number) < b;
+    }
+    bool operator()(std::tuple<StringPiece, int> a,
+                    const ExtensionEntry& b) const {
+      return a < std::tie(b.extendee, b.extension_number);
+    }
+  };
+  std::set<ExtensionEntry, ExtensionCompare> by_extension_;
+  std::vector<ExtensionEntry> by_extension_flat_;
+};
 
 
 bool EncodedDescriptorDatabase::Add(const void* encoded_file_descriptor,
 bool EncodedDescriptorDatabase::Add(const void* encoded_file_descriptor,
                                     int size) {
                                     int size) {
   google::protobuf::Arena arena;
   google::protobuf::Arena arena;
   auto* file = google::protobuf::Arena::CreateMessage<FileDescriptorProto>(&arena);
   auto* file = google::protobuf::Arena::CreateMessage<FileDescriptorProto>(&arena);
   if (file->ParseFromArray(encoded_file_descriptor, size)) {
   if (file->ParseFromArray(encoded_file_descriptor, size)) {
-    return index_.AddFile(*file, std::make_pair(encoded_file_descriptor, size));
+    return index_->AddFile(*file,
+                           std::make_pair(encoded_file_descriptor, size));
   } else {
   } else {
     GOOGLE_LOG(ERROR) << "Invalid file descriptor data passed to "
     GOOGLE_LOG(ERROR) << "Invalid file descriptor data passed to "
                   "EncodedDescriptorDatabase::Add().";
                   "EncodedDescriptorDatabase::Add().";
@@ -408,22 +488,22 @@ bool EncodedDescriptorDatabase::AddCopy(const void* encoded_file_descriptor,
 
 
 bool EncodedDescriptorDatabase::FindFileByName(const std::string& filename,
 bool EncodedDescriptorDatabase::FindFileByName(const std::string& filename,
                                                FileDescriptorProto* output) {
                                                FileDescriptorProto* output) {
-  return MaybeParse(index_.FindFile(filename), output);
+  return MaybeParse(index_->FindFile(filename), output);
 }
 }
 
 
 bool EncodedDescriptorDatabase::FindFileContainingSymbol(
 bool EncodedDescriptorDatabase::FindFileContainingSymbol(
     const std::string& symbol_name, FileDescriptorProto* output) {
     const std::string& symbol_name, FileDescriptorProto* output) {
-  return MaybeParse(index_.FindSymbol(symbol_name), output);
+  return MaybeParse(index_->FindSymbol(symbol_name), output);
 }
 }
 
 
 bool EncodedDescriptorDatabase::FindNameOfFileContainingSymbol(
 bool EncodedDescriptorDatabase::FindNameOfFileContainingSymbol(
     const std::string& symbol_name, std::string* output) {
     const std::string& symbol_name, std::string* output) {
-  std::pair<const void*, int> encoded_file = index_.FindSymbol(symbol_name);
+  auto encoded_file = index_->FindSymbol(symbol_name);
   if (encoded_file.first == NULL) return false;
   if (encoded_file.first == NULL) return false;
 
 
   // Optimization:  The name should be the first field in the encoded message.
   // Optimization:  The name should be the first field in the encoded message.
   //   Try to just read it directly.
   //   Try to just read it directly.
-  io::CodedInputStream input(reinterpret_cast<const uint8*>(encoded_file.first),
+  io::CodedInputStream input(static_cast<const uint8*>(encoded_file.first),
                              encoded_file.second);
                              encoded_file.second);
 
 
   const uint32 kNameTag = internal::WireFormatLite::MakeTag(
   const uint32 kNameTag = internal::WireFormatLite::MakeTag(
@@ -447,18 +527,245 @@ bool EncodedDescriptorDatabase::FindNameOfFileContainingSymbol(
 bool EncodedDescriptorDatabase::FindFileContainingExtension(
 bool EncodedDescriptorDatabase::FindFileContainingExtension(
     const std::string& containing_type, int field_number,
     const std::string& containing_type, int field_number,
     FileDescriptorProto* output) {
     FileDescriptorProto* output) {
-  return MaybeParse(index_.FindExtension(containing_type, field_number),
+  return MaybeParse(index_->FindExtension(containing_type, field_number),
                     output);
                     output);
 }
 }
 
 
 bool EncodedDescriptorDatabase::FindAllExtensionNumbers(
 bool EncodedDescriptorDatabase::FindAllExtensionNumbers(
     const std::string& extendee_type, std::vector<int>* output) {
     const std::string& extendee_type, std::vector<int>* output) {
-  return index_.FindAllExtensionNumbers(extendee_type, output);
+  return index_->FindAllExtensionNumbers(extendee_type, output);
+}
+
+bool EncodedDescriptorDatabase::DescriptorIndex::AddFile(
+    const FileDescriptorProto& file, Value value) {
+  if (!InsertIfNotPresent(&by_name_, Entry{file.name(), value}) ||
+      std::binary_search(by_name_flat_.begin(), by_name_flat_.end(),
+                         file.name(), by_name_.key_comp())) {
+    GOOGLE_LOG(ERROR) << "File already exists in database: " << file.name();
+    return false;
+  }
+
+  // We must be careful here -- calling file.package() if file.has_package() is
+  // false could access an uninitialized static-storage variable if we are being
+  // run at startup time.
+  std::string path = file.has_package() ? file.package() : std::string();
+  if (!path.empty()) path += '.';
+
+  for (const auto& message_type : file.message_type()) {
+    if (!AddSymbol(path + message_type.name(), value)) return false;
+    if (!AddNestedExtensions(file.name(), message_type, value)) return false;
+  }
+  for (const auto& enum_type : file.enum_type()) {
+    if (!AddSymbol(path + enum_type.name(), value)) return false;
+  }
+  for (const auto& extension : file.extension()) {
+    if (!AddSymbol(path + extension.name(), value)) return false;
+    if (!AddExtension(file.name(), extension, value)) return false;
+  }
+  for (const auto& service : file.service()) {
+    if (!AddSymbol(path + service.name(), value)) return false;
+  }
+
+  return true;
 }
 }
 
 
+template <typename Iter, typename Iter2>
+static bool CheckForMutualSubsymbols(StringPiece symbol_name, Iter* iter,
+                                     Iter2 end) {
+  if (*iter != end) {
+    if (IsSubSymbol((*iter)->name, symbol_name)) {
+      GOOGLE_LOG(ERROR) << "Symbol name \"" << symbol_name
+                 << "\" conflicts with the existing symbol \"" << (*iter)->name
+                 << "\".";
+      return false;
+    }
+
+    // OK, that worked.  Now we have to make sure that no symbol in the map is
+    // a sub-symbol of the one we are inserting.  The only symbol which could
+    // be so is the first symbol that is greater than the new symbol.  Since
+    // |iter| points at the last symbol that is less than or equal, we just have
+    // to increment it.
+    ++*iter;
+
+    if (*iter != end && IsSubSymbol(symbol_name, (*iter)->name)) {
+      GOOGLE_LOG(ERROR) << "Symbol name \"" << symbol_name
+                 << "\" conflicts with the existing symbol \"" << (*iter)->name
+                 << "\".";
+      return false;
+    }
+  }
+  return true;
+}
+
+bool EncodedDescriptorDatabase::DescriptorIndex::AddSymbol(
+    StringPiece name, Value value) {
+  // We need to make sure not to violate our map invariant.
+
+  // If the symbol name is invalid it could break our lookup algorithm (which
+  // relies on the fact that '.' sorts before all other characters that are
+  // valid in symbol names).
+  if (!ValidateSymbolName(name)) {
+    GOOGLE_LOG(ERROR) << "Invalid symbol name: " << name;
+    return false;
+  }
+
+  Entry entry = {std::string(name), value};
+
+  auto iter = FindLastLessOrEqual(&by_symbol_, entry);
+  if (!CheckForMutualSubsymbols(name, &iter, by_symbol_.end())) {
+    return false;
+  }
+
+  // Same, but on by_symbol_flat_
+  auto flat_iter =
+      FindLastLessOrEqual(&by_symbol_flat_, name, by_symbol_.key_comp());
+  if (!CheckForMutualSubsymbols(name, &flat_iter, by_symbol_flat_.end())) {
+    return false;
+  }
+
+  // OK, no conflicts.
+
+  // Insert the new symbol using the iterator as a hint, the new entry will
+  // appear immediately before the one the iterator is pointing at.
+  by_symbol_.insert(iter, std::move(entry));
+
+  return true;
+}
+
+bool EncodedDescriptorDatabase::DescriptorIndex::AddNestedExtensions(
+    StringPiece filename, const DescriptorProto& message_type,
+    Value value) {
+  for (const auto& nested_type : message_type.nested_type()) {
+    if (!AddNestedExtensions(filename, nested_type, value)) return false;
+  }
+  for (const auto& extension : message_type.extension()) {
+    if (!AddExtension(filename, extension, value)) return false;
+  }
+  return true;
+}
+
+bool EncodedDescriptorDatabase::DescriptorIndex::AddExtension(
+    StringPiece filename, const FieldDescriptorProto& field,
+    Value value) {
+  if (!field.extendee().empty() && field.extendee()[0] == '.') {
+    // The extension is fully-qualified.  We can use it as a lookup key in
+    // the by_symbol_ table.
+    if (!InsertIfNotPresent(&by_extension_,
+                                 ExtensionEntry{field.extendee().substr(1),
+                                                field.number(), value}) ||
+        std::binary_search(
+            by_extension_flat_.begin(), by_extension_flat_.end(),
+            std::make_pair(field.extendee().substr(1), field.number()),
+            by_extension_.key_comp())) {
+      GOOGLE_LOG(ERROR) << "Extension conflicts with extension already in database: "
+                    "extend "
+                 << field.extendee() << " { " << field.name() << " = "
+                 << field.number() << " } from:" << filename;
+      return false;
+    }
+  } else {
+    // Not fully-qualified.  We can't really do anything here, unfortunately.
+    // We don't consider this an error, though, because the descriptor is
+    // valid.
+  }
+  return true;
+}
+
+std::pair<const void*, int>
+EncodedDescriptorDatabase::DescriptorIndex::FindSymbol(StringPiece name) {
+  EnsureFlat();
+  return FindSymbolOnlyFlat(name);
+}
+
+std::pair<const void*, int>
+EncodedDescriptorDatabase::DescriptorIndex::FindSymbolOnlyFlat(
+    StringPiece name) const {
+  auto iter =
+      FindLastLessOrEqual(&by_symbol_flat_, name, by_symbol_.key_comp());
+
+  return iter != by_symbol_flat_.end() && IsSubSymbol(iter->name, name)
+             ? iter->data
+             : Value();
+}
+
+std::pair<const void*, int>
+EncodedDescriptorDatabase::DescriptorIndex::FindExtension(
+    StringPiece containing_type, int field_number) {
+  EnsureFlat();
+
+  auto it = std::lower_bound(
+      by_extension_flat_.begin(), by_extension_flat_.end(),
+      std::make_tuple(containing_type, field_number), by_extension_.key_comp());
+  return it == by_extension_flat_.end() || it->extendee != containing_type ||
+                 it->extension_number != field_number
+             ? std::make_pair(nullptr, 0)
+             : it->data;
+}
+
+template <typename T, typename Less>
+static void MergeIntoFlat(std::set<T, Less>* s, std::vector<T>* flat) {
+  if (s->empty()) return;
+  std::vector<T> new_flat(s->size() + flat->size());
+  std::merge(s->begin(), s->end(), flat->begin(), flat->end(), &new_flat[0],
+             s->key_comp());
+  *flat = std::move(new_flat);
+  s->clear();
+}
+
+void EncodedDescriptorDatabase::DescriptorIndex::EnsureFlat() {
+  // Merge each of the sets into their flat counterpart.
+  MergeIntoFlat(&by_name_, &by_name_flat_);
+  MergeIntoFlat(&by_symbol_, &by_symbol_flat_);
+  MergeIntoFlat(&by_extension_, &by_extension_flat_);
+}
+
+bool EncodedDescriptorDatabase::DescriptorIndex::FindAllExtensionNumbers(
+    StringPiece containing_type, std::vector<int>* output) {
+  EnsureFlat();
+
+  bool success = false;
+  auto it = std::lower_bound(
+      by_extension_flat_.begin(), by_extension_flat_.end(),
+      std::make_tuple(containing_type, 0), by_extension_.key_comp());
+  for (; it != by_extension_flat_.end() && it->extendee == containing_type;
+       ++it) {
+    output->push_back(it->extension_number);
+    success = true;
+  }
+
+  return success;
+}
+
+void EncodedDescriptorDatabase::DescriptorIndex::FindAllFileNames(
+    std::vector<std::string>* output) const {
+  output->resize(by_name_.size() + by_name_flat_.size());
+  int i = 0;
+  for (const auto& entry : by_name_) {
+    (*output)[i] = entry.name;
+    i++;
+  }
+  for (const auto& entry : by_name_flat_) {
+    (*output)[i] = entry.name;
+    i++;
+  }
+}
+
+std::pair<const void*, int>
+EncodedDescriptorDatabase::DescriptorIndex::FindFile(
+    StringPiece filename) {
+  EnsureFlat();
+
+  auto it = std::lower_bound(by_name_flat_.begin(), by_name_flat_.end(),
+                             filename, by_name_.key_comp());
+  return it == by_name_flat_.end() || it->name != filename
+             ? std::make_pair(nullptr, 0)
+             : it->data;
+}
+
+
 bool EncodedDescriptorDatabase::FindAllFileNames(
 bool EncodedDescriptorDatabase::FindAllFileNames(
     std::vector<std::string>* output) {
     std::vector<std::string>* output) {
-  index_.FindAllFileNames(output);
+  index_->FindAllFileNames(output);
   return true;
   return true;
 }
 }
 
 
@@ -468,6 +775,15 @@ bool EncodedDescriptorDatabase::MaybeParse(
   return output->ParseFromArray(encoded_file.first, encoded_file.second);
   return output->ParseFromArray(encoded_file.first, encoded_file.second);
 }
 }
 
 
+EncodedDescriptorDatabase::EncodedDescriptorDatabase()
+    : index_(new DescriptorIndex()) {}
+
+EncodedDescriptorDatabase::~EncodedDescriptorDatabase() {
+  for (void* p : files_to_delete_) {
+    operator delete(p);
+  }
+}
+
 // ===================================================================
 // ===================================================================
 
 
 DescriptorPoolDatabase::DescriptorPoolDatabase(const DescriptorPool& pool)
 DescriptorPoolDatabase::DescriptorPoolDatabase(const DescriptorPool& pool)

+ 4 - 17
src/google/protobuf/descriptor_database.h

@@ -266,21 +266,6 @@ class PROTOBUF_EXPORT SimpleDescriptorDatabase : public DescriptorDatabase {
     // That symbol cannot be a super-symbol of the search key since if it were,
     // That symbol cannot be a super-symbol of the search key since if it were,
     // then it would be a match, and we're assuming the match key doesn't exist.
     // then it would be a match, and we're assuming the match key doesn't exist.
     // Therefore, step 2 will correctly return no match.
     // Therefore, step 2 will correctly return no match.
-
-    // Find the last entry in the by_symbol_ map whose key is less than or
-    // equal to the given name.
-    typename std::map<std::string, Value>::iterator FindLastLessOrEqual(
-        const std::string& name);
-
-    // True if either the arguments are equal or super_symbol identifies a
-    // parent symbol of sub_symbol (e.g. "foo.bar" is a parent of
-    // "foo.bar.baz", but not a parent of "foo.barbaz").
-    bool IsSubSymbol(const std::string& sub_symbol,
-                     const std::string& super_symbol);
-
-    // Returns true if and only if all characters in the name are alphanumerics,
-    // underscores, or periods.
-    bool ValidateSymbolName(const std::string& name);
   };
   };
 
 
   DescriptorIndex<const FileDescriptorProto*> index_;
   DescriptorIndex<const FileDescriptorProto*> index_;
@@ -332,8 +317,10 @@ class PROTOBUF_EXPORT EncodedDescriptorDatabase : public DescriptorDatabase {
   bool FindAllFileNames(std::vector<std::string>* output) override;
   bool FindAllFileNames(std::vector<std::string>* output) override;
 
 
  private:
  private:
-  SimpleDescriptorDatabase::DescriptorIndex<std::pair<const void*, int> >
-      index_;
+  class DescriptorIndex;
+  // Keep DescriptorIndex by pointer to hide the implementation to keep a
+  // cleaner header.
+  std::unique_ptr<DescriptorIndex> index_;
   std::vector<void*> files_to_delete_;
   std::vector<void*> files_to_delete_;
 
 
   // If encoded_file.first is non-NULL, parse the data into *output and return
   // If encoded_file.first is non-NULL, parse the data into *output and return

+ 1 - 1
src/google/protobuf/io/printer.cc

@@ -305,7 +305,7 @@ void Printer::FormatInternal(const std::vector<std::string>& args,
 }
 }
 
 
 const char* Printer::WriteVariable(
 const char* Printer::WriteVariable(
-    const std::vector<string>& args,
+    const std::vector<std::string>& args,
     const std::map<std::string, std::string>& vars, const char* format,
     const std::map<std::string, std::string>& vars, const char* format,
     int* arg_index, std::vector<AnnotationCollector::Annotation>* annotations) {
     int* arg_index, std::vector<AnnotationCollector::Annotation>* annotations) {
   auto start = format;
   auto start = format;

+ 2 - 2
src/google/protobuf/io/printer_unittest.cc

@@ -573,7 +573,7 @@ TEST(Printer, WriteFailurePartial) {
   EXPECT_TRUE(printer.failed());
   EXPECT_TRUE(printer.failed());
 
 
   // Buffer should contain the first 17 bytes written.
   // Buffer should contain the first 17 bytes written.
-  EXPECT_EQ("0123456789abcdef<", string(buffer, sizeof(buffer)));
+  EXPECT_EQ("0123456789abcdef<", std::string(buffer, sizeof(buffer)));
 }
 }
 
 
 TEST(Printer, WriteFailureExact) {
 TEST(Printer, WriteFailureExact) {
@@ -595,7 +595,7 @@ TEST(Printer, WriteFailureExact) {
   EXPECT_TRUE(printer.failed());
   EXPECT_TRUE(printer.failed());
 
 
   // Buffer should contain the first 16 bytes written.
   // Buffer should contain the first 16 bytes written.
-  EXPECT_EQ("0123456789abcdef", string(buffer, sizeof(buffer)));
+  EXPECT_EQ("0123456789abcdef", std::string(buffer, sizeof(buffer)));
 }
 }
 
 
 TEST(Printer, FormatInternal) {
 TEST(Printer, FormatInternal) {

+ 9 - 3
src/google/protobuf/type.pb.h

@@ -1689,7 +1689,9 @@ inline void Type::unsafe_arena_set_allocated_source_context(
   // @@protoc_insertion_point(field_unsafe_arena_set_allocated:google.protobuf.Type.source_context)
   // @@protoc_insertion_point(field_unsafe_arena_set_allocated:google.protobuf.Type.source_context)
 }
 }
 inline PROTOBUF_NAMESPACE_ID::SourceContext* Type::release_source_context() {
 inline PROTOBUF_NAMESPACE_ID::SourceContext* Type::release_source_context() {
-  auto temp = unsafe_arena_release_source_context();
+  
+  PROTOBUF_NAMESPACE_ID::SourceContext* temp = source_context_;
+  source_context_ = nullptr;
   if (GetArena() != nullptr) {
   if (GetArena() != nullptr) {
     temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp);
     temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp);
   }
   }
@@ -2414,7 +2416,9 @@ inline void Enum::unsafe_arena_set_allocated_source_context(
   // @@protoc_insertion_point(field_unsafe_arena_set_allocated:google.protobuf.Enum.source_context)
   // @@protoc_insertion_point(field_unsafe_arena_set_allocated:google.protobuf.Enum.source_context)
 }
 }
 inline PROTOBUF_NAMESPACE_ID::SourceContext* Enum::release_source_context() {
 inline PROTOBUF_NAMESPACE_ID::SourceContext* Enum::release_source_context() {
-  auto temp = unsafe_arena_release_source_context();
+  
+  PROTOBUF_NAMESPACE_ID::SourceContext* temp = source_context_;
+  source_context_ = nullptr;
   if (GetArena() != nullptr) {
   if (GetArena() != nullptr) {
     temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp);
     temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp);
   }
   }
@@ -2738,7 +2742,9 @@ inline void Option::unsafe_arena_set_allocated_value(
   // @@protoc_insertion_point(field_unsafe_arena_set_allocated:google.protobuf.Option.value)
   // @@protoc_insertion_point(field_unsafe_arena_set_allocated:google.protobuf.Option.value)
 }
 }
 inline PROTOBUF_NAMESPACE_ID::Any* Option::release_value() {
 inline PROTOBUF_NAMESPACE_ID::Any* Option::release_value() {
-  auto temp = unsafe_arena_release_value();
+  
+  PROTOBUF_NAMESPACE_ID::Any* temp = value_;
+  value_ = nullptr;
   if (GetArena() != nullptr) {
   if (GetArena() != nullptr) {
     temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp);
     temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp);
   }
   }

+ 1 - 1
src/google/protobuf/unittest_lite.proto

@@ -38,7 +38,7 @@ package protobuf_unittest;
 
 
 import "google/protobuf/unittest_import_lite.proto";
 import "google/protobuf/unittest_import_lite.proto";
 
 
-option cc_enable_arenas = false;
+option cc_enable_arenas = true;
 option optimize_for = LITE_RUNTIME;
 option optimize_for = LITE_RUNTIME;
 option java_package = "com.google.protobuf";
 option java_package = "com.google.protobuf";