down integration from internal

Makefile.am

@@ -791,6 +791,7 @@ python_EXTRA_DIST=                                                           \
   python/google/protobuf/internal/descriptor_test.py                         \
   python/google/protobuf/internal/encoder.py                                 \
   python/google/protobuf/internal/enum_type_wrapper.py                       \
+  python/google/protobuf/internal/extension_dict.py                          \
   python/google/protobuf/internal/factory_test1.proto                        \
   python/google/protobuf/internal/factory_test2.proto                        \
   python/google/protobuf/internal/file_options_test.proto                    \

cmake/libprotobuf-lite.cmake

@@ -1,4 +1,5 @@
 set(libprotobuf_lite_files
+  ${protobuf_source_dir}/src/google/protobuf/any_lite.cc
   ${protobuf_source_dir}/src/google/protobuf/arena.cc
   ${protobuf_source_dir}/src/google/protobuf/extension_set.cc
   ${protobuf_source_dir}/src/google/protobuf/generated_message_table_driven_lite.cc

conformance/ConformanceJava.java

@@ -249,13 +249,30 @@ class ConformanceJava {
         break;
       }
       case TEXT_PAYLOAD: {
-        try {
-          TestMessagesProto3.TestAllTypesProto3.Builder builder =
-              TestMessagesProto3.TestAllTypesProto3.newBuilder();
-          TextFormat.merge(request.getTextPayload(), builder);
-          testMessage = builder.build();
-        } catch (TextFormat.ParseException e) {
-          return Conformance.ConformanceResponse.newBuilder().setParseError(e.getMessage()).build();
+        if (isProto3) {
+          try {
+            TestMessagesProto3.TestAllTypesProto3.Builder builder =
+                TestMessagesProto3.TestAllTypesProto3.newBuilder();
+            TextFormat.merge(request.getTextPayload(), builder);
+            testMessage = builder.build();
+          } catch (TextFormat.ParseException e) {
+              return Conformance.ConformanceResponse.newBuilder()
+                  .setParseError(e.getMessage())
+                  .build();
+          }
+        } else if (isProto2) {
+          try {
+            TestMessagesProto2.TestAllTypesProto2.Builder builder =
+                TestMessagesProto2.TestAllTypesProto2.newBuilder();
+            TextFormat.merge(request.getTextPayload(), builder);
+            testMessage = builder.build();
+          } catch (TextFormat.ParseException e) {
+              return Conformance.ConformanceResponse.newBuilder()
+                  .setParseError(e.getMessage())
+                  .build();
+          }
+        } else {
+          throw new RuntimeException("Protobuf request doesn't have specific payload type.");
         }
         break;
       }

conformance/Makefile.am

@@ -207,9 +207,11 @@ EXTRA_DIST =                  \
 
 conformance_test_runner_LDADD = $(top_srcdir)/src/libprotobuf.la
 conformance_test_runner_SOURCES = conformance_test.h conformance_test.cc \
-                                  binary_json_conformance_main.cc        \
+                                  conformance_test_main.cc               \
                                   binary_json_conformance_suite.h        \
                                   binary_json_conformance_suite.cc       \
+                                  text_format_conformance_suite.h        \
+                                  text_format_conformance_suite.cc       \
                                   conformance_test_runner.cc             \
                                   third_party/jsoncpp/json.h             \
                                   third_party/jsoncpp/jsoncpp.cpp

conformance/binary_json_conformance_suite.cc

@@ -681,6 +681,20 @@ void BinaryAndJsonConformanceSuite::TestUnknownMessage(
 }
 
 void BinaryAndJsonConformanceSuite::RunSuiteImpl() {
+  // Hack to get the list of test failures based on whether
+  // GOOGLE3_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER is enabled or not.
+  conformance::FailureSet failure_set;
+  ConformanceRequest req;
+  ConformanceResponse res;
+  req.set_message_type(failure_set.GetTypeName());
+  req.set_protobuf_payload("");
+  req.set_requested_output_format(conformance::WireFormat::PROTOBUF);
+  RunTest("FindFailures", req, &res);
+  GOOGLE_CHECK(failure_set.MergeFromString(res.protobuf_payload()));
+  for (const string& failure : failure_set.failure()) {
+    AddExpectedFailedTest(failure);
+  }
+
   type_resolver_.reset(NewTypeResolverForDescriptorPool(
       kTypeUrlPrefix, DescriptorPool::generated_pool()));
   type_url_ = GetTypeUrl(TestAllTypesProto3::descriptor());

conformance/conformance_python.py

@@ -65,7 +65,12 @@ def do_test(request):
     # TODO(gerbens): Remove, this is a hack to detect if the old vs new
     # parser is used by the cpp code. Relying on a bug in the old parser.
     hack_proto = test_messages_proto2_pb2.TestAllTypesProto2()
-    if hack_proto.ParseFromString(b"\322\002\001"):
+    old_parser = True
+    try:
+      hack_proto.ParseFromString(b"\322\002\001")
+    except message.DecodeError as e:
+      old_parser = False
+    if old_parser:
       # the string above is one of the failing conformance test strings of the
       # old parser. If we succeed the c++ implementation is using the old
       # parser so we add the list of failing conformance tests.

conformance/conformance_test.cc

@@ -361,6 +361,10 @@ string ConformanceTestSuite::WireFormatToString(
   return "";
 }
 
+void ConformanceTestSuite::AddExpectedFailedTest(const std::string& test_name) {
+  expected_to_fail_.insert(test_name);
+}
+
 bool ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner,
                                     std::string* output, const string& filename,
                                     conformance::FailureSet* failure_list) {
@@ -374,17 +378,10 @@ bool ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner,
 
   output_ = "\nCONFORMANCE TEST BEGIN ====================================\n\n";
 
-  ConformanceRequest req;
-  ConformanceResponse res;
-  req.set_message_type(failure_list->GetTypeName());
-  req.set_protobuf_payload("");
-  req.set_requested_output_format(conformance::WireFormat::PROTOBUF);
-  RunTest("FindFailures", req, &res);
-  GOOGLE_CHECK(failure_list->MergeFromString(res.protobuf_payload()));
   failure_list_filename_ = filename;
   expected_to_fail_.clear();
   for (const string& failure : failure_list->failure()) {
-    expected_to_fail_.insert(failure);
+    AddExpectedFailedTest(failure);
   }
   RunSuiteImpl();
 

conformance/conformance_test.h

@@ -84,8 +84,9 @@ class ConformanceTestRunner {
 // over a pipe.
 class ForkPipeRunner : public ConformanceTestRunner {
  public:
+  // Note: Run() doesn't take ownership of the pointers inside suites.
   static int Run(int argc, char *argv[],
-                 ConformanceTestSuite* suite);
+                 const std::vector<ConformanceTestSuite*>& suites);
 
   ForkPipeRunner(const std::string &executable)
       : child_pid_(-1), executable_(executable) {}
@@ -139,7 +140,10 @@ class ForkPipeRunner : public ConformanceTestRunner {
 //
 class ConformanceTestSuite {
  public:
-  ConformanceTestSuite() : verbose_(false), enforce_recommended_(false) {}
+  ConformanceTestSuite()
+      : verbose_(false),
+        enforce_recommended_(false),
+        failure_list_flag_name_("--failure_list") {}
   virtual ~ConformanceTestSuite() {}
 
   void SetVerbose(bool verbose) { verbose_ = verbose; }
@@ -156,6 +160,16 @@ class ConformanceTestSuite {
     enforce_recommended_ = value;
   }
 
+  // Gets the flag name to the failure list file.
+  // By default, this would return --failure_list
+  string GetFailureListFlagName() {
+    return failure_list_flag_name_;
+  }
+
+  void SetFailureListFlagName(const std::string& failure_list_flag_name) {
+    failure_list_flag_name_ = failure_list_flag_name;
+  }
+
   // Run all the conformance tests against the given test runner.
   // Test output will be stored in "output".
   //
@@ -259,6 +273,8 @@ class ConformanceTestSuite {
                const conformance::ConformanceRequest& request,
                conformance::ConformanceResponse* response);
 
+  void AddExpectedFailedTest(const std::string& test_name);
+
   virtual void RunSuiteImpl() = 0;
 
   ConformanceTestRunner* runner_;
@@ -267,6 +283,7 @@ class ConformanceTestSuite {
   bool verbose_;
   bool enforce_recommended_;
   std::string output_;
+  std::string failure_list_flag_name_;
   std::string failure_list_filename_;
 
   // The set of test names that are expected to fail in this run, but haven't

conformance/conformance_test_main.cc

@@ -0,0 +1,40 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "binary_json_conformance_suite.h"
+#include "conformance_test.h"
+#include "text_format_conformance_suite.h"
+
+int main(int argc, char *argv[]) {
+  google::protobuf::BinaryAndJsonConformanceSuite binary_and_json_suite;
+  google::protobuf::TextFormatConformanceTestSuite text_format_suite;
+  return google::protobuf::ForkPipeRunner::Run(
+      argc, argv, {&binary_and_json_suite, &text_format_suite});
+}

conformance/conformance_test_runner.cc

@@ -119,6 +119,19 @@ void UsageError() {
           "                              should contain one test name per\n");
   fprintf(stderr,
           "                              line.  Use '#' for comments.\n");
+  fprintf(stderr,
+          "  --text_format_failure_list <filename>   Use to specify list \n");
+  fprintf(stderr,
+          "                              of tests that are expected to \n");
+  fprintf(stderr,
+          "                              fail in the \n");
+  fprintf(stderr,
+          "                              text_format_conformance_suite.  \n");
+  fprintf(stderr,
+          "                              File should contain one test name \n");
+  fprintf(stderr,
+          "                              per line.  Use '#' for comments.\n");
+
   fprintf(stderr,
           "  --enforce_recommended       Enforce that recommended test\n");
   fprintf(stderr,
@@ -175,41 +188,56 @@ void ForkPipeRunner::RunTest(
 }
 
 int ForkPipeRunner::Run(
-    int argc, char *argv[], ConformanceTestSuite* suite) {
-  char *program;
-  string failure_list_filename;
-  conformance::FailureSet failure_list;
-
-  for (int arg = 1; arg < argc; ++arg) {
-    if (strcmp(argv[arg], "--failure_list") == 0) {
-      if (++arg == argc) UsageError();
-      failure_list_filename = argv[arg];
-      ParseFailureList(argv[arg], &failure_list);
-    } else if (strcmp(argv[arg], "--verbose") == 0) {
-      suite->SetVerbose(true);
-    } else if (strcmp(argv[arg], "--enforce_recommended") == 0) {
-      suite->SetEnforceRecommended(true);
-    } else if (argv[arg][0] == '-') {
-      fprintf(stderr, "Unknown option: %s\n", argv[arg]);
-      UsageError();
-    } else {
-      if (arg != argc - 1) {
-        fprintf(stderr, "Too many arguments.\n");
-        UsageError();
+    int argc, char *argv[], const std::vector<ConformanceTestSuite*>& suites) {
+  if (suites.empty()) {
+    fprintf(stderr, "No test suites found.\n");
+    return EXIT_FAILURE;
+  }
+  bool all_ok = true;
+  for (ConformanceTestSuite* suite : suites) {
+    char *program;
+    string failure_list_filename;
+    conformance::FailureSet failure_list;
+
+    for (int arg = 1; arg < argc; ++arg) {
+      if (strcmp(argv[arg], suite->GetFailureListFlagName().c_str()) == 0) {
+        if (++arg == argc) UsageError();
+        failure_list_filename = argv[arg];
+        ParseFailureList(argv[arg], &failure_list);
+      } else if (strcmp(argv[arg], "--verbose") == 0) {
+        suite->SetVerbose(true);
+      } else if (strcmp(argv[arg], "--enforce_recommended") == 0) {
+        suite->SetEnforceRecommended(true);
+      } else if (argv[arg][0] == '-') {
+        bool recognized_flag = false;
+        for (ConformanceTestSuite* suite : suites) {
+          if (strcmp(argv[arg], suite->GetFailureListFlagName().c_str()) == 0) {
+            if (++arg == argc) UsageError();
+            recognized_flag = true;
+          }
+        }
+        if (!recognized_flag) {
+          fprintf(stderr, "Unknown option: %s\n", argv[arg]);
+          UsageError();
+        }
+      } else {
+        if (arg != argc - 1) {
+          fprintf(stderr, "Too many arguments.\n");
+          UsageError();
+        }
+        program = argv[arg];
       }
-      program = argv[arg];
     }
-  }
 
-  ForkPipeRunner runner(program);
+    ForkPipeRunner runner(program);
 
-  std::string output;
-  bool ok =
-      suite->RunSuite(&runner, &output, failure_list_filename, &failure_list);
+    std::string output;
+    all_ok = all_ok &&
+        suite->RunSuite(&runner, &output, failure_list_filename, &failure_list);
 
-  fwrite(output.c_str(), 1, output.size(), stderr);
-
-  return ok ? EXIT_SUCCESS : EXIT_FAILURE;
+    fwrite(output.c_str(), 1, output.size(), stderr);
+  }
+  return all_ok ? EXIT_SUCCESS : EXIT_FAILURE;
 }
 
 // TODO(haberman): make this work on Windows, instead of using these

conformance/failure_list_python.txt

@@ -19,4 +19,3 @@ Required.Proto3.ProtobufInput.IllegalZeroFieldNum_Case_0
 Required.Proto3.ProtobufInput.IllegalZeroFieldNum_Case_1
 Required.Proto3.ProtobufInput.IllegalZeroFieldNum_Case_2
 Required.Proto3.ProtobufInput.IllegalZeroFieldNum_Case_3
-Required.Proto3.JsonInput.EmptyFieldMask.ProtobufOutput

conformance/failure_list_python_cpp.txt

@@ -20,4 +20,3 @@ Required.Proto3.JsonInput.FloatFieldTooLarge
 Required.Proto3.JsonInput.FloatFieldTooSmall
 Required.Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingIntegersGotBool
 Required.Proto3.JsonInput.TimestampJsonInputLowercaseT
-Required.Proto3.JsonInput.EmptyFieldMask.ProtobufOutput

conformance/text_format_conformance_suite.cc

@@ -0,0 +1,240 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "text_format_conformance_suite.h"
+
+#include "conformance_test.h"
+
+#include <google/protobuf/any.pb.h>
+#include <google/protobuf/test_messages_proto2.pb.h>
+#include <google/protobuf/test_messages_proto3.pb.h>
+#include <google/protobuf/text_format.h>
+
+using conformance::ConformanceRequest;
+using conformance::ConformanceResponse;
+using conformance::WireFormat;
+using google::protobuf::Message;
+using google::protobuf::TextFormat;
+using protobuf_test_messages::proto2::TestAllTypesProto2;
+using protobuf_test_messages::proto3::TestAllTypesProto3;
+using std::string;
+
+namespace google {
+namespace protobuf {
+
+TextFormatConformanceTestSuite::TextFormatConformanceTestSuite() {
+  SetFailureListFlagName("--text_format_failure_list");
+}
+
+bool TextFormatConformanceTestSuite::ParseTextFormatResponse(
+    const ConformanceResponse& response, Message* test_message) {
+  if (!TextFormat::ParseFromString(response.text_payload(), test_message)) {
+    GOOGLE_LOG(ERROR) << "INTERNAL ERROR: internal text->protobuf transcode "
+                      << "yielded unparseable proto. Text payload: "
+                      << response.text_payload();
+    return false;
+  }
+
+  return true;
+}
+
+bool TextFormatConformanceTestSuite::ParseResponse(
+    const ConformanceResponse& response,
+    const ConformanceRequestSetting& setting, Message* test_message) {
+  const ConformanceRequest& request = setting.GetRequest();
+  WireFormat requested_output = request.requested_output_format();
+  const string& test_name = setting.GetTestName();
+  ConformanceLevel level = setting.GetLevel();
+
+  switch (response.result_case()) {
+    case ConformanceResponse::kProtobufPayload: {
+      if (requested_output != conformance::PROTOBUF) {
+        ReportFailure(
+            test_name, level, request, response,
+            StrCat("Test was asked for ", WireFormatToString(requested_output),
+                   " output but provided PROTOBUF instead.")
+                .c_str());
+        return false;
+      }
+
+      if (!test_message->ParseFromString(response.protobuf_payload())) {
+        ReportFailure(test_name, level, request, response,
+                      "Protobuf output we received from test was unparseable.");
+        return false;
+      }
+
+      break;
+    }
+
+    case ConformanceResponse::kTextPayload: {
+      if (requested_output != conformance::TEXT_FORMAT) {
+        ReportFailure(
+            test_name, level, request, response,
+            StrCat("Test was asked for ", WireFormatToString(requested_output),
+                   " output but provided TEXT_FORMAT instead.")
+                .c_str());
+        return false;
+      }
+
+      if (!ParseTextFormatResponse(response, test_message)) {
+        ReportFailure(
+            test_name, level, request, response,
+            "TEXT_FORMAT output we received from test was unparseable.");
+        return false;
+      }
+
+      break;
+    }
+
+    default:
+      GOOGLE_LOG(FATAL) << test_name
+                        << ": unknown payload type: " << response.result_case();
+  }
+
+  return true;
+}
+
+void TextFormatConformanceTestSuite::ExpectParseFailure(const string& test_name,
+                                                        ConformanceLevel level,
+                                                        const string& input) {
+  TestAllTypesProto3 prototype;
+  // We don't expect output, but if the program erroneously accepts the protobuf
+  // we let it send its response as this.  We must not leave it unspecified.
+  ConformanceRequestSetting setting(
+      level, conformance::TEXT_FORMAT, conformance::TEXT_FORMAT,
+      conformance::TEXT_FORMAT_TEST, prototype, test_name, input);
+  const ConformanceRequest& request = setting.GetRequest();
+  ConformanceResponse response;
+  string effective_test_name = StrCat(setting.ConformanceLevelToString(level),
+                                      ".Proto3.TextFormatInput.", test_name);
+
+  RunTest(effective_test_name, request, &response);
+  if (response.result_case() == ConformanceResponse::kParseError) {
+    ReportSuccess(effective_test_name);
+  } else if (response.result_case() == ConformanceResponse::kSkipped) {
+    ReportSkip(effective_test_name, request, response);
+  } else {
+    ReportFailure(effective_test_name, level, request, response,
+                  "Should have failed to parse, but didn't.");
+  }
+}
+
+void TextFormatConformanceTestSuite::RunValidTextFormatTest(
+    const string& test_name, ConformanceLevel level, const string& input_text) {
+  TestAllTypesProto3 prototype;
+  RunValidTextFormatTestWithMessage(test_name, level, input_text, prototype);
+}
+
+void TextFormatConformanceTestSuite::RunValidTextFormatTestProto2(
+    const string& test_name, ConformanceLevel level, const string& input_text) {
+  TestAllTypesProto2 prototype;
+  RunValidTextFormatTestWithMessage(test_name, level, input_text, prototype);
+}
+
+void TextFormatConformanceTestSuite::RunValidTextFormatTestWithMessage(
+    const string& test_name, ConformanceLevel level, const string& input_text,
+    const Message& prototype) {
+  ConformanceRequestSetting setting1(
+      level, conformance::TEXT_FORMAT, conformance::PROTOBUF,
+      conformance::TEXT_FORMAT_TEST, prototype, test_name, input_text);
+  RunValidInputTest(setting1, input_text);
+  ConformanceRequestSetting setting2(
+      level, conformance::TEXT_FORMAT, conformance::TEXT_FORMAT,
+      conformance::TEXT_FORMAT_TEST, prototype, test_name, input_text);
+  RunValidInputTest(setting2, input_text);
+}
+
+void TextFormatConformanceTestSuite::RunSuiteImpl() {
+  RunValidTextFormatTest("HelloWorld", REQUIRED,
+                         "optional_string: 'Hello, World!'");
+  // Integer fields.
+  RunValidTextFormatTest("Int32FieldMaxValue", REQUIRED,
+                         "optional_int32: 2147483647");
+  RunValidTextFormatTest("Int32FieldMinValue", REQUIRED,
+                         "optional_int32: -2147483648");
+  RunValidTextFormatTest("Uint32FieldMaxValue", REQUIRED,
+                         "optional_uint32: 4294967295");
+  RunValidTextFormatTest("Int64FieldMaxValue", REQUIRED,
+                         "optional_int64: 9223372036854775807");
+  RunValidTextFormatTest("Int64FieldMinValue", REQUIRED,
+                         "optional_int64: -9223372036854775808");
+  RunValidTextFormatTest("Uint64FieldMaxValue", REQUIRED,
+                         "optional_uint64: 18446744073709551615");
+
+  // Parsers reject out-of-bound integer values.
+  ExpectParseFailure("Int32FieldTooLarge", REQUIRED,
+                     "optional_int32: 2147483648");
+  ExpectParseFailure("Int32FieldTooSmall", REQUIRED,
+                     "optional_int32: -2147483649");
+  ExpectParseFailure("Uint32FieldTooLarge", REQUIRED,
+                     "optional_uint32: 4294967296");
+  ExpectParseFailure("Int64FieldTooLarge", REQUIRED,
+                     "optional_int64: 9223372036854775808");
+  ExpectParseFailure("Int64FieldTooSmall", REQUIRED,
+                     "optional_int64: -9223372036854775809");
+  ExpectParseFailure("Uint64FieldTooLarge", REQUIRED,
+                     "optional_uint64: 18446744073709551616");
+
+  // Floating point fields
+  RunValidTextFormatTest("FloatField", REQUIRED,
+                         "optional_float: 3.192837");
+  RunValidTextFormatTest("FloatFieldWithVeryPreciseNumber", REQUIRED,
+                         "optional_float: 3.123456789123456789");
+  RunValidTextFormatTest("FloatFieldMaxValue", REQUIRED,
+                         "optional_float: 3.40282e+38");
+  RunValidTextFormatTest("FloatFieldMinValue", REQUIRED,
+                         "optional_float: 1.17549e-38");
+  RunValidTextFormatTest("FloatFieldNaNValue", REQUIRED,
+                         "optional_float: NaN");
+  RunValidTextFormatTest("FloatFieldPosInfValue", REQUIRED,
+                         "optional_float: inf");
+  RunValidTextFormatTest("FloatFieldNegInfValue", REQUIRED,
+                         "optional_float: -inf");
+  RunValidTextFormatTest("FloatFieldWithInt32Max", REQUIRED,
+                         "optional_float: 4294967296");
+  RunValidTextFormatTest("FloatFieldLargerThanInt64", REQUIRED,
+                         "optional_float: 9223372036854775808");
+
+  ExpectParseFailure("FloatFieldTooLarge", REQUIRED,
+                     "optional_int32: 3.4028235e+39");
+  ExpectParseFailure("FloatFieldTooSmall", REQUIRED,
+                     "optional_int32: 1.17549e-39");
+
+  // Group fields
+  RunValidTextFormatTestProto2("GroupFieldNoColon", REQUIRED,
+                               "Data { group_int32: 1 }");
+  RunValidTextFormatTestProto2("GroupFieldWithColon", REQUIRED,
+                               "Data: { group_int32: 1 }");
+  RunValidTextFormatTestProto2("GroupFieldEmpty", REQUIRED,
+                               "Data {}");
+}
+
+}  // namespace protobuf
+}  // namespace google

conformance/text_format_conformance_suite.h

@@ -0,0 +1,66 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef TEXT_FORMAT_CONFORMANCE_SUITE_H_
+#define TEXT_FORMAT_CONFORMANCE_SUITE_H_
+
+#include "conformance_test.h"
+
+namespace google {
+namespace protobuf {
+
+class TextFormatConformanceTestSuite : public ConformanceTestSuite {
+ public:
+  TextFormatConformanceTestSuite();
+
+ private:
+  void RunSuiteImpl();
+  void RunValidTextFormatTest(const string& test_name, ConformanceLevel level,
+                              const string& input);
+  void RunValidTextFormatTestProto2(const string& test_name,
+                                    ConformanceLevel level,
+                                    const string& input);
+  void RunValidTextFormatTestWithMessage(const string& test_name,
+                                         ConformanceLevel level,
+                                         const string& input_text,
+                                         const Message& prototype);
+  void ExpectParseFailure(const string& test_name, ConformanceLevel level,
+                          const string& input);
+  bool ParseTextFormatResponse(const conformance::ConformanceResponse& response,
+                               Message* test_message);
+  bool ParseResponse(const conformance::ConformanceResponse& response,
+                     const ConformanceRequestSetting& setting,
+                     Message* test_message) override;
+};
+
+}  // namespace protobuf
+}  // namespace google
+
+#endif  // TEXT_FORMAT_CONFORMANCE_SUITE_H_

java/core/src/main/java/com/google/protobuf/CodedInputStream.java

@@ -3006,25 +3006,11 @@ public abstract class CodedInputStream {
         throw InvalidProtocolBufferException.truncatedMessage();
       }
 
-      if (refillCallback != null) {
-        // Skipping more bytes than are in the buffer.  First skip what we have.
-        int tempPos = bufferSize - pos;
-        pos = bufferSize;
-
-        // Keep refilling the buffer until we get to the point we wanted to skip to.
-        // This has the side effect of ensuring the limits are updated correctly.
-        refillBuffer(1);
-        while (size - tempPos > bufferSize) {
-          tempPos += bufferSize;
-          pos = bufferSize;
-          refillBuffer(1);
-        }
-
-        pos = size - tempPos;
-      } else {
+      int totalSkipped = 0;
+      if (refillCallback == null) {
         // Skipping more bytes than are in the buffer.  First skip what we have.
         totalBytesRetired += pos;
-        int totalSkipped = bufferSize - pos;
+        totalSkipped = bufferSize - pos;
         bufferSize = 0;
         pos = 0;
 
@@ -3038,6 +3024,12 @@ public abstract class CodedInputStream {
                       + "#skip returned invalid result: "
                       + skipped
                       + "\nThe InputStream implementation is buggy.");
+            } else if (skipped == 0) {
+              // The API contract of skip() permits an inputstream to skip zero bytes for any reason
+              // it wants. In particular, ByteArrayInputStream will just return zero over and over
+              // when it's at the end of its input. In order to actually confirm that we've hit the
+              // end of input, we need to issue a read call via the other path.
+              break;
             }
             totalSkipped += (int) skipped;
           }
@@ -3046,6 +3038,22 @@ public abstract class CodedInputStream {
           recomputeBufferSizeAfterLimit();
         }
       }
+      if (totalSkipped < size) {
+        // Skipping more bytes than are in the buffer.  First skip what we have.
+        int tempPos = bufferSize - pos;
+        pos = bufferSize;
+
+        // Keep refilling the buffer until we get to the point we wanted to skip to.
+        // This has the side effect of ensuring the limits are updated correctly.
+        refillBuffer(1);
+        while (size - tempPos > bufferSize) {
+          tempPos += bufferSize;
+          pos = bufferSize;
+          refillBuffer(1);
+        }
+
+        pos = size - tempPos;
+      }
     }
   }
 

java/core/src/main/java/com/google/protobuf/Descriptors.java

@@ -185,8 +185,9 @@ public final class Descriptors {
       if (name.indexOf('.') != -1) {
         return null;
       }
-      if (getPackage().length() > 0) {
-        name = getPackage() + '.' + name;
+      final String packageName = getPackage();
+      if (!packageName.isEmpty()) {
+        name = packageName + '.' + name;
       }
       final GenericDescriptor result = pool.findSymbol(name);
       if (result != null && result instanceof Descriptor && result.getFile() == this) {
@@ -208,8 +209,9 @@ public final class Descriptors {
       if (name.indexOf('.') != -1) {
         return null;
       }
-      if (getPackage().length() > 0) {
-        name = getPackage() + '.' + name;
+      final String packageName = getPackage();
+      if (!packageName.isEmpty()) {
+        name = packageName + '.' + name;
       }
       final GenericDescriptor result = pool.findSymbol(name);
       if (result != null && result instanceof EnumDescriptor && result.getFile() == this) {
@@ -231,8 +233,9 @@ public final class Descriptors {
       if (name.indexOf('.') != -1) {
         return null;
       }
-      if (getPackage().length() > 0) {
-        name = getPackage() + '.' + name;
+      final String packageName = getPackage();
+      if (!packageName.isEmpty()) {
+        name = packageName + '.' + name;
       }
       final GenericDescriptor result = pool.findSymbol(name);
       if (result != null && result instanceof ServiceDescriptor && result.getFile() == this) {
@@ -252,8 +255,9 @@ public final class Descriptors {
       if (name.indexOf('.') != -1) {
         return null;
       }
-      if (getPackage().length() > 0) {
-        name = getPackage() + '.' + name;
+      final String packageName = getPackage();
+      if (!packageName.isEmpty()) {
+        name = packageName + '.' + name;
       }
       final GenericDescriptor result = pool.findSymbol(name);
       if (result != null && result instanceof FieldDescriptor && result.getFile() == this) {
@@ -1223,14 +1227,20 @@ public final class Descriptors {
     // This method should match exactly with the ToJsonName() function in C++
     // descriptor.cc.
     private static String fieldNameToJsonName(String name) {
-      StringBuilder result = new StringBuilder(name.length());
+      final int length = name.length();
+      StringBuilder result = new StringBuilder(length);
       boolean isNextUpperCase = false;
-      for (int i = 0; i < name.length(); i++) {
+      for (int i = 0; i < length; i++) {
         char ch = name.charAt(i);
         if (ch == '_') {
           isNextUpperCase = true;
         } else if (isNextUpperCase) {
-          result.append(Character.toUpperCase(ch));
+          // This closely matches the logic for ASCII characters in:
+          // http://google3/google/protobuf/descriptor.cc?l=249-251&rcl=228891689
+          if ('a' <= ch && ch <= 'z') {
+            ch = (char) (ch - 'a' + 'A');
+          }
+          result.append(ch);
           isNextUpperCase = false;
         } else {
           result.append(ch);
@@ -1787,7 +1797,6 @@ public final class Descriptors {
       file.pool.addEnumValueByNumber(this);
     }
 
-    private Integer number;
     // Create an unknown enum value.
     private EnumValueDescriptor(
         final FileDescriptor file, final EnumDescriptor parent, final Integer number) {
@@ -1799,7 +1808,6 @@ public final class Descriptors {
       this.file = file;
       this.type = parent;
       this.fullName = parent.getFullName() + '.' + proto.getName();
-      this.number = number;
 
       // Don't add this descriptor into pool.
     }
@@ -2029,11 +2037,14 @@ public final class Descriptors {
       final FileDescriptor file, final Descriptor parent, final String name) {
     if (parent != null) {
       return parent.getFullName() + '.' + name;
-    } else if (file.getPackage().length() > 0) {
-      return file.getPackage() + '.' + name;
-    } else {
-      return name;
     }
+
+    final String packageName = file.getPackage();
+    if (!packageName.isEmpty()) {
+      return packageName + '.' + name;
+    }
+
+    return name;
   }
 
   // =================================================================
@@ -2322,13 +2333,13 @@ public final class Descriptors {
       validateSymbolName(descriptor);
 
       final String fullName = descriptor.getFullName();
-      final int dotpos = fullName.lastIndexOf('.');
 
       final GenericDescriptor old = descriptorsByName.put(fullName, descriptor);
       if (old != null) {
         descriptorsByName.put(fullName, old);
 
         if (descriptor.getFile() == old.getFile()) {
+          final int dotpos = fullName.lastIndexOf('.');
           if (dotpos == -1) {
             throw new DescriptorValidationException(
                 descriptor, '\"' + fullName + "\" is already defined.");
@@ -2494,27 +2505,22 @@ public final class Descriptors {
       final String name = descriptor.getName();
       if (name.length() == 0) {
         throw new DescriptorValidationException(descriptor, "Missing name.");
-      } else {
-        boolean valid = true;
-        for (int i = 0; i < name.length(); i++) {
-          final char c = name.charAt(i);
-          // Non-ASCII characters are not valid in protobuf identifiers, even
-          // if they are letters or digits.
-          if (c >= 128) {
-            valid = false;
-          }
-          // First character must be letter or _.  Subsequent characters may
-          // be letters, numbers, or digits.
-          if (Character.isLetter(c) || c == '_' || (Character.isDigit(c) && i > 0)) {
-            // Valid
-          } else {
-            valid = false;
-          }
-        }
-        if (!valid) {
-          throw new DescriptorValidationException(
-              descriptor, '\"' + name + "\" is not a valid identifier.");
+      }
+
+      // Non-ASCII characters are not valid in protobuf identifiers, even
+      // if they are letters or digits.
+      // The first character must be a letter or '_'.
+      // Subsequent characters may be letters, numbers, or digits.
+      for (int i = 0; i < name.length(); i++) {
+        final char c = name.charAt(i);
+        if (('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z')
+          || (c == '_')
+          || ('0' <= c && c <= '9' && i > 0)) {
+          // Valid
+          continue;
         }
+        throw new DescriptorValidationException(
+            descriptor, '\"' + name + "\" is not a valid identifier.");
       }
     }
   }

java/core/src/test/java/com/google/protobuf/CodedInputStreamTest.java

@@ -1186,4 +1186,13 @@ public class CodedInputStreamTest extends TestCase {
       }
     }
   }
+
+  public void testSkipPastEndOfByteArrayInput() throws Exception {
+    try {
+      CodedInputStream.newInstance(new ByteArrayInputStream(new byte[100])).skipRawBytes(101);
+      fail();
+    } catch (InvalidProtocolBufferException e) {
+      // Expected
+    }
+  }
 }

java/core/src/test/java/com/google/protobuf/ParseExceptionsTest.java

@@ -216,6 +216,23 @@ public class ParseExceptionsTest {
         });
   }
 
+  @Test
+  public void messageBuilder_mergeDelimitedFrom_InputStream_malformed() throws Exception {
+    byte[] body = new byte[80];
+    CodedOutputStream cos = CodedOutputStream.newInstance(body);
+    cos.writeRawVarint32(90); // Greater than bytes in stream
+    cos.writeTag(DescriptorProto.ENUM_TYPE_FIELD_NUMBER, WireFormat.WIRETYPE_LENGTH_DELIMITED);
+    cos.writeRawVarint32(98); // Nested message with size larger than parent
+    cos.writeTag(1000, WireFormat.WIRETYPE_LENGTH_DELIMITED);
+    cos.writeRawVarint32(100); // Unknown field with size larger than parent
+    ByteArrayInputStream bais = new ByteArrayInputStream(body);
+    try {
+      DescriptorProto.parseDelimitedFrom(bais);
+      fail();
+    } catch (InvalidProtocolBufferException expected) {
+    }
+  }
+
   @Test
   public void messageBuilder_mergeDelimitedFrom_InputStreamAndExtensionRegistry() {
     setupDelimited();

js/binary/writer.js

@@ -799,6 +799,38 @@ jspb.BinaryWriter.prototype.writeMessage = function(
 };
 
 
+/**
+ * Writes a message set extension to the buffer.
+ * @param {number} field The field number for the extension.
+ * @param {?MessageType} value The extension message object to write. Note that
+ *     message set can only have extensions with type of optional message.
+ * @param {function(!MessageTypeNonNull, !jspb.BinaryWriter)} writerCallback
+ *     Will be invoked with the value to write and the writer to write it with.
+ * @template MessageType
+ * Use go/closure-ttl to declare a non-nullable version of MessageType.  Replace
+ * the null in blah|null with none.  This is necessary because the compiler will
+ * infer MessageType to be nullable if the value parameter is nullable.
+ * @template MessageTypeNonNull :=
+ *     cond(isUnknown(MessageType), unknown(),
+ *       mapunion(MessageType, (X) =>
+ *         cond(eq(X, 'null'), none(), X)))
+ * =:
+ */
+jspb.BinaryWriter.prototype.writeMessageSet = function(
+    field, value, writerCallback) {
+  if (value == null) return;
+  // The wire format for a message set is defined by
+  // google3/net/proto/message_set.proto
+  this.writeFieldHeader_(1, jspb.BinaryConstants.WireType.START_GROUP);
+  this.writeFieldHeader_(2, jspb.BinaryConstants.WireType.VARINT);
+  this.encoder_.writeSignedVarint32(field);
+  var bookmark = this.beginDelimited_(3);
+  writerCallback(value, this);
+  this.endDelimited_(bookmark);
+  this.writeFieldHeader_(1, jspb.BinaryConstants.WireType.END_GROUP);
+};
+
+
 /**
  * Writes a group message to the buffer.
  *

js/message.js

@@ -183,9 +183,6 @@ goog.define('jspb.Message.GENERATE_TO_OBJECT', true);
  *     calling fromObject. Enabling this might disable the JSCompiler's ability
  *     to dead code eliminate fields used in protocol buffers that are never
  *     used in an application.
- *     NOTE: By default no protos actually have a fromObject method. You need to
- *     add the jspb.generate_from_object options to the proto definition to
- *     activate the feature.
  *     By default this is enabled for test code only.
  */
 goog.define('jspb.Message.GENERATE_FROM_OBJECT', !goog.DISALLOW_TEST_ONLY_CODE);
@@ -703,20 +700,7 @@ jspb.Message.getField = function(msg, fieldNumber) {
  * @protected
  */
 jspb.Message.getRepeatedField = function(msg, fieldNumber) {
-  if (fieldNumber < msg.pivot_) {
-    var index = jspb.Message.getIndex_(msg, fieldNumber);
-    var val = msg.array[index];
-    if (val === jspb.Message.EMPTY_LIST_SENTINEL_) {
-      return msg.array[index] = [];
-    }
-    return val;
-  }
-
-  var val = msg.extensionObject_[fieldNumber];
-  if (val === jspb.Message.EMPTY_LIST_SENTINEL_) {
-    return msg.extensionObject_[fieldNumber] = [];
-  }
-  return val;
+  return /** @type {!Array} */ (jspb.Message.getField(msg, fieldNumber));
 };
 
 

js/test.proto

@@ -106,6 +106,13 @@ message OuterMessage {
   }
 }
 
+message MineField {
+  // document.cookie is a banned property in a couple of conformance check
+  // configs at Google. Verify that having a field called cookie doesn't confuse
+  // the compiler and break the build.
+  optional string cookie = 1;
+}
+
 message IsExtension {
   extend HasExtensions {
     optional IsExtension ext_field = 100;
@@ -261,7 +268,6 @@ message Int64Types {
 }
 
 message TestMapFieldsNoBinary {
-
   map<string, string> map_string_string = 1;
   map<string, int32> map_string_int32 = 2;
   map<string, int64> map_string_int64 = 3;
@@ -285,7 +291,6 @@ enum MapValueEnumNoBinary {
 }
 
 message MapValueMessageNoBinary {
-
   optional int32 foo = 1;
 }
 

python/google/protobuf/internal/containers.py

@@ -230,6 +230,8 @@ class BaseContainer(object):
       kwargs['cmp'] = kwargs.pop('sort_function')
     self._values.sort(*args, **kwargs)
 
+collections_abc.MutableSequence.register(BaseContainer)
+
 
 class RepeatedScalarFieldContainer(BaseContainer):
 
@@ -341,8 +343,6 @@ class RepeatedScalarFieldContainer(BaseContainer):
     # We are presumably comparing against some other sequence type.
     return other == self._values
 
-collections_abc.MutableSequence.register(BaseContainer)
-
 
 class RepeatedCompositeFieldContainer(BaseContainer):
 
@@ -380,6 +380,24 @@ class RepeatedCompositeFieldContainer(BaseContainer):
       self._message_listener.Modified()
     return new_element
 
+  def append(self, value):
+    """Appends one element by copying the message."""
+    new_element = self._message_descriptor._concrete_class()
+    new_element._SetListener(self._message_listener)
+    new_element.CopyFrom(value)
+    self._values.append(new_element)
+    if not self._message_listener.dirty:
+      self._message_listener.Modified()
+
+  def insert(self, key, value):
+    """Inserts the item at the specified position by copying."""
+    new_element = self._message_descriptor._concrete_class()
+    new_element._SetListener(self._message_listener)
+    new_element.CopyFrom(value)
+    self._values.insert(key, new_element)
+    if not self._message_listener.dirty:
+      self._message_listener.Modified()
+
   def extend(self, elem_seq):
     """Extends by appending the given sequence of elements of the same type
     as this one, copying each individual message.

python/google/protobuf/internal/extension_dict.py

@@ -0,0 +1,185 @@
+# Protocol Buffers - Google's data interchange format
+# Copyright 2008 Google Inc.  All rights reserved.
+# https://developers.google.com/protocol-buffers/
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Contains _ExtensionDict class to represent extensions.
+"""
+
+from google.protobuf.internal import type_checkers
+from google.protobuf.descriptor import FieldDescriptor
+
+
+def _VerifyExtensionHandle(message, extension_handle):
+  """Verify that the given extension handle is valid."""
+
+  if not isinstance(extension_handle, FieldDescriptor):
+    raise KeyError('HasExtension() expects an extension handle, got: %s' %
+                   extension_handle)
+
+  if not extension_handle.is_extension:
+    raise KeyError('"%s" is not an extension.' % extension_handle.full_name)
+
+  if not extension_handle.containing_type:
+    raise KeyError('"%s" is missing a containing_type.'
+                   % extension_handle.full_name)
+
+  if extension_handle.containing_type is not message.DESCRIPTOR:
+    raise KeyError('Extension "%s" extends message type "%s", but this '
+                   'message is of type "%s".' %
+                   (extension_handle.full_name,
+                    extension_handle.containing_type.full_name,
+                    message.DESCRIPTOR.full_name))
+
+
+# TODO(robinson): Unify error handling of "unknown extension" crap.
+# TODO(robinson): Support iteritems()-style iteration over all
+# extensions with the "has" bits turned on?
+class _ExtensionDict(object):
+
+  """Dict-like container for Extension fields on proto instances.
+
+  Note that in all cases we expect extension handles to be
+  FieldDescriptors.
+  """
+
+  def __init__(self, extended_message):
+    """
+    Args:
+      extended_message: Message instance for which we are the Extensions dict.
+    """
+    self._extended_message = extended_message
+
+  def __getitem__(self, extension_handle):
+    """Returns the current value of the given extension handle."""
+
+    _VerifyExtensionHandle(self._extended_message, extension_handle)
+
+    result = self._extended_message._fields.get(extension_handle)
+    if result is not None:
+      return result
+
+    if extension_handle.label == FieldDescriptor.LABEL_REPEATED:
+      result = extension_handle._default_constructor(self._extended_message)
+    elif extension_handle.cpp_type == FieldDescriptor.CPPTYPE_MESSAGE:
+      assert getattr(extension_handle.message_type, '_concrete_class', None), (
+          'Uninitialized concrete class found for field %r (message type %r)'
+          % (extension_handle.full_name,
+             extension_handle.message_type.full_name))
+      result = extension_handle.message_type._concrete_class()
+      try:
+        result._SetListener(self._extended_message._listener_for_children)
+      except ReferenceError:
+        pass
+    else:
+      # Singular scalar -- just return the default without inserting into the
+      # dict.
+      return extension_handle.default_value
+
+    # Atomically check if another thread has preempted us and, if not, swap
+    # in the new object we just created.  If someone has preempted us, we
+    # take that object and discard ours.
+    # WARNING:  We are relying on setdefault() being atomic.  This is true
+    #   in CPython but we haven't investigated others.  This warning appears
+    #   in several other locations in this file.
+    result = self._extended_message._fields.setdefault(
+        extension_handle, result)
+
+    return result
+
+  def __eq__(self, other):
+    if not isinstance(other, self.__class__):
+      return False
+
+    my_fields = self._extended_message.ListFields()
+    other_fields = other._extended_message.ListFields()
+
+    # Get rid of non-extension fields.
+    my_fields = [field for field in my_fields if field.is_extension]
+    other_fields = [field for field in other_fields if field.is_extension]
+
+    return my_fields == other_fields
+
+  def __ne__(self, other):
+    return not self == other
+
+  def __len__(self):
+    fields = self._extended_message.ListFields()
+    # Get rid of non-extension fields.
+    extension_fields = [field for field in fields if field[0].is_extension]
+    return len(extension_fields)
+
+  def __hash__(self):
+    raise TypeError('unhashable object')
+
+  # Note that this is only meaningful for non-repeated, scalar extension
+  # fields.  Note also that we may have to call _Modified() when we do
+  # successfully set a field this way, to set any necssary "has" bits in the
+  # ancestors of the extended message.
+  def __setitem__(self, extension_handle, value):
+    """If extension_handle specifies a non-repeated, scalar extension
+    field, sets the value of that field.
+    """
+
+    _VerifyExtensionHandle(self._extended_message, extension_handle)
+
+    if (extension_handle.label == FieldDescriptor.LABEL_REPEATED or
+        extension_handle.cpp_type == FieldDescriptor.CPPTYPE_MESSAGE):
+      raise TypeError(
+          'Cannot assign to extension "%s" because it is a repeated or '
+          'composite type.' % extension_handle.full_name)
+
+    # It's slightly wasteful to lookup the type checker each time,
+    # but we expect this to be a vanishingly uncommon case anyway.
+    type_checker = type_checkers.GetTypeChecker(extension_handle)
+    # pylint: disable=protected-access
+    self._extended_message._fields[extension_handle] = (
+        type_checker.CheckValue(value))
+    self._extended_message._Modified()
+
+  def _FindExtensionByName(self, name):
+    """Tries to find a known extension with the specified name.
+
+    Args:
+      name: Extension full name.
+
+    Returns:
+      Extension field descriptor.
+    """
+    return self._extended_message._extensions_by_name.get(name, None)
+
+  def _FindExtensionByNumber(self, number):
+    """Tries to find a known extension with the field number.
+
+    Args:
+      number: Extension field number.
+
+    Returns:
+      Extension field descriptor.
+    """
+    return self._extended_message._extensions_by_number.get(number, None)

python/google/protobuf/internal/json_format_test.py

@@ -52,7 +52,6 @@ from google.protobuf import wrappers_pb2
 from google.protobuf import any_test_pb2
 from google.protobuf import unittest_mset_pb2
 from google.protobuf import unittest_pb2
-from google.protobuf.internal import well_known_types
 from google.protobuf import descriptor_pool
 from google.protobuf import json_format
 from google.protobuf.util import json_format_proto3_pb2
@@ -481,6 +480,14 @@ class JsonFormatTest(JsonFormatBase):
     parsed_message = json_format_proto3_pb2.TestFieldMask()
     self.CheckParseBack(message, parsed_message)
 
+    message.value.Clear()
+    self.assertEqual(
+        json_format.MessageToJson(message, True),
+        '{\n'
+        '  "value": ""\n'
+        '}')
+    self.CheckParseBack(message, parsed_message)
+
   def testWrapperMessage(self):
     message = json_format_proto3_pb2.TestWrapper()
     message.bool_value.value = False
@@ -922,17 +929,18 @@ class JsonFormatTest(JsonFormatBase):
     text = '{"value": "10000-01-01T00:00:00.00Z"}'
     self.assertRaisesRegexp(
         json_format.ParseError,
+        'Failed to parse value field: '
         'time data \'10000-01-01T00:00:00\' does not match'
         ' format \'%Y-%m-%dT%H:%M:%S\'.',
         json_format.Parse, text, message)
     text = '{"value": "1970-01-01T00:00:00.0123456789012Z"}'
     self.assertRaisesRegexp(
-        well_known_types.ParseError,
+        json_format.ParseError,
         'nanos 0123456789012 more than 9 fractional digits.',
         json_format.Parse, text, message)
     text = '{"value": "1972-01-01T01:00:00.01+08"}'
     self.assertRaisesRegexp(
-        well_known_types.ParseError,
+        json_format.ParseError,
         (r'Invalid timezone offset value: \+08.'),
         json_format.Parse, text, message)
     # Time smaller than minimum time.

python/google/protobuf/internal/message_factory_test.py

@@ -136,13 +136,14 @@ class MessageFactoryTest(unittest.TestCase):
           'google.protobuf.python.internal.Factory2Message.one_more_field')
       ext2 = msg1.Extensions._FindExtensionByName(
           'google.protobuf.python.internal.another_field')
+      self.assertEqual(0, len(msg1.Extensions))
       msg1.Extensions[ext1] = 'test1'
       msg1.Extensions[ext2] = 'test2'
       self.assertEqual('test1', msg1.Extensions[ext1])
       self.assertEqual('test2', msg1.Extensions[ext2])
       self.assertEqual(None,
                        msg1.Extensions._FindExtensionByNumber(12321))
-      self.assertRaises(TypeError, len, msg1.Extensions)
+      self.assertEqual(2, len(msg1.Extensions))
       if api_implementation.Type() == 'cpp':
         self.assertRaises(TypeError,
                           msg1.Extensions._FindExtensionByName, 0)

python/google/protobuf/internal/message_test.py

@@ -411,6 +411,58 @@ class MessageTest(BaseTestCase):
     empty.ParseFromString(populated.SerializeToString())
     self.assertEqual(str(empty), '')
 
+  def testAppendRepeatedCompositeField(self, message_module):
+    msg = message_module.TestAllTypes()
+    msg.repeated_nested_message.append(
+        message_module.TestAllTypes.NestedMessage(bb=1))
+    nested = message_module.TestAllTypes.NestedMessage(bb=2)
+    msg.repeated_nested_message.append(nested)
+    try:
+      msg.repeated_nested_message.append(1)
+    except TypeError:
+      pass
+    self.assertEqual(2, len(msg.repeated_nested_message))
+    self.assertEqual([1, 2],
+                     [m.bb for m in msg.repeated_nested_message])
+
+  def testInsertRepeatedCompositeField(self, message_module):
+    msg = message_module.TestAllTypes()
+    msg.repeated_nested_message.insert(
+        -1, message_module.TestAllTypes.NestedMessage(bb=1))
+    sub_msg = msg.repeated_nested_message[0]
+    msg.repeated_nested_message.insert(
+        0, message_module.TestAllTypes.NestedMessage(bb=2))
+    msg.repeated_nested_message.insert(
+        99, message_module.TestAllTypes.NestedMessage(bb=3))
+    msg.repeated_nested_message.insert(
+        -2, message_module.TestAllTypes.NestedMessage(bb=-1))
+    msg.repeated_nested_message.insert(
+        -1000, message_module.TestAllTypes.NestedMessage(bb=-1000))
+    try:
+      msg.repeated_nested_message.insert(1, 999)
+    except TypeError:
+      pass
+    self.assertEqual(5, len(msg.repeated_nested_message))
+    self.assertEqual([-1000, 2, -1, 1, 3],
+                     [m.bb for m in msg.repeated_nested_message])
+    self.assertEqual(str(msg),
+                     'repeated_nested_message {\n'
+                     '  bb: -1000\n'
+                     '}\n'
+                     'repeated_nested_message {\n'
+                     '  bb: 2\n'
+                     '}\n'
+                     'repeated_nested_message {\n'
+                     '  bb: -1\n'
+                     '}\n'
+                     'repeated_nested_message {\n'
+                     '  bb: 1\n'
+                     '}\n'
+                     'repeated_nested_message {\n'
+                     '  bb: 3\n'
+                     '}\n')
+    self.assertEqual(sub_msg.bb, 1)
+
   def testMergeFromRepeatedField(self, message_module):
     msg = message_module.TestAllTypes()
     msg.repeated_int32.append(1)
@@ -442,6 +494,30 @@ class MessageTest(BaseTestCase):
       pass
     self.assertEqual(len(msg.repeated_nested_message), 0)
 
+  def testRepeatedContains(self, message_module):
+    msg = message_module.TestAllTypes()
+    msg.repeated_int32.extend([1, 2, 3])
+    self.assertIn(2, msg.repeated_int32)
+    self.assertNotIn(0, msg.repeated_int32)
+
+    msg.repeated_nested_message.add(bb=1)
+    sub_msg1 = msg.repeated_nested_message[0]
+    sub_msg2 = message_module.TestAllTypes.NestedMessage(bb=2)
+    sub_msg3 = message_module.TestAllTypes.NestedMessage(bb=3)
+    msg.repeated_nested_message.append(sub_msg2)
+    msg.repeated_nested_message.insert(0, sub_msg3)
+    self.assertIn(sub_msg1, msg.repeated_nested_message)
+    self.assertIn(sub_msg2, msg.repeated_nested_message)
+    self.assertIn(sub_msg3, msg.repeated_nested_message)
+
+  def testRepeatedScalarIterable(self, message_module):
+    msg = message_module.TestAllTypes()
+    msg.repeated_int32.extend([1, 2, 3])
+    add = 0
+    for item in msg.repeated_int32:
+      add += item
+    self.assertEqual(add, 6)
+
   def testRepeatedNestedFieldIteration(self, message_module):
     msg = message_module.TestAllTypes()
     msg.repeated_nested_message.add(bb=1)
@@ -1173,6 +1249,27 @@ class MessageTest(BaseTestCase):
     with self.assertRaises(AttributeError):
       m.repeated_int32 = []
 
+  def testReturningType(self, message_module):
+    m = message_module.TestAllTypes()
+    self.assertEqual(float, type(m.optional_float))
+    self.assertEqual(float, type(m.optional_double))
+    self.assertEqual(bool, type(m.optional_bool))
+    m.optional_float = 1
+    m.optional_double = 1
+    m.optional_bool = 1
+    m.repeated_float.append(1)
+    m.repeated_double.append(1)
+    m.repeated_bool.append(1)
+    m.ParseFromString(m.SerializeToString())
+    self.assertEqual(float, type(m.optional_float))
+    self.assertEqual(float, type(m.optional_double))
+    self.assertEqual('1.0', str(m.optional_double))
+    self.assertEqual(bool, type(m.optional_bool))
+    self.assertEqual(float, type(m.repeated_float[0]))
+    self.assertEqual(float, type(m.repeated_double[0]))
+    self.assertEqual(bool, type(m.repeated_bool[0]))
+    self.assertEqual(True, m.repeated_bool[0])
+
 
 # Class to test proto2-only features (required, extensions, etc.)
 class Proto2Test(BaseTestCase):

python/google/protobuf/internal/more_messages.proto

@@ -50,3 +50,262 @@ extend OutOfOrderFields {
   optional   uint64 optional_uint64   =  4;
   optional    int64 optional_int64    =  2;
 }
+
+message LotsNestedMessage {
+  message B0 {}
+  message B1 {}
+  message B2 {}
+  message B3 {}
+  message B4 {}
+  message B5 {}
+  message B6 {}
+  message B7 {}
+  message B8 {}
+  message B9 {}
+  message B10 {}
+  message B11 {}
+  message B12 {}
+  message B13 {}
+  message B14 {}
+  message B15 {}
+  message B16 {}
+  message B17 {}
+  message B18 {}
+  message B19 {}
+  message B20 {}
+  message B21 {}
+  message B22 {}
+  message B23 {}
+  message B24 {}
+  message B25 {}
+  message B26 {}
+  message B27 {}
+  message B28 {}
+  message B29 {}
+  message B30 {}
+  message B31 {}
+  message B32 {}
+  message B33 {}
+  message B34 {}
+  message B35 {}
+  message B36 {}
+  message B37 {}
+  message B38 {}
+  message B39 {}
+  message B40 {}
+  message B41 {}
+  message B42 {}
+  message B43 {}
+  message B44 {}
+  message B45 {}
+  message B46 {}
+  message B47 {}
+  message B48 {}
+  message B49 {}
+  message B50 {}
+  message B51 {}
+  message B52 {}
+  message B53 {}
+  message B54 {}
+  message B55 {}
+  message B56 {}
+  message B57 {}
+  message B58 {}
+  message B59 {}
+  message B60 {}
+  message B61 {}
+  message B62 {}
+  message B63 {}
+  message B64 {}
+  message B65 {}
+  message B66 {}
+  message B67 {}
+  message B68 {}
+  message B69 {}
+  message B70 {}
+  message B71 {}
+  message B72 {}
+  message B73 {}
+  message B74 {}
+  message B75 {}
+  message B76 {}
+  message B77 {}
+  message B78 {}
+  message B79 {}
+  message B80 {}
+  message B81 {}
+  message B82 {}
+  message B83 {}
+  message B84 {}
+  message B85 {}
+  message B86 {}
+  message B87 {}
+  message B88 {}
+  message B89 {}
+  message B90 {}
+  message B91 {}
+  message B92 {}
+  message B93 {}
+  message B94 {}
+  message B95 {}
+  message B96 {}
+  message B97 {}
+  message B98 {}
+  message B99 {}
+  message B100 {}
+  message B101 {}
+  message B102 {}
+  message B103 {}
+  message B104 {}
+  message B105 {}
+  message B106 {}
+  message B107 {}
+  message B108 {}
+  message B109 {}
+  message B110 {}
+  message B111 {}
+  message B112 {}
+  message B113 {}
+  message B114 {}
+  message B115 {}
+  message B116 {}
+  message B117 {}
+  message B118 {}
+  message B119 {}
+  message B120 {}
+  message B121 {}
+  message B122 {}
+  message B123 {}
+  message B124 {}
+  message B125 {}
+  message B126 {}
+  message B127 {}
+  message B128 {}
+  message B129 {}
+  message B130 {}
+  message B131 {}
+  message B132 {}
+  message B133 {}
+  message B134 {}
+  message B135 {}
+  message B136 {}
+  message B137 {}
+  message B138 {}
+  message B139 {}
+  message B140 {}
+  message B141 {}
+  message B142 {}
+  message B143 {}
+  message B144 {}
+  message B145 {}
+  message B146 {}
+  message B147 {}
+  message B148 {}
+  message B149 {}
+  message B150 {}
+  message B151 {}
+  message B152 {}
+  message B153 {}
+  message B154 {}
+  message B155 {}
+  message B156 {}
+  message B157 {}
+  message B158 {}
+  message B159 {}
+  message B160 {}
+  message B161 {}
+  message B162 {}
+  message B163 {}
+  message B164 {}
+  message B165 {}
+  message B166 {}
+  message B167 {}
+  message B168 {}
+  message B169 {}
+  message B170 {}
+  message B171 {}
+  message B172 {}
+  message B173 {}
+  message B174 {}
+  message B175 {}
+  message B176 {}
+  message B177 {}
+  message B178 {}
+  message B179 {}
+  message B180 {}
+  message B181 {}
+  message B182 {}
+  message B183 {}
+  message B184 {}
+  message B185 {}
+  message B186 {}
+  message B187 {}
+  message B188 {}
+  message B189 {}
+  message B190 {}
+  message B191 {}
+  message B192 {}
+  message B193 {}
+  message B194 {}
+  message B195 {}
+  message B196 {}
+  message B197 {}
+  message B198 {}
+  message B199 {}
+  message B200 {}
+  message B201 {}
+  message B202 {}
+  message B203 {}
+  message B204 {}
+  message B205 {}
+  message B206 {}
+  message B207 {}
+  message B208 {}
+  message B209 {}
+  message B210 {}
+  message B211 {}
+  message B212 {}
+  message B213 {}
+  message B214 {}
+  message B215 {}
+  message B216 {}
+  message B217 {}
+  message B218 {}
+  message B219 {}
+  message B220 {}
+  message B221 {}
+  message B222 {}
+  message B223 {}
+  message B224 {}
+  message B225 {}
+  message B226 {}
+  message B227 {}
+  message B228 {}
+  message B229 {}
+  message B230 {}
+  message B231 {}
+  message B232 {}
+  message B233 {}
+  message B234 {}
+  message B235 {}
+  message B236 {}
+  message B237 {}
+  message B238 {}
+  message B239 {}
+  message B240 {}
+  message B241 {}
+  message B242 {}
+  message B243 {}
+  message B244 {}
+  message B245 {}
+  message B246 {}
+  message B247 {}
+  message B248 {}
+  message B249 {}
+  message B250 {}
+  message B251 {}
+  message B252 {}
+  message B253 {}
+  message B254 {}
+  message B255 {}
+}

python/google/protobuf/internal/python_message.py

@@ -64,6 +64,7 @@ from google.protobuf.internal import containers
 from google.protobuf.internal import decoder
 from google.protobuf.internal import encoder
 from google.protobuf.internal import enum_type_wrapper
+from google.protobuf.internal import extension_dict
 from google.protobuf.internal import message_listener as message_listener_mod
 from google.protobuf.internal import type_checkers
 from google.protobuf.internal import well_known_types
@@ -74,7 +75,7 @@ from google.protobuf import text_format
 
 _FieldDescriptor = descriptor_mod.FieldDescriptor
 _AnyFullTypeName = 'google.protobuf.Any'
-
+_ExtensionDict = extension_dict._ExtensionDict
 
 class GeneratedProtocolMessageType(type):
 
@@ -237,28 +238,6 @@ def _PropertyName(proto_field_name):
   return proto_field_name
 
 
-def _VerifyExtensionHandle(message, extension_handle):
-  """Verify that the given extension handle is valid."""
-
-  if not isinstance(extension_handle, _FieldDescriptor):
-    raise KeyError('HasExtension() expects an extension handle, got: %s' %
-                   extension_handle)
-
-  if not extension_handle.is_extension:
-    raise KeyError('"%s" is not an extension.' % extension_handle.full_name)
-
-  if not extension_handle.containing_type:
-    raise KeyError('"%s" is missing a containing_type.'
-                   % extension_handle.full_name)
-
-  if extension_handle.containing_type is not message.DESCRIPTOR:
-    raise KeyError('Extension "%s" extends message type "%s", but this '
-                   'message is of type "%s".' %
-                   (extension_handle.full_name,
-                    extension_handle.containing_type.full_name,
-                    message.DESCRIPTOR.full_name))
-
-
 def _AddSlots(message_descriptor, dictionary):
   """Adds a __slots__ entry to dictionary, containing the names of all valid
   attributes for this message type.
@@ -379,8 +358,8 @@ def _AttachFieldHelpers(cls, field_descriptor):
 
 
 def _AddClassAttributesForNestedExtensions(descriptor, dictionary):
-  extension_dict = descriptor.extensions_by_name
-  for extension_name, extension_field in extension_dict.items():
+  extensions = descriptor.extensions_by_name
+  for extension_name, extension_field in extensions.items():
     assert extension_name not in dictionary
     dictionary[extension_name] = extension_field
 
@@ -784,8 +763,8 @@ def _AddPropertiesForNonRepeatedCompositeField(field, cls):
 
 def _AddPropertiesForExtensions(descriptor, cls):
   """Adds properties for all fields in this protocol message type."""
-  extension_dict = descriptor.extensions_by_name
-  for extension_name, extension_field in extension_dict.items():
+  extensions = descriptor.extensions_by_name
+  for extension_name, extension_field in extensions.items():
     constant_name = extension_name.upper() + '_FIELD_NUMBER'
     setattr(cls, constant_name, extension_field.number)
 
@@ -922,7 +901,7 @@ def _AddClearFieldMethod(message_descriptor, cls):
 def _AddClearExtensionMethod(cls):
   """Helper for _AddMessageMethods()."""
   def ClearExtension(self, extension_handle):
-    _VerifyExtensionHandle(self, extension_handle)
+    extension_dict._VerifyExtensionHandle(self, extension_handle)
 
     # Similar to ClearField(), above.
     if extension_handle in self._fields:
@@ -934,7 +913,7 @@ def _AddClearExtensionMethod(cls):
 def _AddHasExtensionMethod(cls):
   """Helper for _AddMessageMethods()."""
   def HasExtension(self, extension_handle):
-    _VerifyExtensionHandle(self, extension_handle)
+    extension_dict._VerifyExtensionHandle(self, extension_handle)
     if extension_handle.label == _FieldDescriptor.LABEL_REPEATED:
       raise KeyError('"%s" is repeated.' % extension_handle.full_name)
 
@@ -1550,126 +1529,3 @@ class _OneofListener(_Listener):
       super(_OneofListener, self).Modified()
     except ReferenceError:
       pass
-
-
-# TODO(robinson): Move elsewhere?  This file is getting pretty ridiculous...
-# TODO(robinson): Unify error handling of "unknown extension" crap.
-# TODO(robinson): Support iteritems()-style iteration over all
-# extensions with the "has" bits turned on?
-class _ExtensionDict(object):
-
-  """Dict-like container for supporting an indexable "Extensions"
-  field on proto instances.
-
-  Note that in all cases we expect extension handles to be
-  FieldDescriptors.
-  """
-
-  def __init__(self, extended_message):
-    """extended_message: Message instance for which we are the Extensions dict.
-    """
-
-    self._extended_message = extended_message
-
-  def __getitem__(self, extension_handle):
-    """Returns the current value of the given extension handle."""
-
-    _VerifyExtensionHandle(self._extended_message, extension_handle)
-
-    result = self._extended_message._fields.get(extension_handle)
-    if result is not None:
-      return result
-
-    if extension_handle.label == _FieldDescriptor.LABEL_REPEATED:
-      result = extension_handle._default_constructor(self._extended_message)
-    elif extension_handle.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE:
-      assert getattr(extension_handle.message_type, '_concrete_class', None), (
-          'Uninitialized concrete class found for field %r (message type %r)'
-          % (extension_handle.full_name,
-             extension_handle.message_type.full_name))
-      result = extension_handle.message_type._concrete_class()
-      try:
-        result._SetListener(self._extended_message._listener_for_children)
-      except ReferenceError:
-        pass
-    else:
-      # Singular scalar -- just return the default without inserting into the
-      # dict.
-      return extension_handle.default_value
-
-    # Atomically check if another thread has preempted us and, if not, swap
-    # in the new object we just created.  If someone has preempted us, we
-    # take that object and discard ours.
-    # WARNING:  We are relying on setdefault() being atomic.  This is true
-    #   in CPython but we haven't investigated others.  This warning appears
-    #   in several other locations in this file.
-    result = self._extended_message._fields.setdefault(
-        extension_handle, result)
-
-    return result
-
-  def __eq__(self, other):
-    if not isinstance(other, self.__class__):
-      return False
-
-    my_fields = self._extended_message.ListFields()
-    other_fields = other._extended_message.ListFields()
-
-    # Get rid of non-extension fields.
-    my_fields = [field for field in my_fields if field.is_extension]
-    other_fields = [field for field in other_fields if field.is_extension]
-
-    return my_fields == other_fields
-
-  def __ne__(self, other):
-    return not self == other
-
-  def __hash__(self):
-    raise TypeError('unhashable object')
-
-  # Note that this is only meaningful for non-repeated, scalar extension
-  # fields.  Note also that we may have to call _Modified() when we do
-  # successfully set a field this way, to set any necssary "has" bits in the
-  # ancestors of the extended message.
-  def __setitem__(self, extension_handle, value):
-    """If extension_handle specifies a non-repeated, scalar extension
-    field, sets the value of that field.
-    """
-
-    _VerifyExtensionHandle(self._extended_message, extension_handle)
-
-    if (extension_handle.label == _FieldDescriptor.LABEL_REPEATED or
-        extension_handle.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE):
-      raise TypeError(
-          'Cannot assign to extension "%s" because it is a repeated or '
-          'composite type.' % extension_handle.full_name)
-
-    # It's slightly wasteful to lookup the type checker each time,
-    # but we expect this to be a vanishingly uncommon case anyway.
-    type_checker = type_checkers.GetTypeChecker(extension_handle)
-    # pylint: disable=protected-access
-    self._extended_message._fields[extension_handle] = (
-        type_checker.CheckValue(value))
-    self._extended_message._Modified()
-
-  def _FindExtensionByName(self, name):
-    """Tries to find a known extension with the specified name.
-
-    Args:
-      name: Extension full name.
-
-    Returns:
-      Extension field descriptor.
-    """
-    return self._extended_message._extensions_by_name.get(name, None)
-
-  def _FindExtensionByNumber(self, number):
-    """Tries to find a known extension with the field number.
-
-    Args:
-      number: Extension field number.
-
-    Returns:
-      Extension field descriptor.
-    """
-    return self._extended_message._extensions_by_number.get(number, None)

python/google/protobuf/internal/type_checkers.py

@@ -107,13 +107,18 @@ class TypeChecker(object):
       message = ('%.1024r has type %s, but expected one of: %s' %
                  (proposed_value, type(proposed_value), self._acceptable_types))
       raise TypeError(message)
+    # Some field types(float, double and bool) accept other types, must
+    # convert to the correct type in such cases.
+    if self._acceptable_types:
+      if self._acceptable_types[0] in (bool, float):
+        return self._acceptable_types[0](proposed_value)
     return proposed_value
 
 
 class TypeCheckerWithDefault(TypeChecker):
 
   def __init__(self, default_value, *acceptable_types):
-    TypeChecker.__init__(self, acceptable_types)
+    TypeChecker.__init__(self, *acceptable_types)
     self._default_value = default_value
 
   def DefaultValue(self):
@@ -232,9 +237,9 @@ _VALUE_CHECKERS = {
     _FieldDescriptor.CPPTYPE_UINT32: Uint32ValueChecker(),
     _FieldDescriptor.CPPTYPE_UINT64: Uint64ValueChecker(),
     _FieldDescriptor.CPPTYPE_DOUBLE: TypeCheckerWithDefault(
-        0.0, numbers.Real),
+        0.0, float, numbers.Real),
     _FieldDescriptor.CPPTYPE_FLOAT: TypeCheckerWithDefault(
-        0.0, numbers.Real),
+        0.0, float, numbers.Real),
     _FieldDescriptor.CPPTYPE_BOOL: TypeCheckerWithDefault(
         False, bool, numbers.Integral),
     _FieldDescriptor.CPPTYPE_STRING: TypeCheckerWithDefault(b'', bytes),

python/google/protobuf/internal/well_known_types.py

@@ -58,14 +58,6 @@ _SECONDS_PER_DAY = 24 * 3600
 _DURATION_SECONDS_MAX = 315576000000
 
 
-class Error(Exception):
-  """Top-level module error."""
-
-
-class ParseError(Error):
-  """Thrown in case of parsing error."""
-
-
 class Any(object):
   """Class for Any Message type."""
 
@@ -136,7 +128,7 @@ class Timestamp(object):
           Example of accepted format: '1972-01-01T10:00:20.021-05:00'
 
     Raises:
-      ParseError: On parsing problems.
+      ValueError: On parsing problems.
     """
     timezone_offset = value.find('Z')
     if timezone_offset == -1:
@@ -144,7 +136,7 @@ class Timestamp(object):
     if timezone_offset == -1:
       timezone_offset = value.rfind('-')
     if timezone_offset == -1:
-      raise ParseError(
+      raise ValueError(
           'Failed to parse timestamp: missing valid timezone offset.')
     time_value = value[0:timezone_offset]
     # Parse datetime and nanos.
@@ -159,7 +151,7 @@ class Timestamp(object):
     td = date_object - datetime(1970, 1, 1)
     seconds = td.seconds + td.days * _SECONDS_PER_DAY
     if len(nano_value) > 9:
-      raise ParseError(
+      raise ValueError(
           'Failed to parse Timestamp: nanos {0} more than '
           '9 fractional digits.'.format(nano_value))
     if nano_value:
@@ -169,13 +161,13 @@ class Timestamp(object):
     # Parse timezone offsets.
     if value[timezone_offset] == 'Z':
       if len(value) != timezone_offset + 1:
-        raise ParseError('Failed to parse timestamp: invalid trailing'
+        raise ValueError('Failed to parse timestamp: invalid trailing'
                          ' data {0}.'.format(value))
     else:
       timezone = value[timezone_offset:]
       pos = timezone.find(':')
       if pos == -1:
-        raise ParseError(
+        raise ValueError(
             'Invalid timezone offset value: {0}.'.format(timezone))
       if timezone[0] == '+':
         seconds -= (int(timezone[1:pos])*60+int(timezone[pos+1:]))*60
@@ -289,10 +281,10 @@ class Duration(object):
           precision. For example: "1s", "1.01s", "1.0000001s", "-3.100s
 
     Raises:
-      ParseError: On parsing problems.
+      ValueError: On parsing problems.
     """
     if len(value) < 1 or value[-1] != 's':
-      raise ParseError(
+      raise ValueError(
           'Duration must end with letter "s": {0}.'.format(value))
     try:
       pos = value.find('.')
@@ -308,9 +300,9 @@ class Duration(object):
       _CheckDurationValid(seconds, nanos)
       self.seconds = seconds
       self.nanos = nanos
-    except ValueError:
-      raise ParseError(
-          'Couldn\'t parse duration: {0}.'.format(value))
+    except ValueError as e:
+      raise ValueError(
+          'Couldn\'t parse duration: {0} : {1}.'.format(value, e))
 
   def ToNanoseconds(self):
     """Converts a Duration to nanoseconds."""
@@ -375,15 +367,15 @@ class Duration(object):
 
 def _CheckDurationValid(seconds, nanos):
   if seconds < -_DURATION_SECONDS_MAX or seconds > _DURATION_SECONDS_MAX:
-    raise Error(
+    raise ValueError(
         'Duration is not valid: Seconds {0} must be in range '
         '[-315576000000, 315576000000].'.format(seconds))
   if nanos <= -_NANOS_PER_SECOND or nanos >= _NANOS_PER_SECOND:
-    raise Error(
+    raise ValueError(
         'Duration is not valid: Nanos {0} must be in range '
         '[-999999999, 999999999].'.format(nanos))
   if (nanos < 0 and seconds > 0) or (nanos > 0 and seconds < 0):
-    raise Error(
+    raise ValueError(
         'Duration is not valid: Sign mismatch.')
 
 
@@ -415,8 +407,9 @@ class FieldMask(object):
   def FromJsonString(self, value):
     """Converts string to FieldMask according to proto3 JSON spec."""
     self.Clear()
-    for path in value.split(','):
-      self.paths.append(_CamelCaseToSnakeCase(path))
+    if value:
+      for path in value.split(','):
+        self.paths.append(_CamelCaseToSnakeCase(path))
 
   def IsValidForDescriptor(self, message_descriptor):
     """Checks whether the FieldMask is valid for Message Descriptor."""
@@ -509,24 +502,26 @@ def _SnakeCaseToCamelCase(path_name):
   after_underscore = False
   for c in path_name:
     if c.isupper():
-      raise Error('Fail to print FieldMask to Json string: Path name '
-                  '{0} must not contain uppercase letters.'.format(path_name))
+      raise ValueError(
+          'Fail to print FieldMask to Json string: Path name '
+          '{0} must not contain uppercase letters.'.format(path_name))
     if after_underscore:
       if c.islower():
         result.append(c.upper())
         after_underscore = False
       else:
-        raise Error('Fail to print FieldMask to Json string: The '
-                    'character after a "_" must be a lowercase letter '
-                    'in path name {0}.'.format(path_name))
+        raise ValueError(
+            'Fail to print FieldMask to Json string: The '
+            'character after a "_" must be a lowercase letter '
+            'in path name {0}.'.format(path_name))
     elif c == '_':
       after_underscore = True
     else:
       result += c
 
   if after_underscore:
-    raise Error('Fail to print FieldMask to Json string: Trailing "_" '
-                'in path name {0}.'.format(path_name))
+    raise ValueError('Fail to print FieldMask to Json string: Trailing "_" '
+                     'in path name {0}.'.format(path_name))
   return ''.join(result)
 
 
@@ -535,7 +530,7 @@ def _CamelCaseToSnakeCase(path_name):
   result = []
   for c in path_name:
     if c == '_':
-      raise ParseError('Fail to parse FieldMask: Path name '
+      raise ValueError('Fail to parse FieldMask: Path name '
                        '{0} must not contain "_"s.'.format(path_name))
     if c.isupper():
       result += '_'
@@ -682,7 +677,7 @@ def _MergeMessage(
 
 def _AddFieldPaths(node, prefix, field_mask):
   """Adds the field paths descended from node to field_mask."""
-  if not node:
+  if not node and prefix:
     field_mask.paths.append(prefix)
     return
   for name in sorted(node):

python/google/protobuf/internal/well_known_types_test.py

@@ -294,12 +294,12 @@ class TimeUtilTest(TimeUtilTestBase):
   def testInvalidTimestamp(self):
     message = timestamp_pb2.Timestamp()
     self.assertRaisesRegexp(
-        well_known_types.ParseError,
+        ValueError,
         'Failed to parse timestamp: missing valid timezone offset.',
         message.FromJsonString,
         '')
     self.assertRaisesRegexp(
-        well_known_types.ParseError,
+        ValueError,
         'Failed to parse timestamp: invalid trailing data '
         '1970-01-01T00:00:01Ztrail.',
         message.FromJsonString,
@@ -310,12 +310,12 @@ class TimeUtilTest(TimeUtilTestBase):
         ' format \'%Y-%m-%dT%H:%M:%S\'',
         message.FromJsonString, '10000-01-01T00:00:00.00Z')
     self.assertRaisesRegexp(
-        well_known_types.ParseError,
+        ValueError,
         'nanos 0123456789012 more than 9 fractional digits.',
         message.FromJsonString,
         '1970-01-01T00:00:00.0123456789012Z')
     self.assertRaisesRegexp(
-        well_known_types.ParseError,
+        ValueError,
         (r'Invalid timezone offset value: \+08.'),
         message.FromJsonString,
         '1972-01-01T01:00:00.01+08',)
@@ -333,43 +333,43 @@ class TimeUtilTest(TimeUtilTestBase):
   def testInvalidDuration(self):
     message = duration_pb2.Duration()
     self.assertRaisesRegexp(
-        well_known_types.ParseError,
+        ValueError,
         'Duration must end with letter "s": 1.',
         message.FromJsonString, '1')
     self.assertRaisesRegexp(
-        well_known_types.ParseError,
+        ValueError,
         'Couldn\'t parse duration: 1...2s.',
         message.FromJsonString, '1...2s')
     text = '-315576000001.000000000s'
     self.assertRaisesRegexp(
-        well_known_types.Error,
+        ValueError,
         r'Duration is not valid\: Seconds -315576000001 must be in range'
         r' \[-315576000000\, 315576000000\].',
         message.FromJsonString, text)
     text = '315576000001.000000000s'
     self.assertRaisesRegexp(
-        well_known_types.Error,
+        ValueError,
         r'Duration is not valid\: Seconds 315576000001 must be in range'
         r' \[-315576000000\, 315576000000\].',
         message.FromJsonString, text)
     message.seconds = -315576000001
     message.nanos = 0
     self.assertRaisesRegexp(
-        well_known_types.Error,
+        ValueError,
         r'Duration is not valid\: Seconds -315576000001 must be in range'
         r' \[-315576000000\, 315576000000\].',
         message.ToJsonString)
     message.seconds = 0
     message.nanos = 999999999 + 1
     self.assertRaisesRegexp(
-        well_known_types.Error,
+        ValueError,
         r'Duration is not valid\: Nanos 1000000000 must be in range'
         r' \[-999999999\, 999999999\].',
         message.ToJsonString)
     message.seconds = -1
     message.nanos = 1
     self.assertRaisesRegexp(
-        well_known_types.Error,
+        ValueError,
         r'Duration is not valid\: Sign mismatch.',
         message.ToJsonString)
 
@@ -400,6 +400,7 @@ class FieldMaskTest(unittest.TestCase):
 
     mask.FromJsonString('')
     self.assertEqual('', mask.ToJsonString())
+    self.assertEqual([], mask.paths)
     mask.FromJsonString('fooBar')
     self.assertEqual(['foo_bar'], mask.paths)
     mask.FromJsonString('fooBar,barQuz')
@@ -512,6 +513,8 @@ class FieldMaskTest(unittest.TestCase):
     mask2.FromJsonString('bar,quz')
     out_mask.Intersect(mask1, mask2)
     self.assertEqual('', out_mask.ToJsonString())
+    self.assertEqual(len(out_mask.paths), 0)
+    self.assertEqual(out_mask.paths, [])
     # Overlap with duplicated paths.
     mask1.FromJsonString('foo,baz.bb')
     mask2.FromJsonString('baz.bb,quz')
@@ -526,6 +529,15 @@ class FieldMaskTest(unittest.TestCase):
     mask2.FromJsonString('foo.bar.baz,quz')
     out_mask.Intersect(mask1, mask2)
     self.assertEqual('foo.bar.baz', out_mask.ToJsonString())
+    # Intersect '' with ''
+    mask1.Clear()
+    mask2.Clear()
+    mask1.paths.append('')
+    mask2.paths.append('')
+    self.assertEqual(mask1.paths, [''])
+    self.assertEqual('', mask1.ToJsonString())
+    out_mask.Intersect(mask1, mask2)
+    self.assertEqual(out_mask.paths, [])
 
   def testMergeMessageWithoutMapFields(self):
     # Test merge one field.
@@ -682,7 +694,7 @@ class FieldMaskTest(unittest.TestCase):
 
     # No uppercase letter is allowed.
     self.assertRaisesRegexp(
-        well_known_types.Error,
+        ValueError,
         'Fail to print FieldMask to Json string: Path name Foo must '
         'not contain uppercase letters.',
         well_known_types._SnakeCaseToCamelCase,
@@ -692,19 +704,19 @@ class FieldMaskTest(unittest.TestCase):
     #   2. "_" cannot be followed by a digit.
     #   3. "_" cannot appear as the last character.
     self.assertRaisesRegexp(
-        well_known_types.Error,
+        ValueError,
         'Fail to print FieldMask to Json string: The character after a '
         '"_" must be a lowercase letter in path name foo__bar.',
         well_known_types._SnakeCaseToCamelCase,
         'foo__bar')
     self.assertRaisesRegexp(
-        well_known_types.Error,
+        ValueError,
         'Fail to print FieldMask to Json string: The character after a '
         '"_" must be a lowercase letter in path name foo_3bar.',
         well_known_types._SnakeCaseToCamelCase,
         'foo_3bar')
     self.assertRaisesRegexp(
-        well_known_types.Error,
+        ValueError,
         'Fail to print FieldMask to Json string: Trailing "_" in path '
         'name foo_bar_.',
         well_known_types._SnakeCaseToCamelCase,
@@ -718,7 +730,7 @@ class FieldMaskTest(unittest.TestCase):
     self.assertEqual('foo3_bar',
                      well_known_types._CamelCaseToSnakeCase('foo3Bar'))
     self.assertRaisesRegexp(
-        well_known_types.ParseError,
+        ValueError,
         'Fail to parse FieldMask: Path name foo_bar must not contain "_"s.',
         well_known_types._CamelCaseToSnakeCase,
         'foo_bar')

python/google/protobuf/json_format.py

@@ -570,7 +570,7 @@ class _Parser(object):
             setattr(message, field.name, _ConvertScalarFieldValue(value, field))
       except ParseError as e:
         if field and field.containing_oneof is None:
-          raise ParseError('Failed to parse {0} field: {1}'.format(name, e))
+          raise ParseError('Failed to parse {0} field: {1}.'.format(name, e))
         else:
           raise ParseError(str(e))
       except ValueError as e:
@@ -607,7 +607,10 @@ class _Parser(object):
     """Convert a JSON representation into message with FromJsonString."""
     # Duration, Timestamp, FieldMask have a FromJsonString method to do the
     # conversion. Users can also call the method directly.
-    message.FromJsonString(value)
+    try:
+      message.FromJsonString(value)
+    except ValueError as e:
+      raise ParseError(e)
 
   def _ConvertValueMessage(self, value, message):
     """Convert a JSON representation into Value message."""

python/google/protobuf/proto_api.h

@@ -47,6 +47,7 @@
 
 #include <Python.h>
 
+#include <google/protobuf/descriptor_database.h>
 #include <google/protobuf/message.h>
 
 namespace google {
@@ -76,6 +77,11 @@ struct PyProto_API {
   // pointing to the message, like submessages or repeated containers.
   // With the current implementation, only empty messages are in this case.
   virtual Message* GetMutableMessagePointer(PyObject* msg) const = 0;
+
+  // Expose the underlying DescriptorPool and MessageFactory to enable C++ code
+  // to create Python-compatible message.
+  virtual const DescriptorPool* GetDefaultDescriptorPool() const = 0;
+  virtual MessageFactory* GetDefaultMessageFactory() const = 0;
 };
 
 inline const char* PyProtoAPICapsuleName() {

python/google/protobuf/pyext/extension_dict.cc

@@ -65,6 +65,31 @@ namespace python {
 
 namespace extension_dict {
 
+static Py_ssize_t len(ExtensionDict* self) {
+  Py_ssize_t size = 0;
+  std::vector<const FieldDescriptor*> fields;
+  self->parent->message->GetReflection()->ListFields(*self->parent->message,
+                                                     &fields);
+
+  for (size_t i = 0; i < fields.size(); ++i) {
+    if (fields[i]->is_extension()) {
+      // With C++ descriptors, the field can always be retrieved, but for
+      // unknown extensions which have not been imported in Python code, there
+      // is no message class and we cannot retrieve the value.
+      // ListFields() has the same behavior.
+      if (fields[i]->message_type() != nullptr &&
+          message_factory::GetMessageClass(
+              cmessage::GetFactoryForMessage(self->parent),
+              fields[i]->message_type()) == nullptr) {
+        PyErr_Clear();
+        continue;
+      }
+      ++size;
+    }
+  }
+  return size;
+}
+
 PyObject* subscript(ExtensionDict* self, PyObject* key) {
   const FieldDescriptor* descriptor = cmessage::GetExtensionDescriptor(key);
   if (descriptor == NULL) {
@@ -246,7 +271,7 @@ static PyObject* RichCompare(ExtensionDict* self, PyObject* other, int opid) {
 }
 
 static PyMappingMethods MpMethods = {
-  (lenfunc)NULL,               /* mp_length */
+  (lenfunc)len,                /* mp_length */
   (binaryfunc)subscript,       /* mp_subscript */
   (objobjargproc)ass_subscript,/* mp_ass_subscript */
 };

python/google/protobuf/pyext/message_module.cc

@@ -30,7 +30,9 @@
 
 #include <Python.h>
 
+#include <google/protobuf/pyext/descriptor_pool.h>
 #include <google/protobuf/pyext/message.h>
+#include <google/protobuf/pyext/message_factory.h>
 #include <google/protobuf/proto_api.h>
 
 #include <google/protobuf/message_lite.h>
@@ -45,37 +47,42 @@ struct ApiImplementation : google::protobuf::python::PyProto_API {
   google::protobuf::Message* GetMutableMessagePointer(PyObject* msg) const override {
     return google::protobuf::python::PyMessage_GetMutableMessagePointer(msg);
   }
+  const google::protobuf::DescriptorPool* GetDefaultDescriptorPool() const override {
+    return google::protobuf::python::GetDefaultDescriptorPool()->pool;
+  }
+
+  google::protobuf::MessageFactory* GetDefaultMessageFactory() const override {
+    return google::protobuf::python::GetDefaultDescriptorPool()
+        ->py_message_factory->message_factory;
+  }
 };
 
 }  // namespace
 
 static const char module_docstring[] =
-"python-proto2 is a module that can be used to enhance proto2 Python API\n"
-"performance.\n"
-"\n"
-"It provides access to the protocol buffers C++ reflection API that\n"
-"implements the basic protocol buffer functions.";
+    "python-proto2 is a module that can be used to enhance proto2 Python API\n"
+    "performance.\n"
+    "\n"
+    "It provides access to the protocol buffers C++ reflection API that\n"
+    "implements the basic protocol buffer functions.";
 
 static PyMethodDef ModuleMethods[] = {
-  {"SetAllowOversizeProtos",
-    (PyCFunction)google::protobuf::python::cmessage::SetAllowOversizeProtos,
-    METH_O, "Enable/disable oversize proto parsing."},
-  // DO NOT USE: For migration and testing only.
-  { NULL, NULL}
-};
+    {"SetAllowOversizeProtos",
+     (PyCFunction)google::protobuf::python::cmessage::SetAllowOversizeProtos, METH_O,
+     "Enable/disable oversize proto parsing."},
+    // DO NOT USE: For migration and testing only.
+    {NULL, NULL}};
 
 #if PY_MAJOR_VERSION >= 3
-static struct PyModuleDef _module = {
-  PyModuleDef_HEAD_INIT,
-  "_message",
-  module_docstring,
-  -1,
-  ModuleMethods,  /* m_methods */
-  NULL,
-  NULL,
-  NULL,
-  NULL
-};
+static struct PyModuleDef _module = {PyModuleDef_HEAD_INIT,
+                                     "_message",
+                                     module_docstring,
+                                     -1,
+                                     ModuleMethods, /* m_methods */
+                                     NULL,
+                                     NULL,
+                                     NULL,
+                                     NULL};
 #define INITFUNC PyInit__message
 #define INITFUNC_ERRORVAL NULL
 #else  // Python 2

python/google/protobuf/pyext/repeated_composite_container.cc

@@ -159,10 +159,6 @@ static PyObject* AddToAttached(RepeatedCompositeContainer* self,
   }
 
   PyObject* py_cmsg = reinterpret_cast<PyObject*>(cmsg);
-  if (PyList_Append(self->child_messages, py_cmsg) < 0) {
-    Py_DECREF(py_cmsg);
-    return NULL;
-  }
   return py_cmsg;
 }
 
@@ -174,6 +170,18 @@ static PyObject* AddToReleased(RepeatedCompositeContainer* self,
   // Create a new Message detached from the rest.
   PyObject* py_cmsg = PyEval_CallObjectWithKeywords(
       self->child_message_class->AsPyObject(), args, kwargs);
+  return py_cmsg;
+}
+
+PyObject* Add(RepeatedCompositeContainer* self,
+              PyObject* args,
+              PyObject* kwargs) {
+  PyObject* py_cmsg;
+  if (self->message == nullptr)
+    py_cmsg = AddToReleased(self, args, kwargs);
+  else
+    py_cmsg = AddToAttached(self, args, kwargs);
+
   if (py_cmsg == NULL)
     return NULL;
 
@@ -184,19 +192,97 @@ static PyObject* AddToReleased(RepeatedCompositeContainer* self,
   return py_cmsg;
 }
 
-PyObject* Add(RepeatedCompositeContainer* self,
-              PyObject* args,
-              PyObject* kwargs) {
-  if (self->message == NULL)
-    return AddToReleased(self, args, kwargs);
-  else
-    return AddToAttached(self, args, kwargs);
-}
-
 static PyObject* AddMethod(PyObject* self, PyObject* args, PyObject* kwargs) {
   return Add(reinterpret_cast<RepeatedCompositeContainer*>(self), args, kwargs);
 }
 
+// ---------------------------------------------------------------------
+// append()
+
+static PyObject* AddMessage(RepeatedCompositeContainer* self, PyObject* value) {
+  cmessage::AssureWritable(self->parent);
+  if (UpdateChildMessages(self) < 0) {
+    return nullptr;
+  }
+
+  PyObject* py_cmsg;
+  if (self->message == nullptr) {
+    py_cmsg = AddToReleased(self, nullptr, nullptr);
+    if (py_cmsg == nullptr) return nullptr;
+    CMessage* cmsg = reinterpret_cast<CMessage*>(py_cmsg);
+    if (ScopedPyObjectPtr(cmessage::MergeFrom(cmsg, value)) == nullptr) {
+      Py_DECREF(cmsg);
+      return nullptr;
+    }
+  } else {
+    Message* message = self->message;
+    const Reflection* reflection = message->GetReflection();
+    py_cmsg = AddToAttached(self, nullptr, nullptr);
+    if (py_cmsg == nullptr) return nullptr;
+    CMessage* cmsg = reinterpret_cast<CMessage*>(py_cmsg);
+    if (ScopedPyObjectPtr(cmessage::MergeFrom(cmsg, value)) == nullptr) {
+      reflection->RemoveLast(
+          message, self->parent_field_descriptor);
+      Py_DECREF(cmsg);
+      return nullptr;
+    }
+  }
+  return py_cmsg;
+}
+
+static PyObject* AppendMethod(PyObject* pself, PyObject* value) {
+  RepeatedCompositeContainer* self =
+      reinterpret_cast<RepeatedCompositeContainer*>(pself);
+  PyObject* py_cmsg = AddMessage(self, value);
+  if (py_cmsg == nullptr) {
+    return nullptr;
+  }
+
+  if (PyList_Append(self->child_messages, py_cmsg) < 0) {
+    Py_DECREF(py_cmsg);
+    return nullptr;
+  }
+
+  Py_RETURN_NONE;
+}
+
+// ---------------------------------------------------------------------
+// insert()
+static PyObject* Insert(PyObject* pself, PyObject* args) {
+  RepeatedCompositeContainer* self =
+      reinterpret_cast<RepeatedCompositeContainer*>(pself);
+
+  Py_ssize_t index;
+  PyObject* value;
+  if (!PyArg_ParseTuple(args, "nO", &index, &value)) {
+    return nullptr;
+  }
+
+  PyObject* py_cmsg = AddMessage(self, value);
+  if (py_cmsg == nullptr) {
+    return nullptr;
+  }
+
+  if (self->message != nullptr) {
+    // Swap the element to right position.
+    Message* message = self->message;
+    const Reflection* reflection = message->GetReflection();
+    const FieldDescriptor* field_descriptor = self->parent_field_descriptor;
+    Py_ssize_t length = reflection->FieldSize(*message, field_descriptor) - 1;
+    Py_ssize_t end_index = index;
+    if (end_index < 0) end_index += length;
+    if (end_index < 0) end_index = 0;
+    for (Py_ssize_t i = length; i > end_index; i --) {
+      reflection->SwapElements(message, field_descriptor, i, i - 1);
+    }
+  }
+
+  if (PyList_Insert(self->child_messages, index, py_cmsg) < 0) {
+    return nullptr;
+  }
+  Py_RETURN_NONE;
+}
+
 // ---------------------------------------------------------------------
 // extend()
 
@@ -638,6 +724,10 @@ static PyMethodDef Methods[] = {
     "Makes a deep copy of the class." },
   { "add", (PyCFunction)AddMethod, METH_VARARGS | METH_KEYWORDS,
     "Adds an object to the repeated container." },
+  { "append", AppendMethod, METH_O,
+    "Appends a message to the end of the repeated container."},
+  { "insert", Insert, METH_VARARGS,
+    "Inserts a message before the specified index." },
   { "extend", ExtendMethod, METH_O,
     "Adds objects to the repeated container." },
   { "pop", Pop, METH_VARARGS,

src/Makefile.am

@@ -200,6 +200,7 @@ libprotobuf_lite_la_SOURCES =                                  \
   google/protobuf/stubs/strutil.cc                             \
   google/protobuf/stubs/time.cc                                \
   google/protobuf/stubs/time.h                                 \
+  google/protobuf/any_lite.cc                                  \
   google/protobuf/arena.cc                                     \
   google/protobuf/extension_set.cc                             \
   google/protobuf/generated_message_util.cc                    \

src/google/protobuf/any.cc

@@ -30,79 +30,35 @@
 
 #include <google/protobuf/any.h>
 
+#include <google/protobuf/arenastring.h>
+#include <google/protobuf/descriptor.h>
 #include <google/protobuf/generated_message_util.h>
-
+#include <google/protobuf/message.h>
 
 namespace google {
 namespace protobuf {
 namespace internal {
 
-namespace {
-string GetTypeUrl(const Descriptor* message,
-                  const string& type_url_prefix) {
-  if (!type_url_prefix.empty() &&
-      type_url_prefix[type_url_prefix.size() - 1] == '/') {
-    return type_url_prefix + message->full_name();
-  } else {
-    return type_url_prefix + "/" + message->full_name();
-  }
-}
-}  // namespace
-
-const char kAnyFullTypeName[] = "google.protobuf.Any";
-const char kTypeGoogleApisComPrefix[] = "type.googleapis.com/";
-const char kTypeGoogleProdComPrefix[] = "type.googleprod.com/";
-
-AnyMetadata::AnyMetadata(UrlType* type_url, ValueType* value)
-    : type_url_(type_url), value_(value) {
-}
-
 void AnyMetadata::PackFrom(const Message& message) {
   PackFrom(message, kTypeGoogleApisComPrefix);
 }
 
 void AnyMetadata::PackFrom(const Message& message,
                            const string& type_url_prefix) {
-  type_url_->SetNoArena(&::google::protobuf::internal::GetEmptyString(),
-                        GetTypeUrl(message.GetDescriptor(), type_url_prefix));
+  type_url_->SetNoArena(
+      &::google::protobuf::internal::GetEmptyString(),
+      GetTypeUrl(message.GetDescriptor()->full_name(), type_url_prefix));
   message.SerializeToString(value_->MutableNoArena(
       &::google::protobuf::internal::GetEmptyStringAlreadyInited()));
 }
 
 bool AnyMetadata::UnpackTo(Message* message) const {
-  if (!InternalIs(message->GetDescriptor())) {
+  if (!InternalIs(message->GetDescriptor()->full_name())) {
     return false;
   }
   return message->ParseFromString(value_->GetNoArena());
 }
 
-bool AnyMetadata::InternalIs(const Descriptor* descriptor) const {
-  const string type_url = type_url_->GetNoArena();
-  string full_name;
-  if (!ParseAnyTypeUrl(type_url, &full_name)) {
-    return false;
-  }
-  return full_name == descriptor->full_name();
-}
-
-bool ParseAnyTypeUrl(const string& type_url, string* url_prefix,
-                     string* full_type_name) {
-  size_t pos = type_url.find_last_of("/");
-  if (pos == string::npos || pos + 1 == type_url.size()) {
-    return false;
-  }
-  if (url_prefix) {
-    *url_prefix = type_url.substr(0, pos + 1);
-  }
-  *full_type_name = type_url.substr(pos + 1);
-  return true;
-}
-
-bool ParseAnyTypeUrl(const string& type_url, string* full_type_name) {
-  return ParseAnyTypeUrl(type_url, NULL, full_type_name);
-}
-
-
 bool GetAnyFieldDescriptors(const Message& message,
                             const FieldDescriptor** type_url_field,
                             const FieldDescriptor** value_field) {

src/google/protobuf/any.h

@@ -34,16 +34,26 @@
 #include <string>
 
 #include <google/protobuf/stubs/common.h>
-#include <google/protobuf/descriptor.h>
-#include <google/protobuf/message.h>
 #include <google/protobuf/arenastring.h>
+#include <google/protobuf/message_lite.h>
 
 #include <google/protobuf/port_def.inc>
 
 namespace google {
 namespace protobuf {
+
+class FieldDescriptor;
+class Message;
+
 namespace internal {
 
+extern const char kAnyFullTypeName[];          // "google.protobuf.Any".
+extern const char kTypeGoogleApisComPrefix[];  // "type.googleapis.com/".
+extern const char kTypeGoogleProdComPrefix[];  // "type.googleprod.com/".
+
+std::string GetTypeUrl(StringPiece message_name,
+                  StringPiece type_url_prefix);
+
 // Helper class used to implement google::protobuf::Any.
 class PROTOBUF_EXPORT AnyMetadata {
   typedef ArenaStringPtr UrlType;
@@ -54,31 +64,52 @@ class PROTOBUF_EXPORT AnyMetadata {
 
   // Packs a message using the default type URL prefix: "type.googleapis.com".
   // The resulted type URL will be "type.googleapis.com/<message_full_name>".
+  template <typename T>
+  void PackFrom(const T& message) {
+    InternalPackFrom(message, kTypeGoogleApisComPrefix, T::FullMessageName());
+  }
+
   void PackFrom(const Message& message);
+
   // Packs a message using the given type URL prefix. The type URL will be
   // constructed by concatenating the message type's full name to the prefix
   // with an optional "/" separator if the prefix doesn't already end up "/".
   // For example, both PackFrom(message, "type.googleapis.com") and
   // PackFrom(message, "type.googleapis.com/") yield the same result type
   // URL: "type.googleapis.com/<message_full_name>".
+  template <typename T>
+  void PackFrom(const T& message, StringPiece type_url_prefix) {
+    InternalPackFrom(message, type_url_prefix, T::FullMessageName());
+  }
+
   void PackFrom(const Message& message, const std::string& type_url_prefix);
 
   // Unpacks the payload into the given message. Returns false if the message's
   // type doesn't match the type specified in the type URL (i.e., the full
   // name after the last "/" of the type URL doesn't match the message's actual
   // full name) or parsing the payload has failed.
+  template <typename T>
+  bool UnpackTo(T* message) const {
+    return InternalUnpackTo(T::FullMessageName(), message);
+  }
+
   bool UnpackTo(Message* message) const;
 
   // Checks whether the type specified in the type URL matches the given type.
   // A type is consdiered matching if its full name matches the full name after
   // the last "/" in the type URL.
-  template<typename T>
+  template <typename T>
   bool Is() const {
-    return InternalIs(T::default_instance().GetDescriptor());
+    return InternalIs(T::FullMessageName());
   }
 
  private:
-  bool InternalIs(const Descriptor* message) const;
+  void InternalPackFrom(const MessageLite& message,
+                        StringPiece type_url_prefix,
+                        StringPiece type_name);
+  bool InternalUnpackTo(StringPiece type_name,
+                        MessageLite* message) const;
+  bool InternalIs(StringPiece type_name) const;
 
   UrlType* type_url_;
   ValueType* value_;
@@ -86,10 +117,6 @@ class PROTOBUF_EXPORT AnyMetadata {
   GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(AnyMetadata);
 };
 
-extern const char kAnyFullTypeName[];          // "google.protobuf.Any".
-extern const char kTypeGoogleApisComPrefix[];  // "type.googleapis.com/".
-extern const char kTypeGoogleProdComPrefix[];  // "type.googleprod.com/".
-
 // Get the proto type name from Any::type_url value. For example, passing
 // "type.googleapis.com/rpc.QueryOrigin" will return "rpc.QueryOrigin" in
 // *full_type_name. Returns false if the type_url does not have a "/"

src/google/protobuf/any.pb.cc

@@ -42,9 +42,9 @@ void InitDefaults_google_2fprotobuf_2fany_2eproto() {
   ::google::protobuf::internal::InitSCC(&scc_info_Any_google_2fprotobuf_2fany_2eproto.base);
 }
 
-::google::protobuf::Metadata file_level_metadata_google_2fprotobuf_2fany_2eproto[1];
-constexpr ::google::protobuf::EnumDescriptor const** file_level_enum_descriptors_google_2fprotobuf_2fany_2eproto = nullptr;
-constexpr ::google::protobuf::ServiceDescriptor const** file_level_service_descriptors_google_2fprotobuf_2fany_2eproto = nullptr;
+static ::google::protobuf::Metadata file_level_metadata_google_2fprotobuf_2fany_2eproto[1];
+static constexpr ::google::protobuf::EnumDescriptor const** file_level_enum_descriptors_google_2fprotobuf_2fany_2eproto = nullptr;
+static constexpr ::google::protobuf::ServiceDescriptor const** file_level_service_descriptors_google_2fprotobuf_2fany_2eproto = nullptr;
 
 const ::google::protobuf::uint32 TableStruct_google_2fprotobuf_2fany_2eproto::offsets[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = {
   ~0u,  // no _has_bits_
@@ -63,7 +63,7 @@ static ::google::protobuf::Message const * const file_default_instances[] = {
   reinterpret_cast<const ::google::protobuf::Message*>(&::google::protobuf::_Any_default_instance_),
 };
 
-::google::protobuf::internal::AssignDescriptorsTable assign_descriptors_table_google_2fprotobuf_2fany_2eproto = {
+static ::google::protobuf::internal::AssignDescriptorsTable assign_descriptors_table_google_2fprotobuf_2fany_2eproto = {
   {}, AddDescriptors_google_2fprotobuf_2fany_2eproto, "google/protobuf/any.proto", schemas,
   file_default_instances, TableStruct_google_2fprotobuf_2fany_2eproto::offsets,
   file_level_metadata_google_2fprotobuf_2fany_2eproto, 1, file_level_enum_descriptors_google_2fprotobuf_2fany_2eproto, file_level_service_descriptors_google_2fprotobuf_2fany_2eproto,
@@ -77,7 +77,7 @@ const char descriptor_table_protodef_google_2fprotobuf_2fany_2eproto[] =
   "\003GPB\252\002\036Google.Protobuf.WellKnownTypesb\006p"
   "roto3"
   ;
-::google::protobuf::internal::DescriptorTable descriptor_table_google_2fprotobuf_2fany_2eproto = {
+static ::google::protobuf::internal::DescriptorTable descriptor_table_google_2fprotobuf_2fany_2eproto = {
   false, InitDefaults_google_2fprotobuf_2fany_2eproto, 
   descriptor_table_protodef_google_2fprotobuf_2fany_2eproto,
   "google/protobuf/any.proto", &assign_descriptors_table_google_2fprotobuf_2fany_2eproto, 205,
@@ -111,11 +111,6 @@ void Any::PackFrom(const ::google::protobuf::Message& message,
 bool Any::UnpackTo(::google::protobuf::Message* message) const {
   return _any_metadata_.UnpackTo(message);
 }
-bool Any::ParseAnyTypeUrl(const string& type_url,
-                                  string* full_type_name) {
-  return ::google::protobuf::internal::ParseAnyTypeUrl(type_url,
-                                             full_type_name);
-}
 bool Any::GetAnyFieldDescriptors(
     const ::google::protobuf::Message& message,
     const ::google::protobuf::FieldDescriptor** type_url_field,
@@ -123,6 +118,11 @@ bool Any::GetAnyFieldDescriptors(
   return ::google::protobuf::internal::GetAnyFieldDescriptors(
       message, type_url_field, value_field);
 }
+bool Any::ParseAnyTypeUrl(const string& type_url,
+                                  string* full_type_name) {
+  return ::google::protobuf::internal::ParseAnyTypeUrl(type_url,
+                                             full_type_name);
+}
 
 class Any::HasBitSetters {
  public:
@@ -192,71 +192,40 @@ void Any::Clear() {
 }
 
 #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
-const char* Any::_InternalParse(const char* begin, const char* end, void* object,
-                  ::google::protobuf::internal::ParseContext* ctx) {
-  auto msg = static_cast<Any*>(object);
-  ::google::protobuf::int32 size; (void)size;
-  int depth; (void)depth;
-  ::google::protobuf::uint32 tag;
-  ::google::protobuf::internal::ParseFunc parser_till_end; (void)parser_till_end;
-  auto ptr = begin;
-  while (ptr < end) {
-    ptr = ::google::protobuf::io::Parse32(ptr, &tag);
+const char* Any::_InternalParse(const char* ptr, ::google::protobuf::internal::ParseContext* ctx) {
+  while (!ctx->Done(&ptr)) {
+    ::google::protobuf::uint32 tag;
+    ptr = ::google::protobuf::internal::ReadTag(ptr, &tag);
     GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
     switch (tag >> 3) {
       // string type_url = 1;
       case 1: {
         if (static_cast<::google::protobuf::uint8>(tag) != 10) goto handle_unusual;
-        ptr = ::google::protobuf::io::ReadSize(ptr, &size);
+        ptr = ::google::protobuf::internal::InlineGreedyStringParserUTF8(mutable_type_url(), ptr, ctx, "google.protobuf.Any.type_url");
         GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
-        ctx->extra_parse_data().SetFieldName("google.protobuf.Any.type_url");
-        object = msg->mutable_type_url();
-        if (size > end - ptr + ::google::protobuf::internal::ParseContext::kSlopBytes) {
-          parser_till_end = ::google::protobuf::internal::GreedyStringParserUTF8;
-          goto string_till_end;
-        }
-        GOOGLE_PROTOBUF_PARSER_ASSERT(::google::protobuf::internal::StringCheckUTF8(ptr, size, ctx));
-        ::google::protobuf::internal::InlineGreedyStringParser(object, ptr, size, ctx);
-        ptr += size;
         break;
       }
       // bytes value = 2;
       case 2: {
         if (static_cast<::google::protobuf::uint8>(tag) != 18) goto handle_unusual;
-        ptr = ::google::protobuf::io::ReadSize(ptr, &size);
+        ptr = ::google::protobuf::internal::InlineGreedyStringParser(mutable_value(), ptr, ctx);
         GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
-        object = msg->mutable_value();
-        if (size > end - ptr + ::google::protobuf::internal::ParseContext::kSlopBytes) {
-          parser_till_end = ::google::protobuf::internal::GreedyStringParser;
-          goto string_till_end;
-        }
-        GOOGLE_PROTOBUF_PARSER_ASSERT(::google::protobuf::internal::StringCheck(ptr, size, ctx));
-        ::google::protobuf::internal::InlineGreedyStringParser(object, ptr, size, ctx);
-        ptr += size;
         break;
       }
       default: {
       handle_unusual:
         if ((tag & 7) == 4 || tag == 0) {
-          ctx->EndGroup(tag);
+          ctx->SetLastTag(tag);
           return ptr;
         }
-        auto res = UnknownFieldParse(tag, {_InternalParse, msg},
-          ptr, end, msg->_internal_metadata_.mutable_unknown_fields(), ctx);
-        ptr = res.first;
+        ptr = UnknownFieldParse(tag,
+          _internal_metadata_.mutable_unknown_fields(), ptr, ctx);
         GOOGLE_PROTOBUF_PARSER_ASSERT(ptr != nullptr);
-        if (res.second) return ptr;
+        break;
       }
     }  // switch
   }  // while
   return ptr;
-string_till_end:
-  static_cast<::std::string*>(object)->clear();
-  static_cast<::std::string*>(object)->reserve(size);
-  goto len_delim_till_end;
-len_delim_till_end:
-  return ctx->StoreAndTailCall(ptr, end, {_InternalParse, msg},
-                               {parser_till_end, object}, size);
 }
 #else  // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
 bool Any::MergePartialFromCodedStream(

src/google/protobuf/any.pb.h

@@ -108,15 +108,15 @@ class PROTOBUF_EXPORT Any final :
   void PackFrom(const ::google::protobuf::Message& message,
                 const ::std::string& type_url_prefix);
   bool UnpackTo(::google::protobuf::Message* message) const;
+  static bool GetAnyFieldDescriptors(
+      const ::google::protobuf::Message& message,
+      const ::google::protobuf::FieldDescriptor** type_url_field,
+      const ::google::protobuf::FieldDescriptor** value_field);
   template<typename T> bool Is() const {
     return _any_metadata_.Is<T>();
   }
   static bool ParseAnyTypeUrl(const string& type_url,
                               string* full_type_name);
-  static bool GetAnyFieldDescriptors(
-      const ::google::protobuf::Message& message,
-      const ::google::protobuf::FieldDescriptor** type_url_field,
-      const ::google::protobuf::FieldDescriptor** value_field);
   void Swap(Any* other);
   friend void swap(Any& a, Any& b) {
     a.Swap(&b);
@@ -140,8 +140,7 @@ class PROTOBUF_EXPORT Any final :
 
   size_t ByteSizeLong() const final;
   #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
-  static const char* _InternalParse(const char* begin, const char* end, void* object, ::google::protobuf::internal::ParseContext* ctx);
-  ::google::protobuf::internal::ParseFunc _ParseFunc() const final { return _InternalParse; }
+  const char* _InternalParse(const char* ptr, ::google::protobuf::internal::ParseContext* ctx) final;
   #else
   bool MergePartialFromCodedStream(
       ::google::protobuf::io::CodedInputStream* input) final;
@@ -153,10 +152,14 @@ class PROTOBUF_EXPORT Any final :
   int GetCachedSize() const final { return _cached_size_.Get(); }
 
   private:
-  void SharedCtor();
-  void SharedDtor();
+  inline void SharedCtor();
+  inline void SharedDtor();
   void SetCachedSize(int size) const final;
   void InternalSwap(Any* other);
+  friend class ::google::protobuf::internal::AnyMetadata;
+  static ::google::protobuf::StringPiece FullMessageName() {
+    return "google.protobuf.Any";
+  }
   private:
   inline ::google::protobuf::Arena* GetArenaNoVirtual() const {
     return nullptr;

src/google/protobuf/any_lite.cc

@@ -0,0 +1,123 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <google/protobuf/any.h>
+
+#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
+#include <google/protobuf/arenastring.h>
+#include <google/protobuf/generated_message_util.h>
+#include <google/protobuf/stubs/strutil.h>
+
+
+namespace google {
+namespace protobuf {
+namespace internal {
+
+string GetTypeUrl(StringPiece message_name,
+                  StringPiece type_url_prefix) {
+  if (!type_url_prefix.empty() &&
+      type_url_prefix[type_url_prefix.size() - 1] == '/') {
+    return StrCat(type_url_prefix, message_name);
+  } else {
+    return StrCat(type_url_prefix, "/", message_name);
+  }
+}
+
+const char kAnyFullTypeName[] = "google.protobuf.Any";
+const char kTypeGoogleApisComPrefix[] = "type.googleapis.com/";
+const char kTypeGoogleProdComPrefix[] = "type.googleprod.com/";
+
+AnyMetadata::AnyMetadata(UrlType* type_url, ValueType* value)
+    : type_url_(type_url), value_(value) {}
+
+void AnyMetadata::InternalPackFrom(const MessageLite& message,
+                                   StringPiece type_url_prefix,
+                                   StringPiece type_name) {
+  type_url_->SetNoArena(&::google::protobuf::internal::GetEmptyString(),
+                        GetTypeUrl(type_name, type_url_prefix));
+  message.SerializeToString(value_->MutableNoArena(
+      &::google::protobuf::internal::GetEmptyStringAlreadyInited()));
+}
+
+bool AnyMetadata::InternalUnpackTo(StringPiece type_name,
+                                   MessageLite* message) const {
+  if (!InternalIs(type_name)) {
+    return false;
+  }
+  return message->ParseFromString(value_->GetNoArena());
+}
+
+namespace {
+
+// The type URL could be stored in either an ArenaStringPtr or a
+// StringPieceField, so we provide these helpers to get a string_view from
+// either type. We use a template function as a way to avoid depending on
+// StringPieceField.
+
+template <typename T>
+StringPiece Get(const T* ptr) {
+  return ptr->Get();
+}
+
+template <>
+// NOLINTNEXTLINE: clang-diagnostic-unused-function
+StringPiece Get(const ArenaStringPtr* ptr) {
+  return ptr->GetNoArena();
+}
+
+}  // namespace
+
+bool AnyMetadata::InternalIs(StringPiece type_name) const {
+  StringPiece type_url = Get(type_url_);
+  return type_url.size() >= type_name.size() + 1 &&
+         type_url[type_url.size() - type_name.size() - 1] == '/' &&
+         HasSuffixString(type_url, type_name);
+}
+
+bool ParseAnyTypeUrl(const string& type_url, string* url_prefix,
+                     string* full_type_name) {
+  size_t pos = type_url.find_last_of("/");
+  if (pos == string::npos || pos + 1 == type_url.size()) {
+    return false;
+  }
+  if (url_prefix) {
+    *url_prefix = type_url.substr(0, pos + 1);
+  }
+  *full_type_name = type_url.substr(pos + 1);
+  return true;
+}
+
+bool ParseAnyTypeUrl(const string& type_url, string* full_type_name) {
+  return ParseAnyTypeUrl(type_url, nullptr, full_type_name);
+}
+
+}  // namespace internal
+}  // namespace protobuf
+}  // namespace google

src/google/protobuf/any_test.cc

@@ -29,9 +29,11 @@
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <google/protobuf/any_test.pb.h>
+#include <google/protobuf/unittest.pb.h>
 #include <gtest/gtest.h>
 
 
+
 namespace google {
 namespace protobuf {
 namespace {
@@ -46,10 +48,22 @@ TEST(AnyTest, TestPackAndUnpack) {
 
   ASSERT_TRUE(message.ParseFromString(data));
   EXPECT_TRUE(message.has_any_value());
+  submessage.Clear();
   ASSERT_TRUE(message.any_value().UnpackTo(&submessage));
   EXPECT_EQ(12345, submessage.int32_value());
 }
 
+TEST(AnyTest, TestUnpackWithTypeMismatch) {
+  protobuf_unittest::TestAny payload;
+  payload.set_int32_value(13);
+  google::protobuf::Any any;
+  any.PackFrom(payload);
+
+  // Attempt to unpack into the wrong type.
+  protobuf_unittest::TestAllTypes dest;
+  EXPECT_FALSE(any.UnpackTo(&dest));
+}
+
 TEST(AnyTest, TestPackAndUnpackAny) {
   // We can pack a Any message inside another Any message.
   protobuf_unittest::TestAny submessage;
@@ -63,26 +77,26 @@ TEST(AnyTest, TestPackAndUnpackAny) {
 
   ASSERT_TRUE(message.ParseFromString(data));
   EXPECT_TRUE(message.has_any_value());
+  any.Clear();
+  submessage.Clear();
   ASSERT_TRUE(message.any_value().UnpackTo(&any));
   ASSERT_TRUE(any.UnpackTo(&submessage));
   EXPECT_EQ(12345, submessage.int32_value());
 }
 
-TEST(AnyType, TestPackWithCustomTypeUrl) {
+TEST(AnyTest, TestPackWithCustomTypeUrl) {
   protobuf_unittest::TestAny submessage;
   submessage.set_int32_value(12345);
   google::protobuf::Any any;
   // Pack with a custom type URL prefix.
   any.PackFrom(submessage, "type.myservice.com");
-  EXPECT_EQ("type.myservice.com/" + submessage.GetDescriptor()->full_name(),
-            any.type_url());
+  EXPECT_EQ("type.myservice.com/protobuf_unittest.TestAny", any.type_url());
   // Pack with a custom type URL prefix ending with '/'.
   any.PackFrom(submessage, "type.myservice.com/");
-  EXPECT_EQ("type.myservice.com/" + submessage.GetDescriptor()->full_name(),
-            any.type_url());
+  EXPECT_EQ("type.myservice.com/protobuf_unittest.TestAny", any.type_url());
   // Pack with an empty type URL prefix.
   any.PackFrom(submessage, "");
-  EXPECT_EQ("/" + submessage.GetDescriptor()->full_name(), any.type_url());
+  EXPECT_EQ("/protobuf_unittest.TestAny", any.type_url());
 
   // Test unpacking the type.
   submessage.Clear();
@@ -104,6 +118,15 @@ TEST(AnyTest, TestIs) {
   ASSERT_TRUE(message.ParseFromString(message.SerializeAsString()));
   EXPECT_FALSE(message.any_value().Is<protobuf_unittest::TestAny>());
   EXPECT_TRUE(message.any_value().Is<google::protobuf::Any>());
+
+  any.set_type_url("/protobuf_unittest.TestAny");
+  EXPECT_TRUE(any.Is<protobuf_unittest::TestAny>());
+  // The type URL must contain at least one "/".
+  any.set_type_url("protobuf_unittest.TestAny");
+  EXPECT_FALSE(any.Is<protobuf_unittest::TestAny>());
+  // The type name after the slash must be fully qualified.
+  any.set_type_url("/TestAny");
+  EXPECT_FALSE(any.Is<protobuf_unittest::TestAny>());
 }
 
 TEST(AnyTest, MoveConstructor) {
@@ -117,6 +140,7 @@ TEST(AnyTest, MoveConstructor) {
 
   google::protobuf::Any dst(std::move(src));
   EXPECT_EQ(type_url, dst.type_url().data());
+  payload.Clear();
   ASSERT_TRUE(dst.UnpackTo(&payload));
   EXPECT_EQ(12345, payload.int32_value());
 }
@@ -133,6 +157,7 @@ TEST(AnyTest, MoveAssignment) {
   google::protobuf::Any dst;
   dst = std::move(src);
   EXPECT_EQ(type_url, dst.type_url().data());
+  payload.Clear();
   ASSERT_TRUE(dst.UnpackTo(&payload));
   EXPECT_EQ(12345, payload.int32_value());
 }

src/google/protobuf/api.pb.cc

@@ -89,9 +89,9 @@ void InitDefaults_google_2fprotobuf_2fapi_2eproto() {
   ::google::protobuf::internal::InitSCC(&scc_info_Mixin_google_2fprotobuf_2fapi_2eproto.base);
 }
 
-::google::protobuf::Metadata file_level_metadata_google_2fprotobuf_2fapi_2eproto[3];
-constexpr ::google::protobuf::EnumDescriptor const** file_level_enum_descriptors_google_2fprotobuf_2fapi_2eproto = nullptr;
-constexpr ::google::protobuf::ServiceDescriptor const** file_level_service_descriptors_google_2fprotobuf_2fapi_2eproto = nullptr;
+static ::google::protobuf::Metadata file_level_metadata_google_2fprotobuf_2fapi_2eproto[3];
+static constexpr ::google::protobuf::EnumDescriptor const** file_level_enum_descriptors_google_2fprotobuf_2fapi_2eproto = nullptr;
+static constexpr ::google::protobuf::ServiceDescriptor const** file_level_service_descriptors_google_2fprotobuf_2fapi_2eproto = nullptr;
 
 const ::google::protobuf::uint32 TableStruct_google_2fprotobuf_2fapi_2eproto::offsets[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = {
   ~0u,  // no _has_bits_
@@ -138,7 +138,7 @@ static ::google::protobuf::Message const * const file_default_instances[] = {
   reinterpret_cast<const ::google::protobuf::Message*>(&::google::protobuf::_Mixin_default_instance_),
 };
 
-::google::protobuf::internal::AssignDescriptorsTable assign_descriptors_table_google_2fprotobuf_2fapi_2eproto = {
+static ::google::protobuf::internal::AssignDescriptorsTable assign_descriptors_table_google_2fprotobuf_2fapi_2eproto = {
   {}, AddDescriptors_google_2fprotobuf_2fapi_2eproto, "google/protobuf/api.proto", schemas,
   file_default_instances, TableStruct_google_2fprotobuf_2fapi_2eproto::offsets,
   file_level_metadata_google_2fprotobuf_2fapi_2eproto, 3, file_level_enum_descriptors_google_2fprotobuf_2fapi_2eproto, file_level_service_descriptors_google_2fprotobuf_2fapi_2eproto,
@@ -165,7 +165,7 @@ const char descriptor_table_protodef_google_2fprotobuf_2fapi_2eproto[] =
   "nproto/protobuf/api;api\242\002\003GPB\252\002\036Google.P"
   "rotobuf.WellKnownTypesb\006proto3"
   ;
-::google::protobuf::internal::DescriptorTable descriptor_table_google_2fprotobuf_2fapi_2eproto = {
+static ::google::protobuf::internal::DescriptorTable descriptor_table_google_2fprotobuf_2fapi_2eproto = {
   false, InitDefaults_google_2fprotobuf_2fapi_2eproto, 
   descriptor_table_protodef_google_2fprotobuf_2fapi_2eproto,
   "google/protobuf/api.proto", &assign_descriptors_table_google_2fprotobuf_2fapi_2eproto, 750,
@@ -298,141 +298,85 @@ void Api::Clear() {
 }
 
 #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
-const char* Api::_InternalParse(const char* begin, const char* end, void* object,
-                  ::google::protobuf::internal::ParseContext* ctx) {
-  auto msg = static_cast<Api*>(object);
-  ::google::protobuf::int32 size; (void)size;
-  int depth; (void)depth;
-  ::google::protobuf::uint32 tag;
-  ::google::protobuf::internal::ParseFunc parser_till_end; (void)parser_till_end;
-  auto ptr = begin;
-  while (ptr < end) {
-    ptr = ::google::protobuf::io::Parse32(ptr, &tag);
+const char* Api::_InternalParse(const char* ptr, ::google::protobuf::internal::ParseContext* ctx) {
+  while (!ctx->Done(&ptr)) {
+    ::google::protobuf::uint32 tag;
+    ptr = ::google::protobuf::internal::ReadTag(ptr, &tag);
     GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
     switch (tag >> 3) {
       // string name = 1;
       case 1: {
         if (static_cast<::google::protobuf::uint8>(tag) != 10) goto handle_unusual;
-        ptr = ::google::protobuf::io::ReadSize(ptr, &size);
+        ptr = ::google::protobuf::internal::InlineGreedyStringParserUTF8(mutable_name(), ptr, ctx, "google.protobuf.Api.name");
         GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
-        ctx->extra_parse_data().SetFieldName("google.protobuf.Api.name");
-        object = msg->mutable_name();
-        if (size > end - ptr + ::google::protobuf::internal::ParseContext::kSlopBytes) {
-          parser_till_end = ::google::protobuf::internal::GreedyStringParserUTF8;
-          goto string_till_end;
-        }
-        GOOGLE_PROTOBUF_PARSER_ASSERT(::google::protobuf::internal::StringCheckUTF8(ptr, size, ctx));
-        ::google::protobuf::internal::InlineGreedyStringParser(object, ptr, size, ctx);
-        ptr += size;
         break;
       }
       // repeated .google.protobuf.Method methods = 2;
       case 2: {
         if (static_cast<::google::protobuf::uint8>(tag) != 18) goto handle_unusual;
         do {
-          ptr = ::google::protobuf::io::ReadSize(ptr, &size);
+          ptr = ctx->ParseMessage(add_methods(), ptr);
           GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
-          parser_till_end = ::google::protobuf::Method::_InternalParse;
-          object = msg->add_methods();
-          if (size > end - ptr) goto len_delim_till_end;
-          ptr += size;
-          GOOGLE_PROTOBUF_PARSER_ASSERT(ctx->ParseExactRange(
-              {parser_till_end, object}, ptr - size, ptr));
-          if (ptr >= end) break;
-        } while ((::google::protobuf::io::UnalignedLoad<::google::protobuf::uint64>(ptr) & 255) == 18 && (ptr += 1));
+          if (ctx->Done(&ptr)) return ptr;
+        } while ((::google::protobuf::internal::UnalignedLoad<::google::protobuf::uint64>(ptr) & 255) == 18 && (ptr += 1));
         break;
       }
       // repeated .google.protobuf.Option options = 3;
       case 3: {
         if (static_cast<::google::protobuf::uint8>(tag) != 26) goto handle_unusual;
         do {
-          ptr = ::google::protobuf::io::ReadSize(ptr, &size);
+          ptr = ctx->ParseMessage(add_options(), ptr);
           GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
-          parser_till_end = ::google::protobuf::Option::_InternalParse;
-          object = msg->add_options();
-          if (size > end - ptr) goto len_delim_till_end;
-          ptr += size;
-          GOOGLE_PROTOBUF_PARSER_ASSERT(ctx->ParseExactRange(
-              {parser_till_end, object}, ptr - size, ptr));
-          if (ptr >= end) break;
-        } while ((::google::protobuf::io::UnalignedLoad<::google::protobuf::uint64>(ptr) & 255) == 26 && (ptr += 1));
+          if (ctx->Done(&ptr)) return ptr;
+        } while ((::google::protobuf::internal::UnalignedLoad<::google::protobuf::uint64>(ptr) & 255) == 26 && (ptr += 1));
         break;
       }
       // string version = 4;
       case 4: {
         if (static_cast<::google::protobuf::uint8>(tag) != 34) goto handle_unusual;
-        ptr = ::google::protobuf::io::ReadSize(ptr, &size);
+        ptr = ::google::protobuf::internal::InlineGreedyStringParserUTF8(mutable_version(), ptr, ctx, "google.protobuf.Api.version");
         GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
-        ctx->extra_parse_data().SetFieldName("google.protobuf.Api.version");
-        object = msg->mutable_version();
-        if (size > end - ptr + ::google::protobuf::internal::ParseContext::kSlopBytes) {
-          parser_till_end = ::google::protobuf::internal::GreedyStringParserUTF8;
-          goto string_till_end;
-        }
-        GOOGLE_PROTOBUF_PARSER_ASSERT(::google::protobuf::internal::StringCheckUTF8(ptr, size, ctx));
-        ::google::protobuf::internal::InlineGreedyStringParser(object, ptr, size, ctx);
-        ptr += size;
         break;
       }
       // .google.protobuf.SourceContext source_context = 5;
       case 5: {
         if (static_cast<::google::protobuf::uint8>(tag) != 42) goto handle_unusual;
-        ptr = ::google::protobuf::io::ReadSize(ptr, &size);
+        ptr = ctx->ParseMessage(mutable_source_context(), ptr);
         GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
-        parser_till_end = ::google::protobuf::SourceContext::_InternalParse;
-        object = msg->mutable_source_context();
-        if (size > end - ptr) goto len_delim_till_end;
-        ptr += size;
-        GOOGLE_PROTOBUF_PARSER_ASSERT(ctx->ParseExactRange(
-            {parser_till_end, object}, ptr - size, ptr));
         break;
       }
       // repeated .google.protobuf.Mixin mixins = 6;
       case 6: {
         if (static_cast<::google::protobuf::uint8>(tag) != 50) goto handle_unusual;
         do {
-          ptr = ::google::protobuf::io::ReadSize(ptr, &size);
+          ptr = ctx->ParseMessage(add_mixins(), ptr);
           GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
-          parser_till_end = ::google::protobuf::Mixin::_InternalParse;
-          object = msg->add_mixins();
-          if (size > end - ptr) goto len_delim_till_end;
-          ptr += size;
-          GOOGLE_PROTOBUF_PARSER_ASSERT(ctx->ParseExactRange(
-              {parser_till_end, object}, ptr - size, ptr));
-          if (ptr >= end) break;
-        } while ((::google::protobuf::io::UnalignedLoad<::google::protobuf::uint64>(ptr) & 255) == 50 && (ptr += 1));
+          if (ctx->Done(&ptr)) return ptr;
+        } while ((::google::protobuf::internal::UnalignedLoad<::google::protobuf::uint64>(ptr) & 255) == 50 && (ptr += 1));
         break;
       }
       // .google.protobuf.Syntax syntax = 7;
       case 7: {
         if (static_cast<::google::protobuf::uint8>(tag) != 56) goto handle_unusual;
         ::google::protobuf::uint64 val = ::google::protobuf::internal::ReadVarint(&ptr);
-        msg->set_syntax(static_cast<::google::protobuf::Syntax>(val));
         GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
+        set_syntax(static_cast<::google::protobuf::Syntax>(val));
         break;
       }
       default: {
       handle_unusual:
         if ((tag & 7) == 4 || tag == 0) {
-          ctx->EndGroup(tag);
+          ctx->SetLastTag(tag);
           return ptr;
         }
-        auto res = UnknownFieldParse(tag, {_InternalParse, msg},
-          ptr, end, msg->_internal_metadata_.mutable_unknown_fields(), ctx);
-        ptr = res.first;
+        ptr = UnknownFieldParse(tag,
+          _internal_metadata_.mutable_unknown_fields(), ptr, ctx);
         GOOGLE_PROTOBUF_PARSER_ASSERT(ptr != nullptr);
-        if (res.second) return ptr;
+        break;
       }
     }  // switch
   }  // while
   return ptr;
-string_till_end:
-  static_cast<::std::string*>(object)->clear();
-  static_cast<::std::string*>(object)->reserve(size);
-  goto len_delim_till_end;
-len_delim_till_end:
-  return ctx->StoreAndTailCall(ptr, end, {_InternalParse, msg},
-                               {parser_till_end, object}, size);
 }
 #else  // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
 bool Api::MergePartialFromCodedStream(
@@ -957,77 +901,44 @@ void Method::Clear() {
 }
 
 #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
-const char* Method::_InternalParse(const char* begin, const char* end, void* object,
-                  ::google::protobuf::internal::ParseContext* ctx) {
-  auto msg = static_cast<Method*>(object);
-  ::google::protobuf::int32 size; (void)size;
-  int depth; (void)depth;
-  ::google::protobuf::uint32 tag;
-  ::google::protobuf::internal::ParseFunc parser_till_end; (void)parser_till_end;
-  auto ptr = begin;
-  while (ptr < end) {
-    ptr = ::google::protobuf::io::Parse32(ptr, &tag);
+const char* Method::_InternalParse(const char* ptr, ::google::protobuf::internal::ParseContext* ctx) {
+  while (!ctx->Done(&ptr)) {
+    ::google::protobuf::uint32 tag;
+    ptr = ::google::protobuf::internal::ReadTag(ptr, &tag);
     GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
     switch (tag >> 3) {
       // string name = 1;
       case 1: {
         if (static_cast<::google::protobuf::uint8>(tag) != 10) goto handle_unusual;
-        ptr = ::google::protobuf::io::ReadSize(ptr, &size);
+        ptr = ::google::protobuf::internal::InlineGreedyStringParserUTF8(mutable_name(), ptr, ctx, "google.protobuf.Method.name");
         GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
-        ctx->extra_parse_data().SetFieldName("google.protobuf.Method.name");
-        object = msg->mutable_name();
-        if (size > end - ptr + ::google::protobuf::internal::ParseContext::kSlopBytes) {
-          parser_till_end = ::google::protobuf::internal::GreedyStringParserUTF8;
-          goto string_till_end;
-        }
-        GOOGLE_PROTOBUF_PARSER_ASSERT(::google::protobuf::internal::StringCheckUTF8(ptr, size, ctx));
-        ::google::protobuf::internal::InlineGreedyStringParser(object, ptr, size, ctx);
-        ptr += size;
         break;
       }
       // string request_type_url = 2;
       case 2: {
         if (static_cast<::google::protobuf::uint8>(tag) != 18) goto handle_unusual;
-        ptr = ::google::protobuf::io::ReadSize(ptr, &size);
+        ptr = ::google::protobuf::internal::InlineGreedyStringParserUTF8(mutable_request_type_url(), ptr, ctx, "google.protobuf.Method.request_type_url");
         GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
-        ctx->extra_parse_data().SetFieldName("google.protobuf.Method.request_type_url");
-        object = msg->mutable_request_type_url();
-        if (size > end - ptr + ::google::protobuf::internal::ParseContext::kSlopBytes) {
-          parser_till_end = ::google::protobuf::internal::GreedyStringParserUTF8;
-          goto string_till_end;
-        }
-        GOOGLE_PROTOBUF_PARSER_ASSERT(::google::protobuf::internal::StringCheckUTF8(ptr, size, ctx));
-        ::google::protobuf::internal::InlineGreedyStringParser(object, ptr, size, ctx);
-        ptr += size;
         break;
       }
       // bool request_streaming = 3;
       case 3: {
         if (static_cast<::google::protobuf::uint8>(tag) != 24) goto handle_unusual;
-        msg->set_request_streaming(::google::protobuf::internal::ReadVarint(&ptr));
+        set_request_streaming(::google::protobuf::internal::ReadVarint(&ptr));
         GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
         break;
       }
       // string response_type_url = 4;
       case 4: {
         if (static_cast<::google::protobuf::uint8>(tag) != 34) goto handle_unusual;
-        ptr = ::google::protobuf::io::ReadSize(ptr, &size);
+        ptr = ::google::protobuf::internal::InlineGreedyStringParserUTF8(mutable_response_type_url(), ptr, ctx, "google.protobuf.Method.response_type_url");
         GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
-        ctx->extra_parse_data().SetFieldName("google.protobuf.Method.response_type_url");
-        object = msg->mutable_response_type_url();
-        if (size > end - ptr + ::google::protobuf::internal::ParseContext::kSlopBytes) {
-          parser_till_end = ::google::protobuf::internal::GreedyStringParserUTF8;
-          goto string_till_end;
-        }
-        GOOGLE_PROTOBUF_PARSER_ASSERT(::google::protobuf::internal::StringCheckUTF8(ptr, size, ctx));
-        ::google::protobuf::internal::InlineGreedyStringParser(object, ptr, size, ctx);
-        ptr += size;
         break;
       }
       // bool response_streaming = 5;
       case 5: {
         if (static_cast<::google::protobuf::uint8>(tag) != 40) goto handle_unusual;
-        msg->set_response_streaming(::google::protobuf::internal::ReadVarint(&ptr));
+        set_response_streaming(::google::protobuf::internal::ReadVarint(&ptr));
         GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
         break;
       }
@@ -1035,48 +946,34 @@ const char* Method::_InternalParse(const char* begin, const char* end, void* obj
       case 6: {
         if (static_cast<::google::protobuf::uint8>(tag) != 50) goto handle_unusual;
         do {
-          ptr = ::google::protobuf::io::ReadSize(ptr, &size);
+          ptr = ctx->ParseMessage(add_options(), ptr);
           GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
-          parser_till_end = ::google::protobuf::Option::_InternalParse;
-          object = msg->add_options();
-          if (size > end - ptr) goto len_delim_till_end;
-          ptr += size;
-          GOOGLE_PROTOBUF_PARSER_ASSERT(ctx->ParseExactRange(
-              {parser_till_end, object}, ptr - size, ptr));
-          if (ptr >= end) break;
-        } while ((::google::protobuf::io::UnalignedLoad<::google::protobuf::uint64>(ptr) & 255) == 50 && (ptr += 1));
+          if (ctx->Done(&ptr)) return ptr;
+        } while ((::google::protobuf::internal::UnalignedLoad<::google::protobuf::uint64>(ptr) & 255) == 50 && (ptr += 1));
         break;
       }
       // .google.protobuf.Syntax syntax = 7;
       case 7: {
         if (static_cast<::google::protobuf::uint8>(tag) != 56) goto handle_unusual;
         ::google::protobuf::uint64 val = ::google::protobuf::internal::ReadVarint(&ptr);
-        msg->set_syntax(static_cast<::google::protobuf::Syntax>(val));
         GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
+        set_syntax(static_cast<::google::protobuf::Syntax>(val));
         break;
       }
       default: {
       handle_unusual:
         if ((tag & 7) == 4 || tag == 0) {
-          ctx->EndGroup(tag);
+          ctx->SetLastTag(tag);
           return ptr;
         }
-        auto res = UnknownFieldParse(tag, {_InternalParse, msg},
-          ptr, end, msg->_internal_metadata_.mutable_unknown_fields(), ctx);
-        ptr = res.first;
+        ptr = UnknownFieldParse(tag,
+          _internal_metadata_.mutable_unknown_fields(), ptr, ctx);
         GOOGLE_PROTOBUF_PARSER_ASSERT(ptr != nullptr);
-        if (res.second) return ptr;
+        break;
       }
     }  // switch
   }  // while
   return ptr;
-string_till_end:
-  static_cast<::std::string*>(object)->clear();
-  static_cast<::std::string*>(object)->reserve(size);
-  goto len_delim_till_end;
-len_delim_till_end:
-  return ctx->StoreAndTailCall(ptr, end, {_InternalParse, msg},
-                               {parser_till_end, object}, size);
 }
 #else  // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
 bool Method::MergePartialFromCodedStream(
@@ -1571,72 +1468,40 @@ void Mixin::Clear() {
 }
 
 #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
-const char* Mixin::_InternalParse(const char* begin, const char* end, void* object,
-                  ::google::protobuf::internal::ParseContext* ctx) {
-  auto msg = static_cast<Mixin*>(object);
-  ::google::protobuf::int32 size; (void)size;
-  int depth; (void)depth;
-  ::google::protobuf::uint32 tag;
-  ::google::protobuf::internal::ParseFunc parser_till_end; (void)parser_till_end;
-  auto ptr = begin;
-  while (ptr < end) {
-    ptr = ::google::protobuf::io::Parse32(ptr, &tag);
+const char* Mixin::_InternalParse(const char* ptr, ::google::protobuf::internal::ParseContext* ctx) {
+  while (!ctx->Done(&ptr)) {
+    ::google::protobuf::uint32 tag;
+    ptr = ::google::protobuf::internal::ReadTag(ptr, &tag);
     GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
     switch (tag >> 3) {
       // string name = 1;
       case 1: {
         if (static_cast<::google::protobuf::uint8>(tag) != 10) goto handle_unusual;
-        ptr = ::google::protobuf::io::ReadSize(ptr, &size);
+        ptr = ::google::protobuf::internal::InlineGreedyStringParserUTF8(mutable_name(), ptr, ctx, "google.protobuf.Mixin.name");
         GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
-        ctx->extra_parse_data().SetFieldName("google.protobuf.Mixin.name");
-        object = msg->mutable_name();
-        if (size > end - ptr + ::google::protobuf::internal::ParseContext::kSlopBytes) {
-          parser_till_end = ::google::protobuf::internal::GreedyStringParserUTF8;
-          goto string_till_end;
-        }
-        GOOGLE_PROTOBUF_PARSER_ASSERT(::google::protobuf::internal::StringCheckUTF8(ptr, size, ctx));
-        ::google::protobuf::internal::InlineGreedyStringParser(object, ptr, size, ctx);
-        ptr += size;
         break;
       }
       // string root = 2;
       case 2: {
         if (static_cast<::google::protobuf::uint8>(tag) != 18) goto handle_unusual;
-        ptr = ::google::protobuf::io::ReadSize(ptr, &size);
+        ptr = ::google::protobuf::internal::InlineGreedyStringParserUTF8(mutable_root(), ptr, ctx, "google.protobuf.Mixin.root");
         GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
-        ctx->extra_parse_data().SetFieldName("google.protobuf.Mixin.root");
-        object = msg->mutable_root();
-        if (size > end - ptr + ::google::protobuf::internal::ParseContext::kSlopBytes) {
-          parser_till_end = ::google::protobuf::internal::GreedyStringParserUTF8;
-          goto string_till_end;
-        }
-        GOOGLE_PROTOBUF_PARSER_ASSERT(::google::protobuf::internal::StringCheckUTF8(ptr, size, ctx));
-        ::google::protobuf::internal::InlineGreedyStringParser(object, ptr, size, ctx);
-        ptr += size;
         break;
       }
       default: {
       handle_unusual:
         if ((tag & 7) == 4 || tag == 0) {
-          ctx->EndGroup(tag);
+          ctx->SetLastTag(tag);
           return ptr;
         }
-        auto res = UnknownFieldParse(tag, {_InternalParse, msg},
-          ptr, end, msg->_internal_metadata_.mutable_unknown_fields(), ctx);
-        ptr = res.first;
+        ptr = UnknownFieldParse(tag,
+          _internal_metadata_.mutable_unknown_fields(), ptr, ctx);
         GOOGLE_PROTOBUF_PARSER_ASSERT(ptr != nullptr);
-        if (res.second) return ptr;
+        break;
       }
     }  // switch
   }  // while
   return ptr;
-string_till_end:
-  static_cast<::std::string*>(object)->clear();
-  static_cast<::std::string*>(object)->reserve(size);
-  goto len_delim_till_end;
-len_delim_till_end:
-  return ctx->StoreAndTailCall(ptr, end, {_InternalParse, msg},
-                               {parser_till_end, object}, size);
 }
 #else  // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
 bool Mixin::MergePartialFromCodedStream(

src/google/protobuf/api.pb.h

@@ -31,6 +31,13 @@
 #include <google/protobuf/repeated_field.h>  // IWYU pragma: export
 #include <google/protobuf/extension_set.h>  // IWYU pragma: export
 #include <google/protobuf/unknown_field_set.h>
+namespace google {
+namespace protobuf {
+namespace internal {
+class AnyMetadata;
+}  // namespace internal
+}  // namespace protobuf
+}  // namespace google
 #include <google/protobuf/source_context.pb.h>
 #include <google/protobuf/type.pb.h>
 // @@protoc_insertion_point(includes)
@@ -134,8 +141,7 @@ class PROTOBUF_EXPORT Api final :
 
   size_t ByteSizeLong() const final;
   #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
-  static const char* _InternalParse(const char* begin, const char* end, void* object, ::google::protobuf::internal::ParseContext* ctx);
-  ::google::protobuf::internal::ParseFunc _ParseFunc() const final { return _InternalParse; }
+  const char* _InternalParse(const char* ptr, ::google::protobuf::internal::ParseContext* ctx) final;
   #else
   bool MergePartialFromCodedStream(
       ::google::protobuf::io::CodedInputStream* input) final;
@@ -147,10 +153,14 @@ class PROTOBUF_EXPORT Api final :
   int GetCachedSize() const final { return _cached_size_.Get(); }
 
   private:
-  void SharedCtor();
-  void SharedDtor();
+  inline void SharedCtor();
+  inline void SharedDtor();
   void SetCachedSize(int size) const final;
   void InternalSwap(Api* other);
+  friend class ::google::protobuf::internal::AnyMetadata;
+  static ::google::protobuf::StringPiece FullMessageName() {
+    return "google.protobuf.Api";
+  }
   private:
   inline ::google::protobuf::Arena* GetArenaNoVirtual() const {
     return nullptr;
@@ -325,8 +335,7 @@ class PROTOBUF_EXPORT Method final :
 
   size_t ByteSizeLong() const final;
   #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
-  static const char* _InternalParse(const char* begin, const char* end, void* object, ::google::protobuf::internal::ParseContext* ctx);
-  ::google::protobuf::internal::ParseFunc _ParseFunc() const final { return _InternalParse; }
+  const char* _InternalParse(const char* ptr, ::google::protobuf::internal::ParseContext* ctx) final;
   #else
   bool MergePartialFromCodedStream(
       ::google::protobuf::io::CodedInputStream* input) final;
@@ -338,10 +347,14 @@ class PROTOBUF_EXPORT Method final :
   int GetCachedSize() const final { return _cached_size_.Get(); }
 
   private:
-  void SharedCtor();
-  void SharedDtor();
+  inline void SharedCtor();
+  inline void SharedDtor();
   void SetCachedSize(int size) const final;
   void InternalSwap(Method* other);
+  friend class ::google::protobuf::internal::AnyMetadata;
+  static ::google::protobuf::StringPiece FullMessageName() {
+    return "google.protobuf.Method";
+  }
   private:
   inline ::google::protobuf::Arena* GetArenaNoVirtual() const {
     return nullptr;
@@ -509,8 +522,7 @@ class PROTOBUF_EXPORT Mixin final :
 
   size_t ByteSizeLong() const final;
   #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
-  static const char* _InternalParse(const char* begin, const char* end, void* object, ::google::protobuf::internal::ParseContext* ctx);
-  ::google::protobuf::internal::ParseFunc _ParseFunc() const final { return _InternalParse; }
+  const char* _InternalParse(const char* ptr, ::google::protobuf::internal::ParseContext* ctx) final;
   #else
   bool MergePartialFromCodedStream(
       ::google::protobuf::io::CodedInputStream* input) final;
@@ -522,10 +534,14 @@ class PROTOBUF_EXPORT Mixin final :
   int GetCachedSize() const final { return _cached_size_.Get(); }
 
   private:
-  void SharedCtor();
-  void SharedDtor();
+  inline void SharedCtor();
+  inline void SharedDtor();
   void SetCachedSize(int size) const final;
   void InternalSwap(Mixin* other);
+  friend class ::google::protobuf::internal::AnyMetadata;
+  static ::google::protobuf::StringPiece FullMessageName() {
+    return "google.protobuf.Mixin";
+  }
   private:
   inline ::google::protobuf::Arena* GetArenaNoVirtual() const {
     return nullptr;

src/google/protobuf/arena.h

@@ -34,6 +34,7 @@
 #define GOOGLE_PROTOBUF_ARENA_H__
 
 #include <limits>
+#include <type_traits>
 #ifdef max
 #undef max  // Visual Studio defines this macro
 #endif
@@ -156,13 +157,14 @@ struct ArenaOptions {
  private:
   // Hooks for adding external functionality such as user-specific metrics
   // collection, specific debugging abilities, etc.
-  // Init hook may return a pointer to a cookie to be stored in the arena.
-  // reset and destruction hooks will then be called with the same cookie
-  // pointer. This allows us to save an external object per arena instance and
-  // use it on the other hooks (Note: It is just as legal for init to return
-  // NULL and not use the cookie feature).
-  // on_arena_reset and on_arena_destruction also receive the space used in
-  // the arena just before the reset.
+  // Init hook (if set) will always be called at Arena init time. Init hook may
+  // return a pointer to a cookie to be stored in the arena. Reset and
+  // destruction hooks will then be called with the same cookie pointer. This
+  // allows us to save an external object per arena instance and use it on the
+  // other hooks (Note: If init hook returns NULL, the other hooks will NOT be
+  // called on this arena instance).
+  // on_arena_reset and on_arena_destruction also receive the space used in the
+  // arena just before the reset.
   void* (*on_arena_init)(Arena* arena);
   void (*on_arena_reset)(Arena* arena, void* cookie, uint64 space_used);
   void (*on_arena_destruction)(Arena* arena, void* cookie, uint64 space_used);
@@ -408,12 +410,12 @@ class PROTOBUF_EXPORT Arena {
   }
 
   // Retrieves the arena associated with |value| if |value| is an arena-capable
-  // message, or NULL otherwise. This differs from value->GetArena() in that the
-  // latter is a virtual call, while this method is a templated call that
-  // resolves at compile-time.
+  // message, or NULL otherwise. If possible, the call resolves at compile time.
+  // Note that we can often devirtualize calls to `value->GetArena()` so usually
+  // calling this method is unnecessary.
   template <typename T>
   PROTOBUF_ALWAYS_INLINE static Arena* GetArena(const T* value) {
-    return GetArenaInternal(value, is_arena_constructable<T>());
+    return GetArenaInternal(value);
   }
 
   template <typename T>
@@ -440,6 +442,15 @@ class PROTOBUF_EXPORT Arena {
                                              sizeof(char)>
         is_arena_constructable;
 
+    template <typename U>
+    static char HasGetArena(decltype(&U::GetArena));
+    template <typename U>
+    static double HasGetArena(...);
+
+    typedef std::integral_constant<bool, sizeof(HasGetArena<T>(nullptr)) ==
+                                             sizeof(char)>
+        has_get_arena;
+
     template <typename... Args>
     static T* Construct(void* ptr, Args&&... args) {
       return new (ptr) T(std::forward<Args>(args)...);
@@ -655,16 +666,24 @@ class PROTOBUF_EXPORT Arena {
   // Implementation for GetArena(). Only message objects with
   // InternalArenaConstructable_ tags can be associated with an arena, and such
   // objects must implement a GetArenaNoVirtual() method.
-  template <typename T>
-  PROTOBUF_ALWAYS_INLINE static Arena* GetArenaInternal(const T* value,
-                                                        std::true_type) {
+  template <typename T, typename std::enable_if<
+                            is_arena_constructable<T>::value, int>::type = 0>
+  PROTOBUF_ALWAYS_INLINE static Arena* GetArenaInternal(const T* value) {
     return InternalHelper<T>::GetArena(value);
   }
-
-  template <typename T>
-  PROTOBUF_ALWAYS_INLINE static Arena* GetArenaInternal(const T* /* value */,
-                                                        std::false_type) {
-    return NULL;
+  template <typename T,
+            typename std::enable_if<!is_arena_constructable<T>::value &&
+                                        InternalHelper<T>::has_get_arena::value,
+                                    int>::type = 0>
+  PROTOBUF_ALWAYS_INLINE static Arena* GetArenaInternal(const T* value) {
+    return value->GetArena();
+  }
+  template <typename T, typename std::enable_if<
+                            !is_arena_constructable<T>::value &&
+                                !InternalHelper<T>::has_get_arena::value,
+                            int>::type = 0>
+  PROTOBUF_ALWAYS_INLINE static Arena* GetArenaInternal(const T* value) {
+    return nullptr;
   }
 
   // For friends of arena.

src/google/protobuf/arena_unittest.cc

@@ -1324,6 +1324,12 @@ TEST(ArenaTest, GetArenaShouldReturnTheArenaForArenaAllocatedMessages) {
   const ArenaMessage* const_pointer_to_message = message;
   EXPECT_EQ(&arena, Arena::GetArena(message));
   EXPECT_EQ(&arena, Arena::GetArena(const_pointer_to_message));
+
+  // Test that the Message* / MessageLite* specialization SFINAE works.
+  const Message* const_pointer_to_message_type = message;
+  EXPECT_EQ(&arena, Arena::GetArena(const_pointer_to_message_type));
+  const MessageLite* const_pointer_to_message_lite_type = message;
+  EXPECT_EQ(&arena, Arena::GetArena(const_pointer_to_message_lite_type));
 }
 
 TEST(ArenaTest, GetArenaShouldReturnNullForNonArenaAllocatedMessages) {

src/google/protobuf/compiler/command_line_interface_unittest.cc

@@ -2463,9 +2463,8 @@ TEST_P(EncodeDecodeTest, ProtoParseError) {
       "net/proto2/internal/no_such_file.proto: No such file or directory\n");
 }
 
-INSTANTIATE_TEST_CASE_P(FileDescriptorSetSource,
-                        EncodeDecodeTest,
-                        testing::Values(PROTO_PATH, DESCRIPTOR_SET_IN));
+INSTANTIATE_TEST_SUITE_P(FileDescriptorSetSource, EncodeDecodeTest,
+                         testing::Values(PROTO_PATH, DESCRIPTOR_SET_IN));
 }  // anonymous namespace
 
 #endif  // !GOOGLE_PROTOBUF_HEAP_CHECK_DRACONIAN

src/google/protobuf/compiler/cpp/cpp_enum.cc

@@ -72,7 +72,6 @@ EnumGenerator::EnumGenerator(const EnumDescriptor* descriptor,
   variables_["short_name"] = descriptor_->name();
   variables_["enumbase"] = options_.proto_h ? " : int" : "";
   variables_["nested_name"] = descriptor_->name();
-  variables_["constexpr"] = options_.proto_h ? "constexpr" : "";
   variables_["prefix"] =
       (descriptor_->containing_type() == NULL) ? "" : classname_ + "_";
 }
@@ -127,15 +126,15 @@ void EnumGenerator::GenerateDefinition(io::Printer* printer) {
 
   format(
       "$dllexport_decl $bool $classname$_IsValid(int value);\n"
-      "const $classname$ ${1$$prefix$$short_name$_MIN$}$ = "
+      "constexpr $classname$ ${1$$prefix$$short_name$_MIN$}$ = "
       "$prefix$$2$;\n"
-      "const $classname$ ${1$$prefix$$short_name$_MAX$}$ = "
+      "constexpr $classname$ ${1$$prefix$$short_name$_MAX$}$ = "
       "$prefix$$3$;\n",
       descriptor_, EnumValueName(min_value), EnumValueName(max_value));
 
   if (generate_array_size_) {
     format(
-        "const int ${1$$prefix$$short_name$_ARRAYSIZE$}$ = "
+        "constexpr int ${1$$prefix$$short_name$_ARRAYSIZE$}$ = "
         "$prefix$$short_name$_MAX + 1;\n\n",
         descriptor_);
   }
@@ -199,7 +198,7 @@ void EnumGenerator::GenerateSymbolImports(io::Printer* printer) const {
     string deprecated_attr = DeprecatedAttribute(
         options_, descriptor_->value(j)->options().deprecated());
     format(
-        "$1$static $constexpr $const $nested_name$ ${2$$3$$}$ =\n"
+        "$1$static constexpr $nested_name$ ${2$$3$$}$ =\n"
         "  $classname$_$3$;\n",
         deprecated_attr, descriptor_->value(j),
         EnumValueName(descriptor_->value(j)));
@@ -209,14 +208,14 @@ void EnumGenerator::GenerateSymbolImports(io::Printer* printer) const {
       "static inline bool $nested_name$_IsValid(int value) {\n"
       "  return $classname$_IsValid(value);\n"
       "}\n"
-      "static const $nested_name$ ${1$$nested_name$_MIN$}$ =\n"
+      "static constexpr $nested_name$ ${1$$nested_name$_MIN$}$ =\n"
       "  $classname$_$nested_name$_MIN;\n"
-      "static const $nested_name$ ${1$$nested_name$_MAX$}$ =\n"
+      "static constexpr $nested_name$ ${1$$nested_name$_MAX$}$ =\n"
       "  $classname$_$nested_name$_MAX;\n",
       descriptor_);
   if (generate_array_size_) {
     format(
-        "static const int ${1$$nested_name$_ARRAYSIZE$}$ =\n"
+        "static constexpr int ${1$$nested_name$_ARRAYSIZE$}$ =\n"
         "  $classname$_$nested_name$_ARRAYSIZE;\n",
         descriptor_);
   }
@@ -297,25 +296,26 @@ void EnumGenerator::GenerateMethods(int idx, io::Printer* printer) {
 
   if (descriptor_->containing_type() != NULL) {
     string parent = ClassName(descriptor_->containing_type(), false);
-    // We need to "define" the static constants which were declared in the
-    // header, to give the linker a place to put them.  Or at least the C++
-    // standard says we have to.  MSVC actually insists that we do _not_ define
-    // them again in the .cc file, prior to VC++ 2015.
-    format("#if !defined(_MSC_VER) || _MSC_VER >= 1900\n");
+    // Before C++17, we must define the static constants which were
+    // declared in the header, to give the linker a place to put them.
+    // But pre-2015 MSVC++ insists that we not.
+    format("#if (__cplusplus < 201703) && "
+           "(!defined(_MSC_VER) || _MSC_VER >= 1900)\n");
 
     for (int i = 0; i < descriptor_->value_count(); i++) {
-      format("$constexpr $const $classname$ $1$::$2$;\n", parent,
+      format("constexpr $classname$ $1$::$2$;\n", parent,
              EnumValueName(descriptor_->value(i)));
     }
     format(
-        "const $classname$ $1$::$nested_name$_MIN;\n"
-        "const $classname$ $1$::$nested_name$_MAX;\n",
+        "constexpr $classname$ $1$::$nested_name$_MIN;\n"
+        "constexpr $classname$ $1$::$nested_name$_MAX;\n",
         parent);
     if (generate_array_size_) {
-      format("const int $1$::$nested_name$_ARRAYSIZE;\n", parent);
+      format("constexpr int $1$::$nested_name$_ARRAYSIZE;\n", parent);
     }
 
-    format("#endif  // !defined(_MSC_VER) || _MSC_VER >= 1900\n");
+    format("#endif  // (__cplusplus < 201703) && "
+           "(!defined(_MSC_VER) || _MSC_VER >= 1900)\n");
   }
 }
 

src/google/protobuf/compiler/cpp/cpp_file.cc

@@ -706,29 +706,34 @@ void FileGenerator::GenerateReflectionInitializationCode(io::Printer* printer) {
   // in the file.
 
   if (!message_generators_.empty()) {
-    format("::$proto_ns$::Metadata $file_level_metadata$[$1$];\n",
+    format("static ::$proto_ns$::Metadata $file_level_metadata$[$1$];\n",
            message_generators_.size());
   } else {
     format(
+        "static "
         "constexpr ::$proto_ns$::Metadata* $file_level_metadata$ = nullptr;\n");
   }
   if (!enum_generators_.empty()) {
     format(
+        "static "
         "const ::$proto_ns$::EnumDescriptor* "
         "$file_level_enum_descriptors$[$1$];\n",
         enum_generators_.size());
   } else {
     format(
+        "static "
         "constexpr ::$proto_ns$::EnumDescriptor const** "
         "$file_level_enum_descriptors$ = nullptr;\n");
   }
   if (HasGenericServices(file_, options_) && file_->service_count() > 0) {
     format(
+        "static "
         "const ::$proto_ns$::ServiceDescriptor* "
         "$file_level_service_descriptors$[$1$];\n",
         file_->service_count());
   } else {
     format(
+        "static "
         "constexpr ::$proto_ns$::ServiceDescriptor const** "
         "$file_level_service_descriptors$ = nullptr;\n");
   }
@@ -795,6 +800,7 @@ void FileGenerator::GenerateReflectionInitializationCode(io::Printer* printer) {
   // AssignDescriptors().  All later times, waits for the first call to
   // complete and then returns.
   format(
+      "static "
       "::$proto_ns$::internal::AssignDescriptorsTable $assign_desc_table$ = "
       "{\n"
       "  {}, $add_descriptors$, \"$filename$\", schemas,\n"
@@ -846,6 +852,7 @@ void FileGenerator::GenerateReflectionInitializationCode(io::Printer* printer) {
 
   // Now generate the AddDescriptors() function.
   format(
+      "static "
       "::$proto_ns$::internal::DescriptorTable $1$ = {\n"
       "  false, $init_defaults$, \n"
       "  $2$,\n",
@@ -1295,6 +1302,26 @@ void FileGenerator::GenerateLibraryIncludes(io::Printer* printer) {
 
   if (IsAnyMessage(file_, options_)) {
     IncludeFile("net/proto2/internal/any.h", printer);
+  } else {
+    // For Any support with lite protos, we need to friend AnyMetadata, so we
+    // forward-declare it here.
+    if (options_.opensource_runtime) {
+      format(
+          "namespace google {\n"
+          "namespace protobuf {\n"
+          "namespace internal {\n"
+          "class AnyMetadata;\n"
+          "}  // namespace internal\n"
+          "}  // namespace protobuf\n"
+          "}  // namespace google\n");
+    } else {
+      format(
+          "namespace google {\nnamespace protobuf {\n"
+          "namespace internal {\n"
+          "class AnyMetadata;\n"
+          "}  // namespace internal\n"
+          "}  // namespace protobuf\n}  // namespace google\n");
+    }
   }
 }
 

src/google/protobuf/compiler/cpp/cpp_helpers.cc

@@ -32,6 +32,7 @@
 //  Based on original Protocol Buffers design by
 //  Sanjay Ghemawat, Jeff Dean, and others.
 
+#include <functional>
 #include <limits>
 #include <map>
 #include <queue>
@@ -75,7 +76,7 @@ string DotsToColons(const string& name) {
   return StringReplace(name, ".", "::", true);
 }
 
-const char* const kKeywordList[] = {  //
+static const char* const kKeywordList[] = {  //
     "NULL",
     "alignas",
     "alignof",
@@ -160,15 +161,15 @@ const char* const kKeywordList[] = {  //
     "xor",
     "xor_eq"};
 
-std::unordered_set<string> MakeKeywordsMap() {
-  std::unordered_set<string> result;
-  for (int i = 0; i < GOOGLE_ARRAYSIZE(kKeywordList); i++) {
-    result.insert(kKeywordList[i]);
+static std::unordered_set<string>* MakeKeywordsMap() {
+  auto* result = new std::unordered_set<string>();
+  for (const auto keyword : kKeywordList) {
+    result->emplace(keyword);
   }
   return result;
 }
 
-std::unordered_set<string> kKeywords = MakeKeywordsMap();
+static std::unordered_set<string>& kKeywords = *MakeKeywordsMap();
 
 // Returns whether the provided descriptor has an extension. This includes its
 // nested types.
@@ -255,6 +256,7 @@ void SetCommonVars(const Options& options,
         "ECK";
   }
 
+  SetIntVar(options, "int8", variables);
   SetIntVar(options, "uint8", variables);
   SetIntVar(options, "uint32", variables);
   SetIntVar(options, "uint64", variables);
@@ -910,11 +912,7 @@ FieldOptions::CType EffectiveStringCType(const FieldDescriptor* field,
 }
 
 bool IsAnyMessage(const FileDescriptor* descriptor, const Options& options) {
-  // For now we do not support Any in lite mode, so if we're building for lite
-  // then we just treat Any as if it's an ordinary message with no special
-  // behavior.
-  return descriptor->name() == kAnyProtoFile &&
-         GetOptimizeFor(descriptor, options) != FileOptions::LITE_RUNTIME;
+  return descriptor->name() == kAnyProtoFile;
 }
 
 bool IsAnyMessage(const Descriptor* descriptor, const Options& options) {
@@ -1302,14 +1300,20 @@ class ParseLoopGenerator {
 
   void GenerateParserLoop(const Descriptor* descriptor) {
     format_.Set("classname", ClassName(descriptor));
-    format_.Set("proto_ns", ProtobufNamespace(options_));
+    format_.Set("p_ns", "::" + ProtobufNamespace(options_));
+    format_.Set("pi_ns", StrCat("::", ProtobufNamespace(options_), "::internal"));
     format_.Set("GOOGLE_PROTOBUF", MacroPrefix(options_));
+    format_.Set("kSlopBytes",
+                static_cast<int>(internal::ParseContext::kSlopBytes));
     std::map<string, string> vars;
     SetCommonVars(options_, &vars);
     format_.AddMap(vars);
 
     std::vector<const FieldDescriptor*> ordered_fields;
     for (auto field : FieldRange(descriptor)) {
+      if (IsProto1(descriptor->file(), options_)) {
+        if (field->number() >= (1 << 14)) continue;
+      }
       ordered_fields.push_back(field);
     }
     std::sort(ordered_fields.begin(), ordered_fields.end(),
@@ -1318,19 +1322,11 @@ class ParseLoopGenerator {
               });
 
     format_(
-        "const char* $classname$::_InternalParse(const char* begin, const "
-        "char* "
-        "end, void* object,\n"
-        "                  ::$proto_ns$::internal::ParseContext* ctx) {\n"
-        "  auto msg = static_cast<$classname$*>(object);\n"
-        "  $int32$ size; (void)size;\n"
-        "  int depth; (void)depth;\n"
-        "  $uint32$ tag;\n"
-        "  ::$proto_ns$::internal::ParseFunc parser_till_end; "
-        "(void)parser_till_end;\n"
-        "  auto ptr = begin;\n"
-        "  while (ptr < end) {\n"
-        "    ptr = ::$proto_ns$::io::Parse32(ptr, &tag);\n"
+        "const char* $classname$::_InternalParse(const char* ptr, "
+        "$pi_ns$::ParseContext* ctx) {\n"
+        "  while (!ctx->Done(&ptr)) {\n"
+        "    $uint32$ tag;\n"
+        "    ptr = $pi_ns$::ReadTag(ptr, &tag);\n"
         "    $GOOGLE_PROTOBUF$_PARSER_ASSERT(ptr);\n"
         "    switch (tag >> 3) {\n");
 
@@ -1338,11 +1334,7 @@ class ParseLoopGenerator {
     format_.Indent();
     format_.Indent();
 
-    bool use_handle_unusual = false;
     for (const auto* field : ordered_fields) {
-      if (IsProto1(descriptor->file(), options_)) {
-        if (field->number() >= (1 << 14)) continue;
-      }
       // Print the field's (or oneof's) proto-syntax definition as a comment.
       // We don't want to print group bodies so we cut off after the first
       // line.
@@ -1359,22 +1351,21 @@ class ParseLoopGenerator {
           "case $2$: {\n",
           def, field->number());
       format_.Indent();
-      use_handle_unusual = true;
       GenerateCaseBody(field);
       format_.Outdent();
       format_("}\n");  // case
     }                  // for fields
+
+    // Default case
     format_("default: {\n");
-    if (use_handle_unusual) format_("handle_unusual:\n");
+    if (!ordered_fields.empty()) format_("handle_unusual:\n");
     format_(
         "  if ((tag & 7) == 4 || tag == 0) {\n"
-        "    ctx->EndGroup(tag);\n"
+        "    ctx->SetLastTag(tag);\n"
         "    return ptr;\n"
         "  }\n");
     if (IsMapEntryMessage(descriptor)) {
-      format_(
-          "  break;\n"
-          "}\n");
+      format_("  break;\n");
     } else {
       if (descriptor->extension_range_count() > 0) {
         format_("if (");
@@ -1396,113 +1387,59 @@ class ParseLoopGenerator {
         }
         format_(") {\n");
         format_(
-            "  auto res = msg->_extensions_.ParseField(tag, {_InternalParse, "
-            "msg}, ptr, end,\n"
-            "      internal_default_instance(), &msg->_internal_metadata_, "
+            "  ptr = _extensions_.ParseField(tag, ptr, \n"
+            "      internal_default_instance(), &_internal_metadata_, "
             "ctx);\n"
-            "  ptr = res.first;\n"
             "  $GOOGLE_PROTOBUF$_PARSER_ASSERT(ptr != nullptr);\n"
-            "  if (res.second) return ptr;\n"
-            "  continue;\n"
+            "  break;\n"
             "}\n");
       }
       format_(
-          "  auto res = UnknownFieldParse(tag, {_InternalParse, msg},\n"
-          "    ptr, end, msg->_internal_metadata_.mutable_unknown_fields(), "
-          "ctx);\n"
-          "  ptr = res.first;\n"
+          "  ptr = UnknownFieldParse(tag,\n"
+          "    _internal_metadata_.mutable_unknown_fields(), ptr, ctx);\n"
           "  $GOOGLE_PROTOBUF$_PARSER_ASSERT(ptr != nullptr);\n"
-          "  if (res.second) return ptr;\n"
-          "}\n");  // default case
+          "  break;\n");
     }
+    format_("}\n");  // default case
     format_.Outdent();
     format_.Outdent();
     format_.Outdent();
     format_(
         "    }  // switch\n"
         "  }  // while\n"
-        "  return ptr;\n");
-    if (use_string_) {
-      format_(
-          "string_till_end:\n"
-          "  static_cast<$string$*>(object)->clear();\n"
-          // TODO(gerbens) evaluate security
-          "  static_cast<$string$*>(object)->reserve(size);\n"
-          "  goto len_delim_till_end;\n");
-    }
-    if (use_arena_string_) {
-      format_(
-          "arena_string_till_end:\n"
-          "  object = "
-          "static_cast<::$proto_ns$::internal::ArenaStringPtr*>(object)->"
-          "Mutable(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), "
-          "msg->GetArenaNoVirtual());\n"
-          "  static_cast<$string$*>(object)->clear();\n"
-          // TODO(gerbens) evaluate security
-          "  static_cast<$string$*>(object)->reserve(size);\n"
-          "  goto len_delim_till_end;\n");
-    }
-    if (use_length_delimited_) {
-      format_(
-          "len_delim_till_end:\n"
-          "  return ctx->StoreAndTailCall(ptr, end, {_InternalParse, msg},\n"
-          "                               {parser_till_end, object}, size);\n");
-    }
-    if (use_group_) {
-      // Group crossed end and must be continued. Either this a parse failure
-      // or we need to resume on the next chunk and thus save the state.
-      format_(
-          "group_continues:\n"
-          "  $DCHK$(ptr >= end);\n"
-          "  $GOOGLE_PROTOBUF$_PARSER_ASSERT(ctx->StoreGroup(\n   "
-          "   {_InternalParse, msg}, {parser_till_end, object}, depth, tag));\n"
-          "  return ptr;\n");
-    }
-    format_("}\n");
+        "  return ptr;\n"
+        "}\n");
   }
 
  private:
   MessageSCCAnalyzer* scc_analyzer_;
   const Options& options_;
   Formatter format_;
-  bool use_length_delimited_ = false;
-  bool use_group_ = false;
-  bool use_string_ = false;
-  bool use_arena_string_ = false;
 
   using WireFormat = internal::WireFormat;
   using WireFormatLite = internal::WireFormatLite;
 
-  void GenerateArenaString(const FieldDescriptor* field, const string& utf8) {
-    use_arena_string_ = true;
+  void GenerateArenaString(const FieldDescriptor* field, const string& utf8,
+                           const string& field_name) {
     if (HasFieldPresence(field->file())) {
-      format_("HasBitSetters::set_has_$1$(msg);\n", FieldName(field));
+      format_("HasBitSetters::set_has_$1$(this);\n", FieldName(field));
     }
     format_(
-        "if (size > end - ptr + "
-        "::$proto_ns$::internal::ParseContext::kSlopBytes) {\n"
-        "  object = &msg->$1$_;\n"
-        "  parser_till_end = ::$proto_ns$::internal::GreedyStringParser$2$;\n"
-        "  goto arena_string_till_end;\n"
-        "}\n"
-        "$GOOGLE_PROTOBUF$_PARSER_ASSERT(::$proto_ns$::internal::StringCheck$2$"
-        "(ptr, size, ctx));\n"
-        "::$proto_ns$::internal::CopyIntoArenaString(ptr, size, &msg->$1$_, "
-        "msg->GetArenaNoVirtual());\n"
-        "ptr += size;\n",
-        FieldName(field), utf8);
+        "ptr = $pi_ns$::InlineCopyIntoArenaString$1$(&$2$_, ptr, ctx, "
+        "GetArenaNoVirtual()$3$);\n",
+        utf8, FieldName(field), field_name);
   }
 
   void GenerateStrings(const FieldDescriptor* field, bool check_utf8) {
     string utf8;
+    string field_name;
     if (check_utf8) {
       utf8 = GetUtf8Suffix(field, options_);
       if (!utf8.empty()) {
-        string name = "nullptr";
+        field_name = ", nullptr";
         if (HasDescriptorMethods(field->file(), options_)) {
-          name = "\"" + field->full_name() + "\"";
+          field_name = StrCat(", \"", field->full_name(), "\"");
         }
-        format_("ctx->extra_parse_data().SetFieldName($1$);\n", name);
       }
     }
     FieldOptions::CType ctype = FieldOptions::STRING;
@@ -1523,73 +1460,40 @@ class ParseLoopGenerator {
         !IsProto1(field->file(), options_) &&
         !IsStringInlined(field, options_) &&
         field->containing_oneof() == nullptr && ctype == FieldOptions::STRING) {
-      GenerateArenaString(field, utf8);
+      GenerateArenaString(field, utf8, field_name);
       return;
     }
-    format_(
-        "object = msg->$1$_$2$();\n"
-        "if (size > end - ptr + "
-        "::$proto_ns$::internal::ParseContext::kSlopBytes) {\n",
-        field->is_repeated() && !field->is_packable() ? "add" : "mutable",
-        FieldName(field));
     string name;
-    string label = "len_delim_till_end";
     switch (ctype) {
       case FieldOptions::STRING:
-        name = "GreedyStringParser";
-        use_string_ = true;
-        label = "string_till_end";
+        name = "GreedyStringParser" + utf8;
         break;
       case FieldOptions::CORD:
-        name = "CordParser";
-        format_("  static_cast<::Cord*>(object)->Clear();\n");
+        name = "CordParser" + utf8;
         break;
       case FieldOptions::STRING_PIECE:
-        name = "StringPieceParser";
-        format_(
-            "  "
-            "static_cast<::$proto_ns$::internal::StringPieceField*>(object)->"
-            "Clear();\n");
+        name = "StringPieceParser" + utf8;
         break;
     }
     format_(
-        "  parser_till_end = ::$proto_ns$::internal::$1$$2$;\n"
-        "  goto $3$;\n"
-        "}\n"
-        "$GOOGLE_PROTOBUF$_PARSER_ASSERT(::$proto_ns$::internal::StringCheck$2$"
-        "(ptr, size, ctx));\n"
-        "::$proto_ns$::internal::Inline$1$(object, ptr, size, ctx);\n"
-        "ptr += size;\n",
-        name, utf8, label);
+        "ptr = $pi_ns$::Inline$1$($2$_$3$(), ptr, ctx$4$);\n",
+        name, field->is_repeated() && !field->is_packable() ? "add" : "mutable",
+        FieldName(field), field_name);
   }
 
   void GenerateLengthDelim(const FieldDescriptor* field) {
-    format_(
-        "ptr = ::$proto_ns$::io::ReadSize(ptr, &size);\n"
-        "$GOOGLE_PROTOBUF$_PARSER_ASSERT(ptr);\n");
     if (!IsProto1(field->file(), options_) && field->is_packable()) {
+      string enum_validator;
       if (!HasPreservingUnknownEnumSemantics(field->file()) &&
           field->type() == FieldDescriptor::TYPE_ENUM) {
-        format_(
-            "ctx->extra_parse_data().SetEnumValidator($1$_IsValid, "
-            "msg->mutable_unknown_fields(), $2$);\n"
-            "parser_till_end = "
-            "::$proto_ns$::internal::PackedValidEnumParser$3$;\n"
-            "object = msg->mutable_$4$();\n",
-            QualifiedClassName(field->enum_type()), field->number(),
-            UseUnknownFieldSet(field->file(), options_) ? "" : "Lite",
-            FieldName(field));
-      } else {
-        format_(
-            "parser_till_end = ::$proto_ns$::internal::Packed$1$Parser;\n"
-            "object = msg->mutable_$2$();\n",
-            DeclaredTypeMethodName(field->type()), FieldName(field));
+        enum_validator = StrCat(
+            ", ", QualifiedClassName(field->enum_type()),
+            "_IsValid, mutable_unknown_fields(), ", field->number());
       }
       format_(
-          "if (size > end - ptr) goto len_delim_till_end;\n"
-          "auto newend = ptr + size;\n"
-          "if (size) ptr = parser_till_end(ptr, newend, object, ctx);\n"
-          "$GOOGLE_PROTOBUF$_PARSER_ASSERT(ptr == newend);\n");
+          "ptr = $pi_ns$::Packed$1$Parser(mutable_$2$(), ptr, ctx$3$);\n",
+          DeclaredTypeMethodName(field->type()), FieldName(field),
+          enum_validator);
     } else {
       auto field_type = field->type();
       if (IsProto1(field->file(), options_)) {
@@ -1610,106 +1514,62 @@ class ParseLoopGenerator {
           GenerateStrings(field, false /* utf8 */);
           break;
         case FieldDescriptor::TYPE_MESSAGE: {
-          GOOGLE_CHECK(field->message_type());
           if (!IsProto1(field->file(), options_) && field->is_map()) {
-            const FieldDescriptor* val =
-                field->message_type()->FindFieldByName("value");
-            GOOGLE_CHECK(val);
-            if (HasFieldPresence(field->file()) &&
-                val->type() == FieldDescriptor::TYPE_ENUM) {
-              format_(
-                  "ctx->extra_parse_data().field_number = $1$;\n"
-                  "ctx->extra_parse_data().unknown_fields = "
-                  "&msg->_internal_metadata_;\n",
-                  field->number());
-            }
-            format_(
-                "parser_till_end = "
-                "::$proto_ns$::internal::SlowMapEntryParser;\n"
-                "auto parse_map = $1$::_ParseMap;\n"
-                "ctx->extra_parse_data().payload.clear();\n"
-                "ctx->extra_parse_data().parse_map = parse_map;\n"
-                "object = &msg->$2$_;\n"
-                "if (size > end - ptr) goto len_delim_till_end;\n"
-                "auto newend = ptr + size;\n"
-                "$GOOGLE_PROTOBUF$_PARSER_ASSERT(parse_map(ptr, newend, "
-                "object, ctx));\n"
-                "ptr = newend;\n",
-                QualifiedClassName(field->message_type()), FieldName(field));
-            break;
-          }
-          if (!IsProto1(field->file(), options_) && IsLazy(field, options_)) {
+            format_("ptr = ctx->ParseMessage(&$1$_, ptr);\n", FieldName(field));
+          } else if (!IsProto1(field->file(), options_) &&
+                     IsLazy(field, options_)) {
             if (field->containing_oneof() != nullptr) {
               format_(
-                  "if (!msg->has_$1$()) {\n"
-                  "  msg->clear_$1$();\n"
-                  "  msg->$2$_.$1$_ = ::$proto_ns$::Arena::CreateMessage<\n"
-                  "      ::$proto_ns$::internal::LazyField>("
-                  "msg->GetArenaNoVirtual());\n"
-                  "  msg->set_has_$1$();\n"
+                  "if (!has_$1$()) {\n"
+                  "  clear_$1$();\n"
+                  "  $2$_.$1$_ = ::$proto_ns$::Arena::CreateMessage<\n"
+                  "      $pi_ns$::LazyField>("
+                  "GetArenaNoVirtual());\n"
+                  "  set_has_$1$();\n"
                   "}\n"
-                  "auto parse_closure = msg->$2$_.$1$_->_ParseClosure();\n",
+                  "ptr = ctx->ParseMessage($2$_.$1$_, ptr);\n",
                   FieldName(field), field->containing_oneof()->name());
             } else if (HasFieldPresence(field->file())) {
               format_(
-                  "HasBitSetters::set_has_$1$(msg);\n"
-                  "auto parse_closure = msg->$1$_._ParseClosure();\n",
+                  "HasBitSetters::set_has_$1$(this);\n"
+                  "ptr = ctx->ParseMessage(&$1$_, ptr);\n",
                   FieldName(field));
             } else {
-              format_("auto parse_closure = msg->$1$_._ParseClosure();\n",
-                      FieldName(field));
+              format_(
+                  "ptr = ctx->ParseMessage(&$1$_, ptr);\n", FieldName(field));
             }
-            format_(
-                "parser_till_end = parse_closure.func;\n"
-                "object = parse_closure.object;\n"
-                "if (size > end - ptr) goto len_delim_till_end;\n"
-                "auto newend = ptr + size;\n"
-                "$GOOGLE_PROTOBUF$_PARSER_ASSERT(ctx->ParseExactRange(\n"
-                "    parse_closure, ptr, newend));\n"
-                "ptr = newend;\n");
-            break;
-          }
-          if (IsImplicitWeakField(field, options_, scc_analyzer_)) {
+          } else if (IsImplicitWeakField(field, options_, scc_analyzer_)) {
             if (!field->is_repeated()) {
-              format_("object = HasBitSetters::mutable_$1$(msg);\n",
-                      FieldName(field));
+              format_(
+                  "ptr = ctx->ParseMessage(HasBitSetters::mutable_$1$(this), "
+                  "ptr);\n",
+                  FieldName(field));
             } else {
               format_(
-                  "object = "
-                  "CastToBase(&msg->$1$_)->AddWeak(reinterpret_cast<const "
-                  "::$proto_ns$::MessageLite*>(&$2$::_$3$_default_instance_));"
-                  "\n",
+                  "ptr = ctx->ParseMessage("
+                  "CastToBase(&$1$_)->AddWeak(reinterpret_cast<const "
+                  "::$proto_ns$::MessageLite*>(&$2$::_$3$_default_instance_)), "
+                  "ptr);\n",
                   FieldName(field), Namespace(field->message_type()),
                   ClassName(field->message_type()));
             }
-            format_(
-                "parser_till_end = static_cast<::$proto_ns$::MessageLite*>("
-                "object)->_ParseFunc();\n");
           } else if (IsWeak(field, options_)) {
             if (IsProto1(field->file(), options_)) {
-              format_("object = msg->internal_mutable_$1$();\n",
-                      FieldName(field));
+              format_(
+                  "ptr = ctx->ParseMessage("
+                  "reinterpret_cast<$p_ns$::MessageLite*>(internal_mutable_$1$("
+                  ")), ptr);\n",
+                  FieldName(field));
             } else {
               format_(
-                  "object = msg->_weak_field_map_.MutableMessage($1$, "
-                  "_$classname$_default_instance_.$2$_);\n",
+                  "ptr = ctx->ParseMessage(_weak_field_map_.MutableMessage($1$,"
+                  " _$classname$_default_instance_.$2$_), ptr);\n",
                   field->number(), FieldName(field));
             }
-            format_(
-                "parser_till_end = static_cast<::$proto_ns$::MessageLite*>("
-                "object)->_ParseFunc();\n");
           } else {
-            format_(
-                "parser_till_end = $1$::_InternalParse;\n"
-                "object = msg->$2$_$3$();\n",
-                QualifiedClassName(field->message_type()),
-                field->is_repeated() ? "add" : "mutable", FieldName(field));
+            format_("ptr = ctx->ParseMessage($1$_$2$(), ptr);\n",
+                    field->is_repeated() ? "add" : "mutable", FieldName(field));
           }
-          format_(
-              "if (size > end - ptr) goto len_delim_till_end;\n"
-              "ptr += size;\n"
-              "$GOOGLE_PROTOBUF$_PARSER_ASSERT(ctx->ParseExactRange(\n"
-              "    {parser_till_end, object}, ptr - size, ptr));\n");
           break;
         }
         default:
@@ -1731,31 +1591,33 @@ class ParseLoopGenerator {
         string prefix = field->is_repeated() ? "add" : "set";
         if (field->type() == FieldDescriptor::TYPE_ENUM &&
             !IsProto1(field->file(), options_)) {
-          format_("$uint64$ val = ::$proto_ns$::internal::ReadVarint(&ptr);\n");
+          format_(
+              "$uint64$ val = $pi_ns$::ReadVarint(&ptr);\n"
+              "$GOOGLE_PROTOBUF$_PARSER_ASSERT(ptr);\n");
           if (!HasPreservingUnknownEnumSemantics(field->file())) {
             format_(
                 "if (!$1$_IsValid(val)) {\n"
-                "  ::$proto_ns$::internal::WriteVarint($2$, val, "
-                "msg->mutable_unknown_fields());\n"
+                "  $pi_ns$::WriteVarint($2$, val, "
+                "mutable_unknown_fields());\n"
                 "  break;\n"
                 "}\n",
                 QualifiedClassName(field->enum_type()), field->number());
           }
-          format_("msg->$1$_$2$(static_cast<$3$>(val));\n", prefix,
-                  FieldName(field), QualifiedClassName(field->enum_type()));
+          format_("$1$_$2$(static_cast<$3$>(val));\n", prefix, FieldName(field),
+                  QualifiedClassName(field->enum_type()));
         } else {
+          int size = field->type() == FieldDescriptor::TYPE_SINT32 ? 32 : 64;
           string zigzag;
           if ((field->type() == FieldDescriptor::TYPE_SINT32 ||
                field->type() == FieldDescriptor::TYPE_SINT64) &&
               !IsProto1(field->file(), options_)) {
-            int size = field->type() == FieldDescriptor::TYPE_SINT32 ? 32 : 64;
             zigzag = StrCat("ZigZag", size);
           }
           format_(
-              "msg->$1$_$2$(::$proto_ns$::internal::ReadVarint$3$(&ptr));\n",
+              "$1$_$2$($pi_ns$::ReadVarint$3$(&ptr));\n"
+              "$GOOGLE_PROTOBUF$_PARSER_ASSERT(ptr);\n",
               prefix, FieldName(field), zigzag);
         }
-        format_("$GOOGLE_PROTOBUF$_PARSER_ASSERT(ptr);\n");
         break;
       }
       case WireFormatLite::WIRETYPE_FIXED32:
@@ -1763,28 +1625,20 @@ class ParseLoopGenerator {
         string prefix = field->is_repeated() ? "add" : "set";
         string type = PrimitiveTypeName(options_, field->cpp_type());
         format_(
-            "msg->$1$_$2$(::$proto_ns$::io::UnalignedLoad<$3$>(ptr));\n"
+            "$1$_$2$($pi_ns$::UnalignedLoad<$3$>(ptr));\n"
             "ptr += sizeof($3$);\n",
             prefix, FieldName(field), type);
         break;
       }
       case WireFormatLite::WIRETYPE_LENGTH_DELIMITED: {
-        use_length_delimited_ = true;
         GenerateLengthDelim(field);
+        format_("$GOOGLE_PROTOBUF$_PARSER_ASSERT(ptr);\n");
         break;
       }
       case WireFormatLite::WIRETYPE_START_GROUP: {
-        use_group_ = true;
         format_(
-            "parser_till_end = $1$::_InternalParse;\n"
-            "object = msg->$2$_$3$();\n"
-            "auto res = ctx->ParseGroup(tag, {parser_till_end, object}, ptr, "
-            "end, "
-            "&depth);\n"
-            "ptr = res.first;\n"
-            "$GOOGLE_PROTOBUF$_PARSER_ASSERT(ptr);\n"
-            "if (res.second) goto group_continues;\n",
-            QualifiedClassName(field->message_type()),
+            "ptr = ctx->ParseGroup($1$_$2$(), ptr, tag);\n"
+            "$GOOGLE_PROTOBUF$_PARSER_ASSERT(ptr);\n",
             field->is_repeated() ? "add" : "mutable", FieldName(field));
         break;
       }
@@ -1795,7 +1649,6 @@ class ParseLoopGenerator {
     }  // switch (wire_type)
 
     if (ShouldRepeat(field, wiretype)) {
-      format_("if (ptr >= end) break;\n");
       uint32 x = field->number() * 8 + wiretype;
       uint64 y = 0;
       int cnt = 0;
@@ -1805,10 +1658,11 @@ class ParseLoopGenerator {
         x >>= 7;
       } while (x);
       uint64 mask = (1ull << (cnt * 8)) - 1;
+      format_("if (ctx->Done(&ptr)) return ptr;\n");
       format_.Outdent();
       format_(
-          "} while ((::$proto_ns$::io::UnalignedLoad<$uint64$>(ptr) & $1$) == "
-          "$2$ && (ptr += $3$));\n",
+          "} while (($pi_ns$::UnalignedLoad<$uint64$>(ptr)"
+          " & $1$) == $2$ && (ptr += $3$));\n",
           mask, y, cnt);
     }
     format_("break;\n");

src/google/protobuf/compiler/cpp/cpp_map_field.cc

@@ -65,15 +65,15 @@ void SetMessageVariables(const FieldDescriptor* descriptor,
   switch (val->cpp_type()) {
     case FieldDescriptor::CPPTYPE_MESSAGE:
       (*variables)["val_cpp"] = FieldMessageTypeName(val);
-      (*variables)["wrapper"] = "EntryWrapper";
+      (*variables)["wrapper"] = "MapEntryWrapper";
       break;
     case FieldDescriptor::CPPTYPE_ENUM:
       (*variables)["val_cpp"] = ClassName(val->enum_type(), true);
-      (*variables)["wrapper"] = "EnumEntryWrapper";
+      (*variables)["wrapper"] = "MapEnumEntryWrapper";
       break;
     default:
       (*variables)["val_cpp"] = PrimitiveTypeName(options, val->cpp_type());
-      (*variables)["wrapper"] = "EntryWrapper";
+      (*variables)["wrapper"] = "MapEntryWrapper";
   }
   (*variables)["key_wire_type"] =
       "TYPE_" + ToUpper(DeclaredTypeMethodName(key->type()));
@@ -238,11 +238,9 @@ GenerateMergeFromCodedStream(io::Printer* printer) const {
   }
 }
 
-static void GenerateSerializationLoop(const Formatter& format,
-                                      bool supports_arenas, bool string_key,
+static void GenerateSerializationLoop(const Formatter& format, bool string_key,
                                       bool string_value, bool to_array,
                                       bool is_deterministic) {
-  format("::std::unique_ptr<$map_classname$> entry;\n");
   string ptr;
   if (is_deterministic) {
     format("for (size_type i = 0; i < n; i++) {\n");
@@ -257,24 +255,17 @@ static void GenerateSerializationLoop(const Formatter& format,
   }
   format.Indent();
 
-  format("entry.reset($name$_.New$wrapper$($1$->first, $1$->second));\n", ptr);
+  format(
+      "$map_classname$::$wrapper$ entry(nullptr, $1$->first, $1$->second);\n",
+      ptr);
   if (to_array) {
     format(
         "target = ::$proto_ns$::internal::WireFormatLite::InternalWrite"
-        "$declared_type$NoVirtualToArray($number$, *entry, target);\n");
+        "$declared_type$NoVirtualToArray($number$, entry, target);\n");
   } else {
     format(
         "::$proto_ns$::internal::WireFormatLite::Write$stream_writer$($number$,"
-        " "
-        "*entry, output);\n");
-  }
-
-  // If entry is allocated by arena, its desctructor should be avoided.
-  if (supports_arenas) {
-    format(
-        "if (entry->GetArena() != nullptr) {\n"
-        "  entry.release();\n"
-        "}\n");
+        " entry, output);\n");
   }
 
   if (string_key || string_value) {
@@ -365,13 +356,11 @@ void MapFieldGenerator::GenerateSerializeWithCachedSizes(io::Printer* printer,
       "  ::std::sort(&items[0], &items[static_cast<ptrdiff_t>(n)], Less());\n",
       to_array ? "false" : "output->IsSerializationDeterministic()");
   format.Indent();
-  GenerateSerializationLoop(format, SupportsArenas(descriptor_), string_key,
-                            string_value, to_array, true);
+  GenerateSerializationLoop(format, string_key, string_value, to_array, true);
   format.Outdent();
   format("} else {\n");
   format.Indent();
-  GenerateSerializationLoop(format, SupportsArenas(descriptor_), string_key,
-                            string_value, to_array, false);
+  GenerateSerializationLoop(format, string_key, string_value, to_array, false);
   format.Outdent();
   format("}\n");
   format.Outdent();
@@ -384,35 +373,13 @@ GenerateByteSize(io::Printer* printer) const {
   format(
       "total_size += $tag_size$ *\n"
       "    ::$proto_ns$::internal::FromIntSize(this->$name$_size());\n"
-      "{\n"
-      "  ::std::unique_ptr<$map_classname$> entry;\n"
-      "  for (::$proto_ns$::Map< $key_cpp$, $val_cpp$ >::const_iterator\n"
-      "      it = this->$name$().begin();\n"
-      "      it != this->$name$().end(); ++it) {\n");
-
-  // If entry is allocated by arena, its desctructor should be avoided.
-  if (SupportsArenas(descriptor_)) {
-    format(
-        "    if (entry.get() != nullptr && entry->GetArena() != nullptr) {\n"
-        "      entry.release();\n"
-        "    }\n");
-  }
-
-  format(
-      "    entry.reset($name$_.New$wrapper$(it->first, it->second));\n"
-      "    total_size += ::$proto_ns$::internal::WireFormatLite::\n"
-      "        $declared_type$SizeNoVirtual(*entry);\n"
-      "  }\n");
-
-  // If entry is allocated by arena, its desctructor should be avoided.
-  if (SupportsArenas(descriptor_)) {
-    format(
-        "  if (entry.get() != nullptr && entry->GetArena() != nullptr) {\n"
-        "    entry.release();\n"
-        "  }\n");
-  }
-
-  format("}\n");
+      "for (::$proto_ns$::Map< $key_cpp$, $val_cpp$ >::const_iterator\n"
+      "    it = this->$name$().begin();\n"
+      "    it != this->$name$().end(); ++it) {\n"
+      "  $map_classname$::$wrapper$ entry(nullptr, it->first, it->second);\n"
+      "  total_size += ::$proto_ns$::internal::WireFormatLite::\n"
+      "      $declared_type$SizeNoVirtual(entry);\n"
+      "}\n");
 }
 
 }  // namespace cpp

src/google/protobuf/compiler/cpp/cpp_message.cc

@@ -948,10 +948,6 @@ void MessageGenerator::GenerateClassDefinition(io::Printer* printer) {
         "    ::$proto_ns$::internal::WireFormatLite::$val_wire_type$,\n"
         "    $default_enum_value$ > {\n"
         "public:\n"
-        "#if $GOOGLE_PROTOBUF$_ENABLE_EXPERIMENTAL_PARSER\n"
-        "static bool _ParseMap(const char* begin, const "
-        "char* end, void* object, ::$proto_ns$::internal::ParseContext* ctx);\n"
-        "#endif  // $GOOGLE_PROTOBUF$_ENABLE_EXPERIMENTAL_PARSER\n"
         "  typedef ::$proto_ns$::internal::MapEntry$lite$<$classname$, \n"
         "    $key_cpp$, $val_cpp$,\n"
         "    ::$proto_ns$::internal::WireFormatLite::$key_wire_type$,\n"
@@ -1097,20 +1093,39 @@ void MessageGenerator::GenerateClassDefinition(io::Printer* printer) {
   if (IsAnyMessage(descriptor_, options_)) {
     format(
         "// implements Any -----------------------------------------------\n"
-        "\n"
-        "void PackFrom(const ::$proto_ns$::Message& message);\n"
-        "void PackFrom(const ::$proto_ns$::Message& message,\n"
-        "              const $string$& type_url_prefix);\n"
-        "bool UnpackTo(::$proto_ns$::Message* message) const;\n"
+        "\n");
+    if (HasDescriptorMethods(descriptor_->file(), options_)) {
+      format(
+          "void PackFrom(const ::$proto_ns$::Message& message);\n"
+          "void PackFrom(const ::$proto_ns$::Message& message,\n"
+          "              const $string$& type_url_prefix);\n"
+          "bool UnpackTo(::$proto_ns$::Message* message) const;\n"
+          "static bool GetAnyFieldDescriptors(\n"
+          "    const ::$proto_ns$::Message& message,\n"
+          "    const ::$proto_ns$::FieldDescriptor** type_url_field,\n"
+          "    const ::$proto_ns$::FieldDescriptor** value_field);\n");
+    } else {
+      format(
+          "template <typename T>\n"
+          "void PackFrom(const T& message) {\n"
+          "  _any_metadata_.PackFrom(message);\n"
+          "}\n"
+          "template <typename T>\n"
+          "void PackFrom(const T& message,\n"
+          "              const $string$& type_url_prefix) {\n"
+          "  _any_metadata_.PackFrom(message, type_url_prefix);"
+          "}\n"
+          "template <typename T>\n"
+          "bool UnpackTo(T* message) const {\n"
+          "  return _any_metadata_.UnpackTo(message);\n"
+          "}\n");
+    }
+    format(
         "template<typename T> bool Is() const {\n"
         "  return _any_metadata_.Is<T>();\n"
         "}\n"
         "static bool ParseAnyTypeUrl(const string& type_url,\n"
-        "                            string* full_type_name);\n"
-        "static bool GetAnyFieldDescriptors(\n"
-        "    const ::$proto_ns$::Message& message,\n"
-        "    const ::$proto_ns$::FieldDescriptor** type_url_field,\n"
-        "    const ::$proto_ns$::FieldDescriptor** value_field);\n");
+        "                            string* full_type_name);\n");
   }
 
   format.Set("new_final",
@@ -1166,24 +1181,13 @@ void MessageGenerator::GenerateClassDefinition(io::Printer* printer) {
         "\n"
         "size_t ByteSizeLong() const final;\n"
         "#if $GOOGLE_PROTOBUF$_ENABLE_EXPERIMENTAL_PARSER\n"
-        "static const char* _InternalParse(const char* begin, const char* end, "
-        "void* object, ::$proto_ns$::internal::ParseContext* ctx);\n"
-        "::$proto_ns$::internal::ParseFunc _ParseFunc() const final { return "
-        "_InternalParse; }\n"
+        "const char* _InternalParse(const char* ptr, "
+        "::$proto_ns$::internal::ParseContext* ctx) final;\n"
         "#else\n"
         "bool MergePartialFromCodedStream(\n"
         "    ::$proto_ns$::io::CodedInputStream* input)$ "
         "merge_partial_final$;\n"
         "#endif  // $GOOGLE_PROTOBUF$_ENABLE_EXPERIMENTAL_PARSER\n");
-    if (descriptor_->options().message_set_wire_format()) {
-      format(
-          "#if $GOOGLE_PROTOBUF$_ENABLE_EXPERIMENTAL_PARSER\n"
-          "static const char* InternalParseMessageSetItem(const char* begin, "
-          "const char* end, void* object, "
-          "::$proto_ns$::internal::ParseContext* "
-          "ctx);\n"
-          "#endif  // $GOOGLE_PROTOBUF$_ENABLE_EXPERIMENTAL_PARSER\n");
-    }
 
     if (!options_.table_driven_serialization ||
         descriptor_->options().message_set_wire_format()) {
@@ -1206,10 +1210,20 @@ void MessageGenerator::GenerateClassDefinition(io::Printer* printer) {
   format(
       "int GetCachedSize() const final { return _cached_size_.Get(); }"
       "\n\nprivate:\n"
-      "void SharedCtor();\n"
-      "void SharedDtor();\n"
+      "inline void SharedCtor();\n"
+      "inline void SharedDtor();\n"
       "void SetCachedSize(int size) const$ full_final$;\n"
       "void InternalSwap($classname$* other);\n");
+
+  format(
+      // Friend AnyMetadata so that it can call this FullMessageName() method.
+      "friend class ::$proto_ns$::internal::AnyMetadata;\n"
+      "static $1$ FullMessageName() {\n"
+      "  return \"$full_name$\";\n"
+      "}\n",
+      options_.opensource_runtime ? "::google::protobuf::StringPiece"
+                                  : "::StringPiece");
+
   if (SupportsArenas(descriptor_)) {
     format(
         // TODO(gerbens) Make this private! Currently people are deriving from
@@ -1871,65 +1885,6 @@ void MessageGenerator::GenerateClassMethods(io::Printer* printer) {
           "}\n"
           "\n");
     }
-    format(
-        "#if $GOOGLE_PROTOBUF$_ENABLE_EXPERIMENTAL_PARSER\n"
-        "bool $classname$::_ParseMap(const char* begin, const char* end, "
-        "void* object, ::$proto_ns$::internal::ParseContext* ctx) {\n"
-        "  using MF = ::$proto_ns$::internal::MapField$1$<\n"
-        "      $classname$, EntryKeyType, EntryValueType,\n"
-        "      kEntryKeyFieldType, kEntryValueFieldType,\n"
-        "      kEntryDefaultEnumValue>;\n"
-        "  auto mf = static_cast<MF*>(object);\n"
-        "  Parser<MF, ::$proto_ns$::Map<EntryKeyType, EntryValueType>> "
-        "parser(mf);\n"
-        "#define DO_(x) if (!(x)) return false\n",
-        HasDescriptorMethods(descriptor_->file(), options_) ? "" : "Lite");
-    const FieldDescriptor* key = descriptor_->FindFieldByName("key");
-    const FieldDescriptor* val = descriptor_->FindFieldByName("value");
-    GOOGLE_CHECK(val);
-    string key_string;
-    string value_string;
-    if (HasFieldPresence(descriptor_->file()) &&
-        val->type() == FieldDescriptor::TYPE_ENUM) {
-      format(
-          "  DO_(parser.ParseMapEnumValidation(\n"
-          "    begin, end, ctx->extra_parse_data().field_number,\n"
-          "    static_cast<::$proto_ns$::internal::"
-          "InternalMetadataWithArena$1$*>("
-          "ctx->extra_parse_data().unknown_fields), $2$_IsValid));\n",
-          HasDescriptorMethods(descriptor_->file(), options_) ? "" : "Lite",
-          QualifiedClassName(val->enum_type()));
-      key_string = "parser.entry_key()";
-      value_string = "parser.entry_value()";
-    } else {
-      format("  DO_(parser.ParseMap(begin, end));\n");
-      key_string = "parser.key()";
-      value_string = "parser.value()";
-    }
-    format.Indent();
-    if (key->type() == FieldDescriptor::TYPE_STRING) {
-      GenerateUtf8CheckCodeForString(
-          key, options_, true,
-          StrCat(key_string, ".data(), static_cast<int>(", key_string,
-                       ".length()),\n")
-              .data(),
-          format);
-    }
-    if (val->type() == FieldDescriptor::TYPE_STRING) {
-      GenerateUtf8CheckCodeForString(
-          val, options_, true,
-          StrCat(value_string, ".data(), static_cast<int>(", value_string,
-                       ".length()),\n")
-              .data(),
-          format);
-    }
-    format.Outdent();
-    format(
-        "#undef DO_\n"
-        "  return true;\n"
-        "}\n"
-        "#endif  // $GOOGLE_PROTOBUF$_ENABLE_EXPERIMENTAL_PARSER\n");
-    format("\n");
     return;
   }
 
@@ -1942,31 +1897,34 @@ void MessageGenerator::GenerateClassMethods(io::Printer* printer) {
   format("}\n");
 
   if (IsAnyMessage(descriptor_, options_)) {
+    if (HasDescriptorMethods(descriptor_->file(), options_)) {
+      format(
+          "void $classname$::PackFrom(const ::$proto_ns$::Message& message) {\n"
+          "  _any_metadata_.PackFrom(message);\n"
+          "}\n"
+          "\n"
+          "void $classname$::PackFrom(const ::$proto_ns$::Message& message,\n"
+          "                           const $string$& type_url_prefix) {\n"
+          "  _any_metadata_.PackFrom(message, type_url_prefix);\n"
+          "}\n"
+          "\n"
+          "bool $classname$::UnpackTo(::$proto_ns$::Message* message) const {\n"
+          "  return _any_metadata_.UnpackTo(message);\n"
+          "}\n"
+          "bool $classname$::GetAnyFieldDescriptors(\n"
+          "    const ::$proto_ns$::Message& message,\n"
+          "    const ::$proto_ns$::FieldDescriptor** type_url_field,\n"
+          "    const ::$proto_ns$::FieldDescriptor** value_field) {\n"
+          "  return ::$proto_ns$::internal::GetAnyFieldDescriptors(\n"
+          "      message, type_url_field, value_field);\n"
+          "}\n");
+    }
     format(
-        "void $classname$::PackFrom(const ::$proto_ns$::Message& message) {\n"
-        "  _any_metadata_.PackFrom(message);\n"
-        "}\n"
-        "\n"
-        "void $classname$::PackFrom(const ::$proto_ns$::Message& message,\n"
-        "                           const $string$& type_url_prefix) {\n"
-        "  _any_metadata_.PackFrom(message, type_url_prefix);\n"
-        "}\n"
-        "\n"
-        "bool $classname$::UnpackTo(::$proto_ns$::Message* message) const {\n"
-        "  return _any_metadata_.UnpackTo(message);\n"
-        "}\n"
         "bool $classname$::ParseAnyTypeUrl(const string& type_url,\n"
         "                                  string* full_type_name) {\n"
         "  return ::$proto_ns$::internal::ParseAnyTypeUrl(type_url,\n"
         "                                             full_type_name);\n"
         "}\n"
-        "bool $classname$::GetAnyFieldDescriptors(\n"
-        "    const ::$proto_ns$::Message& message,\n"
-        "    const ::$proto_ns$::FieldDescriptor** type_url_field,\n"
-        "    const ::$proto_ns$::FieldDescriptor** value_field) {\n"
-        "  return ::$proto_ns$::internal::GetAnyFieldDescriptors(\n"
-        "      message, type_url_field, value_field);\n"
-        "}\n"
         "\n");
   }
 
@@ -3360,21 +3318,10 @@ void MessageGenerator::GenerateMergeFromCodedStream(io::Printer* printer) {
     // Special-case MessageSet.
     format(
         "#if $GOOGLE_PROTOBUF$_ENABLE_EXPERIMENTAL_PARSER\n"
-        "const char* $classname$::_InternalParse(const char* begin, const "
-        "char* end, void* object,\n"
-        "                  ::$proto_ns$::internal::ParseContext* ctx) {\n"
-        "  auto msg = static_cast<$classname$*>(object);\n"
-        "  return ::$proto_ns$::internal::ParseMessageSet(begin, end, "
-        "msg, &msg->_extensions_, &msg->_internal_metadata_, ctx);\n"
-        "}\n"
-        "const char* $classname$::InternalParseMessageSetItem(const char* "
-        "begin, const char* end, void* object,\n"
+        "const char* $classname$::_InternalParse(const char* ptr,\n"
         "                  ::$proto_ns$::internal::ParseContext* ctx) {\n"
-        "  auto msg = static_cast<$classname$*>(object);\n"
-        "  return "
-        "msg->_extensions_.ParseMessageSetItem({InternalParseMessageSetItem, "
-        "msg}, begin, end, internal_default_instance(), "
-        "&msg->_internal_metadata_, ctx);\n"
+        "  return _extensions_.ParseMessageSet(ptr, \n"
+        "      internal_default_instance(), &_internal_metadata_, ctx);\n"
         "}\n"
         "#else\n"
         "bool $classname$::MergePartialFromCodedStream(\n"

src/google/protobuf/compiler/java/java_enum_field.cc

@@ -152,11 +152,11 @@ ImmutableEnumFieldGenerator(const FieldDescriptor* descriptor,
 ImmutableEnumFieldGenerator::~ImmutableEnumFieldGenerator() {}
 
 int ImmutableEnumFieldGenerator::GetNumBitsForMessage() const {
-  return 1;
+  return SupportFieldPresence(descriptor_->file()) ? 1 : 0;
 }
 
 int ImmutableEnumFieldGenerator::GetNumBitsForBuilder() const {
-  return 1;
+  return GetNumBitsForMessage();
 }
 
 void ImmutableEnumFieldGenerator::

src/google/protobuf/compiler/java/java_enum_field_lite.cc

@@ -132,7 +132,7 @@ ImmutableEnumFieldLiteGenerator(const FieldDescriptor* descriptor,
 ImmutableEnumFieldLiteGenerator::~ImmutableEnumFieldLiteGenerator() {}
 
 int ImmutableEnumFieldLiteGenerator::GetNumBitsForMessage() const {
-  return 1;
+  return SupportFieldPresence(descriptor_->file()) ? 1 : 0;
 }
 
 void ImmutableEnumFieldLiteGenerator::

src/google/protobuf/compiler/java/java_message.cc

@@ -66,11 +66,6 @@ using internal::WireFormat;
 using internal::WireFormatLite;
 
 namespace {
-bool GenerateHasBits(const Descriptor* descriptor) {
-  return SupportFieldPresence(descriptor->file()) ||
-      HasRepeatedFields(descriptor);
-}
-
 string MapValueImmutableClassdName(const Descriptor* descriptor,
                                    ClassNameResolver* name_resolver) {
   const FieldDescriptor* value_field = descriptor->FindFieldByName("value");
@@ -397,18 +392,16 @@ void ImmutableMessageGenerator::Generate(io::Printer* printer) {
     messageGenerator.Generate(printer);
   }
 
-  if (GenerateHasBits(descriptor_)) {
-    // Integers for bit fields.
-    int totalBits = 0;
-    for (int i = 0; i < descriptor_->field_count(); i++) {
-      totalBits += field_generators_.get(descriptor_->field(i))
-          .GetNumBitsForMessage();
-    }
-    int totalInts = (totalBits + 31) / 32;
-    for (int i = 0; i < totalInts; i++) {
-      printer->Print("private int $bit_field_name$;\n",
-        "bit_field_name", GetBitFieldName(i));
-    }
+  // Integers for bit fields.
+  int totalBits = 0;
+  for (int i = 0; i < descriptor_->field_count(); i++) {
+    totalBits +=
+        field_generators_.get(descriptor_->field(i)).GetNumBitsForMessage();
+  }
+  int totalInts = (totalBits + 31) / 32;
+  for (int i = 0; i < totalInts; i++) {
+    printer->Print("private int $bit_field_name$;\n", "bit_field_name",
+                   GetBitFieldName(i));
   }
 
   // oneof

src/google/protobuf/compiler/java/java_message_builder.cc

@@ -61,11 +61,6 @@ namespace compiler {
 namespace java {
 
 namespace {
-bool GenerateHasBits(const Descriptor* descriptor) {
-  return SupportFieldPresence(descriptor->file()) ||
-      HasRepeatedFields(descriptor);
-}
-
 string MapValueImmutableClassdName(const Descriptor* descriptor,
                                    ClassNameResolver* name_resolver) {
   const FieldDescriptor* value_field = descriptor->FindFieldByName("value");
@@ -149,18 +144,16 @@ Generate(io::Printer* printer) {
       "\n");
   }
 
-  if (GenerateHasBits(descriptor_)) {
-    // Integers for bit fields.
-    int totalBits = 0;
-    for (int i = 0; i < descriptor_->field_count(); i++) {
-      totalBits += field_generators_.get(descriptor_->field(i))
-          .GetNumBitsForBuilder();
-    }
-    int totalInts = (totalBits + 31) / 32;
-    for (int i = 0; i < totalInts; i++) {
-      printer->Print("private int $bit_field_name$;\n",
-        "bit_field_name", GetBitFieldName(i));
-    }
+  // Integers for bit fields.
+  int totalBits = 0;
+  for (int i = 0; i < descriptor_->field_count(); i++) {
+    totalBits +=
+        field_generators_.get(descriptor_->field(i)).GetNumBitsForBuilder();
+  }
+  int totalInts = (totalBits + 31) / 32;
+  for (int i = 0; i < totalInts; i++) {
+    printer->Print("private int $bit_field_name$;\n", "bit_field_name",
+                   GetBitFieldName(i));
   }
 
   for (int i = 0; i < descriptor_->field_count(); i++) {
@@ -408,19 +401,17 @@ GenerateCommonBuilderMethods(io::Printer* printer) {
   int totalBuilderInts = (totalBuilderBits + 31) / 32;
   int totalMessageInts = (totalMessageBits + 31) / 32;
 
-  if (GenerateHasBits(descriptor_)) {
-    // Local vars for from and to bit fields to avoid accessing the builder and
-    // message over and over for these fields. Seems to provide a slight
-    // perforamance improvement in micro benchmark and this is also what proto1
-    // code does.
-    for (int i = 0; i < totalBuilderInts; i++) {
-      printer->Print("int from_$bit_field_name$ = $bit_field_name$;\n",
-        "bit_field_name", GetBitFieldName(i));
-    }
-    for (int i = 0; i < totalMessageInts; i++) {
-      printer->Print("int to_$bit_field_name$ = 0;\n",
-        "bit_field_name", GetBitFieldName(i));
-    }
+  // Local vars for from and to bit fields to avoid accessing the builder and
+  // message over and over for these fields. Seems to provide a slight
+  // perforamance improvement in micro benchmark and this is also what proto1
+  // code does.
+  for (int i = 0; i < totalBuilderInts; i++) {
+    printer->Print("int from_$bit_field_name$ = $bit_field_name$;\n",
+                   "bit_field_name", GetBitFieldName(i));
+  }
+  for (int i = 0; i < totalMessageInts; i++) {
+    printer->Print("int to_$bit_field_name$ = 0;\n", "bit_field_name",
+                   GetBitFieldName(i));
   }
 
   // Output generation code for each field.
@@ -428,12 +419,10 @@ GenerateCommonBuilderMethods(io::Printer* printer) {
     field_generators_.get(descriptor_->field(i)).GenerateBuildingCode(printer);
   }
 
-  if (GenerateHasBits(descriptor_)) {
-    // Copy the bit field results to the generated message
-    for (int i = 0; i < totalMessageInts; i++) {
-      printer->Print("result.$bit_field_name$ = to_$bit_field_name$;\n",
-        "bit_field_name", GetBitFieldName(i));
-    }
+  // Copy the bit field results to the generated message
+  for (int i = 0; i < totalMessageInts; i++) {
+    printer->Print("result.$bit_field_name$ = to_$bit_field_name$;\n",
+                   "bit_field_name", GetBitFieldName(i));
   }
 
   for (int i = 0; i < descriptor_->oneof_decl_count(); i++) {

src/google/protobuf/compiler/java/java_message_builder_lite.cc

@@ -59,13 +59,6 @@ namespace protobuf {
 namespace compiler {
 namespace java {
 
-namespace {
-bool GenerateHasBits(const Descriptor* descriptor) {
-  return SupportFieldPresence(descriptor->file()) ||
-      HasRepeatedFields(descriptor);
-}
-}  // namespace
-
 MessageBuilderLiteGenerator::MessageBuilderLiteGenerator(
     const Descriptor* descriptor, Context* context)
   : descriptor_(descriptor), context_(context),

src/google/protobuf/compiler/java/java_message_field.cc

@@ -137,11 +137,11 @@ ImmutableMessageFieldGenerator(const FieldDescriptor* descriptor,
 ImmutableMessageFieldGenerator::~ImmutableMessageFieldGenerator() {}
 
 int ImmutableMessageFieldGenerator::GetNumBitsForMessage() const {
-  return 1;
+  return SupportFieldPresence(descriptor_->file()) ? 1 : 0;
 }
 
 int ImmutableMessageFieldGenerator::GetNumBitsForBuilder() const {
-  return 1;
+  return GetNumBitsForMessage();
 }
 
 void ImmutableMessageFieldGenerator::

src/google/protobuf/compiler/java/java_message_field_lite.cc

@@ -119,7 +119,7 @@ ImmutableMessageFieldLiteGenerator::ImmutableMessageFieldLiteGenerator(
 ImmutableMessageFieldLiteGenerator::~ImmutableMessageFieldLiteGenerator() {}
 
 int ImmutableMessageFieldLiteGenerator::GetNumBitsForMessage() const {
-  return 1;
+  return SupportFieldPresence(descriptor_->file()) ? 1 : 0;
 }
 
 void ImmutableMessageFieldLiteGenerator::

src/google/protobuf/compiler/java/java_message_lite.cc

@@ -74,11 +74,6 @@ bool EnableExperimentalRuntimeForLite() {
 #endif  // !PROTOBUF_EXPERIMENT
 }
 
-bool GenerateHasBits(const Descriptor* descriptor) {
-  return SupportFieldPresence(descriptor->file()) ||
-      HasRepeatedFields(descriptor);
-}
-
 string MapValueImmutableClassdName(const Descriptor* descriptor,
                                    ClassNameResolver* name_resolver) {
   const FieldDescriptor* value_field = descriptor->FindFieldByName("value");
@@ -233,18 +228,16 @@ void ImmutableMessageLiteGenerator::Generate(io::Printer* printer) {
     messageGenerator.Generate(printer);
   }
 
-  if (GenerateHasBits(descriptor_)) {
-    // Integers for bit fields.
-    int totalBits = 0;
-    for (int i = 0; i < descriptor_->field_count(); i++) {
-      totalBits += field_generators_.get(descriptor_->field(i))
-          .GetNumBitsForMessage();
-    }
-    int totalInts = (totalBits + 31) / 32;
-    for (int i = 0; i < totalInts; i++) {
-      printer->Print("private int $bit_field_name$;\n",
-        "bit_field_name", GetBitFieldName(i));
-    }
+  // Integers for bit fields.
+  int totalBits = 0;
+  for (int i = 0; i < descriptor_->field_count(); i++) {
+    totalBits +=
+        field_generators_.get(descriptor_->field(i)).GetNumBitsForMessage();
+  }
+  int totalInts = (totalBits + 31) / 32;
+  for (int i = 0; i < totalInts; i++) {
+    printer->Print("private int $bit_field_name$;\n", "bit_field_name",
+                   GetBitFieldName(i));
   }
 
   // oneof
@@ -952,20 +945,17 @@ void ImmutableMessageLiteGenerator::GenerateDynamicMethodVisit(
       "oneof_name", context_->GetOneofGeneratorInfo(field)->name);
   }
 
-  if (GenerateHasBits(descriptor_)) {
-    // Integers for bit fields.
-    int totalBits = 0;
-    for (int i = 0; i < descriptor_->field_count(); i++) {
-      totalBits += field_generators_.get(descriptor_->field(i))
-          .GetNumBitsForMessage();
-    }
-    int totalInts = (totalBits + 31) / 32;
+  // Integers for bit fields.
+  int totalBits = 0;
+  for (int i = 0; i < descriptor_->field_count(); i++) {
+    totalBits +=
+        field_generators_.get(descriptor_->field(i)).GetNumBitsForMessage();
+  }
+  int totalInts = (totalBits + 31) / 32;
 
-    for (int i = 0; i < totalInts; i++) {
-      printer->Print(
-        "$bit_field_name$ |= other.$bit_field_name$;\n",
-        "bit_field_name", GetBitFieldName(i));
-    }
+  for (int i = 0; i < totalInts; i++) {
+    printer->Print("$bit_field_name$ |= other.$bit_field_name$;\n",
+                   "bit_field_name", GetBitFieldName(i));
   }
   printer->Outdent();
   printer->Print(

src/google/protobuf/compiler/java/java_primitive_field.cc

@@ -201,11 +201,11 @@ ImmutablePrimitiveFieldGenerator(const FieldDescriptor* descriptor,
 ImmutablePrimitiveFieldGenerator::~ImmutablePrimitiveFieldGenerator() {}
 
 int ImmutablePrimitiveFieldGenerator::GetNumBitsForMessage() const {
-  return 1;
+  return SupportFieldPresence(descriptor_->file()) ? 1 : 0;
 }
 
 int ImmutablePrimitiveFieldGenerator::GetNumBitsForBuilder() const {
-  return 1;
+  return GetNumBitsForMessage();
 }
 
 void ImmutablePrimitiveFieldGenerator::

src/google/protobuf/compiler/java/java_primitive_field_lite.cc

@@ -187,7 +187,7 @@ ImmutablePrimitiveFieldLiteGenerator::ImmutablePrimitiveFieldLiteGenerator(
 ImmutablePrimitiveFieldLiteGenerator::~ImmutablePrimitiveFieldLiteGenerator() {}
 
 int ImmutablePrimitiveFieldLiteGenerator::GetNumBitsForMessage() const {
-  return 1;
+  return SupportFieldPresence(descriptor_->file()) ? 1 : 0;
 }
 
 void ImmutablePrimitiveFieldLiteGenerator::

src/google/protobuf/compiler/java/java_service.cc

@@ -304,8 +304,10 @@ void ImmutableServiceGenerator::GenerateGetPrototype(RequestOrResponse which,
     const MethodDescriptor* method = descriptor_->method(i);
     std::map<string, string> vars;
     vars["index"] = StrCat(i);
-    vars["type"] = name_resolver_->GetImmutableClassName(
-      (which == REQUEST) ? method->input_type() : method->output_type());
+    vars["type"] =
+        (which == REQUEST)
+            ? name_resolver_->GetImmutableClassName(method->input_type())
+            : GetOutput(method);
     printer->Print(vars,
       "case $index$:\n"
       "  return $type$.getDefaultInstance();\n");

src/google/protobuf/compiler/java/java_string_field.cc

@@ -154,11 +154,11 @@ ImmutableStringFieldGenerator(const FieldDescriptor* descriptor,
 ImmutableStringFieldGenerator::~ImmutableStringFieldGenerator() {}
 
 int ImmutableStringFieldGenerator::GetNumBitsForMessage() const {
-  return 1;
+  return SupportFieldPresence(descriptor_->file()) ? 1 : 0;
 }
 
 int ImmutableStringFieldGenerator::GetNumBitsForBuilder() const {
-  return 1;
+  return GetNumBitsForMessage();
 }
 
 // A note about how strings are handled. This code used to just store a String

src/google/protobuf/compiler/java/java_string_field_lite.cc

@@ -134,7 +134,7 @@ ImmutableStringFieldLiteGenerator::ImmutableStringFieldLiteGenerator(
 ImmutableStringFieldLiteGenerator::~ImmutableStringFieldLiteGenerator() {}
 
 int ImmutableStringFieldLiteGenerator::GetNumBitsForMessage() const {
-  return 1;
+  return SupportFieldPresence(descriptor_->file()) ? 1 : 0;
 }
 
 // A note about how strings are handled. In the SPEED and CODE_SIZE runtimes,

src/google/protobuf/compiler/js/js_generator.cc

@@ -1135,6 +1135,10 @@ string JSBinaryReaderMethodName(const GeneratorOptions& options,
 
 string JSBinaryWriterMethodName(const GeneratorOptions& options,
                                 const FieldDescriptor* field) {
+  if (field->containing_type() &&
+      field->containing_type()->options().message_set_wire_format()) {
+    return "jspb.BinaryWriter.prototype.writeMessageSet";
+  }
   return "jspb.BinaryWriter.prototype.write" +
          JSBinaryReadWriteMethodName(field, /* is_writer = */ true);
 }
@@ -2243,17 +2247,17 @@ void Generator::GenerateClassToObject(const GeneratorOptions& options,
       "\n"
       "if (jspb.Message.GENERATE_TO_OBJECT) {\n"
       "/**\n"
-      " * Creates an object representation of this proto suitable for use in "
-      "Soy templates.\n"
+      " * Creates an object representation of this proto.\n"
       " * Field names that are reserved in JavaScript and will be renamed to "
       "pb_name.\n"
+      " * Optional fields that are not set will be set to undefined.\n"
       " * To access a reserved field use, foo.pb_<name>, eg, foo.pb_default.\n"
       " * For the list of reserved names please see:\n"
-      " *     com.google.apps.jspb.JsClassTemplate.JS_RESERVED_WORDS.\n"
-      " * @param {boolean=} opt_includeInstance Whether to include the JSPB "
-      "instance\n"
-      " *     for transitional soy proto support: http://goto/soy-param-"
-      "migration\n"
+      " *     net/proto2/compiler/js/internal/generator.cc#kKeyword.\n"
+      " * @param {boolean=} opt_includeInstance Deprecated. whether to include "
+      "the\n"
+      " *     JSPB instance for transitional soy proto support:\n"
+      " *     http://goto/soy-param-migration\n"
       " * @return {!Object}\n"
       " */\n"
       "$classname$.prototype.toObject = function(opt_includeInstance) {\n"
@@ -2263,16 +2267,16 @@ void Generator::GenerateClassToObject(const GeneratorOptions& options,
       "\n"
       "/**\n"
       " * Static version of the {@see toObject} method.\n"
-      " * @param {boolean|undefined} includeInstance Whether to include the "
-      "JSPB\n"
-      " *     instance for transitional soy proto support:\n"
+      " * @param {boolean|undefined} includeInstance Deprecated. Whether to "
+      "include\n"
+      " *     the JSPB instance for transitional soy proto support:\n"
       " *     http://goto/soy-param-migration\n"
       " * @param {!$classname$} msg The msg instance to transform.\n"
       " * @return {!Object}\n"
       " * @suppress {unusedLocalVariables} f is only used for nested messages\n"
       " */\n"
       "$classname$.toObject = function(includeInstance, msg) {\n"
-      "  var obj = {",
+      "  var f, obj = {",
       "classname", GetMessagePath(options, desc));
 
   bool first = true;
@@ -2424,7 +2428,39 @@ void Generator::GenerateClassFieldToObject(const GeneratorOptions& options,
     // We are migrating the accessors to return defaults instead of null, but
     // it may take longer to migrate toObject (or we might not want to do it at
     // all).  So we want to generate independent code.
+    // The accessor for unset optional values without default should return
+    // null. Those are converted to undefined in the generated object.
+    printer->Print("(f = ");
     GenerateFieldValueExpression(printer, "msg", field, use_default);
+    printer->Print(") == null ? undefined : f");
+  }
+}
+
+void Generator::GenerateObjectTypedef(const GeneratorOptions& options,
+                                      io::Printer* printer,
+                                      const Descriptor* desc) const {
+  // TODO(b/122687752): Consider renaming nested messages called ObjectFormat
+  //     to prevent collisions.
+  const string type_name = GetMessagePath(options, desc) + ".ObjectFormat";
+
+  printer->Print(
+      "/**\n"
+      " * The raw object form of $messageName$ as accepted by the `fromObject` "
+      "method.\n"
+      " * @record\n"
+      " */\n"
+      "$typeName$ = function() {};\n\n",
+      "messageName", desc->name(),
+      "typeName", type_name);
+
+  for (int i = 0; i < desc->field_count(); i++) {
+    printer->Print(
+        "/** @type {$fieldType$|undefined} */\n"
+        "$typeName$.prototype.$fieldName$;\n\n",
+        "typeName", type_name,
+        "fieldName", JSObjectFieldName(options, desc->field(i)),
+        // TODO(b/121097361): Add type checking for field values.
+        "fieldType", "?");
   }
 }
 
@@ -2432,15 +2468,16 @@ void Generator::GenerateClassFromObject(const GeneratorOptions& options,
                                         io::Printer* printer,
                                         const Descriptor* desc) const {
   printer->Print(
-      "if (jspb.Message.GENERATE_FROM_OBJECT) {\n"
+      "if (jspb.Message.GENERATE_FROM_OBJECT) {\n\n");
+
+  GenerateObjectTypedef(options, printer, desc);
+
+  printer->Print(
       "/**\n"
       " * Loads data from an object into a new instance of this proto.\n"
-      " * @param {!Object} obj The object representation of this proto to\n"
-      " *     load the data from.\n"
+      " * @param {!$classname$.ObjectFormat} obj\n"
+      " *     The object representation of this proto to load the data from.\n"
       " * @return {!$classname$}\n"
-      " * @suppress {missingProperties} To prevent JSCompiler errors at "
-      "the\n"
-      " *     `goog.isDef(obj.<fieldName>)` lookups.\n"
       " */\n"
       "$classname$.fromObject = function(obj) {\n"
       "  var msg = new $classname$();\n",
@@ -2456,7 +2493,7 @@ void Generator::GenerateClassFromObject(const GeneratorOptions& options,
   printer->Print(
       "  return msg;\n"
       "};\n"
-      "}\n");
+      "}\n\n");
 }
 
 void Generator::GenerateClassFieldFromObject(
@@ -2469,7 +2506,7 @@ void Generator::GenerateClassFieldFromObject(
       // Since the map values are of message type, we have to do some extra work
       // to recursively call fromObject() on them before setting the map field.
       printer->Print(
-          "  goog.isDef(obj.$name$) && jspb.Message.setWrapperField(\n"
+          "  obj.$name$ && jspb.Message.setWrapperField(\n"
           "      msg, $index$, jspb.Map.fromObject(obj.$name$, $fieldclass$, "
           "$fieldclass$.fromObject));\n",
           "name", JSObjectFieldName(options, field),
@@ -2480,7 +2517,7 @@ void Generator::GenerateClassFieldFromObject(
       // map containers wrapping underlying arrays, so we can simply directly
       // set the array here without fear of a stale wrapper.
       printer->Print(
-          "  goog.isDef(obj.$name$) && "
+          "  obj.$name$ && "
           "jspb.Message.setField(msg, $index$, obj.$name$);\n",
           "name", JSObjectFieldName(options, field),
           "index", JSFieldIndex(field));
@@ -2490,7 +2527,7 @@ void Generator::GenerateClassFieldFromObject(
     if (field->is_repeated()) {
       {
         printer->Print(
-            "  goog.isDef(obj.$name$) && "
+            "  obj.$name$ && "
             "jspb.Message.setRepeatedWrapperField(\n"
             "      msg, $index$, obj.$name$.map(\n"
             "          $fieldclass$.fromObject));\n",
@@ -2500,7 +2537,7 @@ void Generator::GenerateClassFieldFromObject(
       }
     } else {
       printer->Print(
-          "  goog.isDef(obj.$name$) && jspb.Message.setWrapperField(\n"
+          "  obj.$name$ && jspb.Message.setWrapperField(\n"
           "      msg, $index$, $fieldclass$.fromObject(obj.$name$));\n",
           "name", JSObjectFieldName(options, field),
           "index", JSFieldIndex(field),
@@ -2509,7 +2546,7 @@ void Generator::GenerateClassFieldFromObject(
   } else {
     // Simple (primitive) field.
     printer->Print(
-        "  goog.isDef(obj.$name$) && jspb.Message.setField(msg, $index$, "
+        "  obj.$name$ != null && jspb.Message.setField(msg, $index$, "
         "obj.$name$);\n",
         "name", JSObjectFieldName(options, field),
         "index", JSFieldIndex(field));

src/google/protobuf/compiler/js/js_generator.h

@@ -272,6 +272,9 @@ class PROTOC_EXPORT Generator : public CodeGenerator {
   void GenerateOneofCaseDefinition(const GeneratorOptions& options,
                                    io::Printer* printer,
                                    const OneofDescriptor* oneof) const;
+  void GenerateObjectTypedef(const GeneratorOptions& options,
+                             io::Printer* printer,
+                             const Descriptor* desc) const;
   void GenerateClassToObject(const GeneratorOptions& options,
                              io::Printer* printer,
                              const Descriptor* desc) const;

src/google/protobuf/compiler/plugin.pb.cc

@@ -107,9 +107,9 @@ void InitDefaults_google_2fprotobuf_2fcompiler_2fplugin_2eproto() {
   ::google::protobuf::internal::InitSCC(&scc_info_CodeGeneratorResponse_google_2fprotobuf_2fcompiler_2fplugin_2eproto.base);
 }
 
-::google::protobuf::Metadata file_level_metadata_google_2fprotobuf_2fcompiler_2fplugin_2eproto[4];
-constexpr ::google::protobuf::EnumDescriptor const** file_level_enum_descriptors_google_2fprotobuf_2fcompiler_2fplugin_2eproto = nullptr;
-constexpr ::google::protobuf::ServiceDescriptor const** file_level_service_descriptors_google_2fprotobuf_2fcompiler_2fplugin_2eproto = nullptr;
+static ::google::protobuf::Metadata file_level_metadata_google_2fprotobuf_2fcompiler_2fplugin_2eproto[4];
+static constexpr ::google::protobuf::EnumDescriptor const** file_level_enum_descriptors_google_2fprotobuf_2fcompiler_2fplugin_2eproto = nullptr;
+static constexpr ::google::protobuf::ServiceDescriptor const** file_level_service_descriptors_google_2fprotobuf_2fcompiler_2fplugin_2eproto = nullptr;
 
 const ::google::protobuf::uint32 TableStruct_google_2fprotobuf_2fcompiler_2fplugin_2eproto::offsets[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = {
   PROTOBUF_FIELD_OFFSET(::google::protobuf::compiler::Version, _has_bits_),
@@ -173,7 +173,7 @@ static ::google::protobuf::Message const * const file_default_instances[] = {
   reinterpret_cast<const ::google::protobuf::Message*>(&::google::protobuf::compiler::_CodeGeneratorResponse_default_instance_),
 };
 
-::google::protobuf::internal::AssignDescriptorsTable assign_descriptors_table_google_2fprotobuf_2fcompiler_2fplugin_2eproto = {
+static ::google::protobuf::internal::AssignDescriptorsTable assign_descriptors_table_google_2fprotobuf_2fcompiler_2fplugin_2eproto = {
   {}, AddDescriptors_google_2fprotobuf_2fcompiler_2fplugin_2eproto, "google/protobuf/compiler/plugin.proto", schemas,
   file_default_instances, TableStruct_google_2fprotobuf_2fcompiler_2fplugin_2eproto::offsets,
   file_level_metadata_google_2fprotobuf_2fcompiler_2fplugin_2eproto, 4, file_level_enum_descriptors_google_2fprotobuf_2fcompiler_2fplugin_2eproto, file_level_service_descriptors_google_2fprotobuf_2fcompiler_2fplugin_2eproto,
@@ -197,7 +197,7 @@ const char descriptor_table_protodef_google_2fprotobuf_2fcompiler_2fplugin_2epro
   "pilerB\014PluginProtosZ9github.com/golang/p"
   "rotobuf/protoc-gen-go/plugin;plugin_go"
   ;
-::google::protobuf::internal::DescriptorTable descriptor_table_google_2fprotobuf_2fcompiler_2fplugin_2eproto = {
+static ::google::protobuf::internal::DescriptorTable descriptor_table_google_2fprotobuf_2fcompiler_2fplugin_2eproto = {
   false, InitDefaults_google_2fprotobuf_2fcompiler_2fplugin_2eproto, 
   descriptor_table_protodef_google_2fprotobuf_2fcompiler_2fplugin_2eproto,
   "google/protobuf/compiler/plugin.proto", &assign_descriptors_table_google_2fprotobuf_2fcompiler_2fplugin_2eproto, 638,
@@ -311,77 +311,54 @@ void Version::Clear() {
 }
 
 #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
-const char* Version::_InternalParse(const char* begin, const char* end, void* object,
-                  ::google::protobuf::internal::ParseContext* ctx) {
-  auto msg = static_cast<Version*>(object);
-  ::google::protobuf::int32 size; (void)size;
-  int depth; (void)depth;
-  ::google::protobuf::uint32 tag;
-  ::google::protobuf::internal::ParseFunc parser_till_end; (void)parser_till_end;
-  auto ptr = begin;
-  while (ptr < end) {
-    ptr = ::google::protobuf::io::Parse32(ptr, &tag);
+const char* Version::_InternalParse(const char* ptr, ::google::protobuf::internal::ParseContext* ctx) {
+  while (!ctx->Done(&ptr)) {
+    ::google::protobuf::uint32 tag;
+    ptr = ::google::protobuf::internal::ReadTag(ptr, &tag);
     GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
     switch (tag >> 3) {
       // optional int32 major = 1;
       case 1: {
         if (static_cast<::google::protobuf::uint8>(tag) != 8) goto handle_unusual;
-        msg->set_major(::google::protobuf::internal::ReadVarint(&ptr));
+        set_major(::google::protobuf::internal::ReadVarint(&ptr));
         GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
         break;
       }
       // optional int32 minor = 2;
       case 2: {
         if (static_cast<::google::protobuf::uint8>(tag) != 16) goto handle_unusual;
-        msg->set_minor(::google::protobuf::internal::ReadVarint(&ptr));
+        set_minor(::google::protobuf::internal::ReadVarint(&ptr));
         GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
         break;
       }
       // optional int32 patch = 3;
       case 3: {
         if (static_cast<::google::protobuf::uint8>(tag) != 24) goto handle_unusual;
-        msg->set_patch(::google::protobuf::internal::ReadVarint(&ptr));
+        set_patch(::google::protobuf::internal::ReadVarint(&ptr));
         GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
         break;
       }
       // optional string suffix = 4;
       case 4: {
         if (static_cast<::google::protobuf::uint8>(tag) != 34) goto handle_unusual;
-        ptr = ::google::protobuf::io::ReadSize(ptr, &size);
+        ptr = ::google::protobuf::internal::InlineGreedyStringParserUTF8Verify(mutable_suffix(), ptr, ctx, "google.protobuf.compiler.Version.suffix");
         GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
-        ctx->extra_parse_data().SetFieldName("google.protobuf.compiler.Version.suffix");
-        object = msg->mutable_suffix();
-        if (size > end - ptr + ::google::protobuf::internal::ParseContext::kSlopBytes) {
-          parser_till_end = ::google::protobuf::internal::GreedyStringParserUTF8Verify;
-          goto string_till_end;
-        }
-        GOOGLE_PROTOBUF_PARSER_ASSERT(::google::protobuf::internal::StringCheckUTF8Verify(ptr, size, ctx));
-        ::google::protobuf::internal::InlineGreedyStringParser(object, ptr, size, ctx);
-        ptr += size;
         break;
       }
       default: {
       handle_unusual:
         if ((tag & 7) == 4 || tag == 0) {
-          ctx->EndGroup(tag);
+          ctx->SetLastTag(tag);
           return ptr;
         }
-        auto res = UnknownFieldParse(tag, {_InternalParse, msg},
-          ptr, end, msg->_internal_metadata_.mutable_unknown_fields(), ctx);
-        ptr = res.first;
+        ptr = UnknownFieldParse(tag,
+          _internal_metadata_.mutable_unknown_fields(), ptr, ctx);
         GOOGLE_PROTOBUF_PARSER_ASSERT(ptr != nullptr);
-        if (res.second) return ptr;
+        break;
       }
     }  // switch
   }  // while
   return ptr;
-string_till_end:
-  static_cast<::std::string*>(object)->clear();
-  static_cast<::std::string*>(object)->reserve(size);
-  goto len_delim_till_end;
-len_delim_till_end:
-  return ctx->StoreAndTailCall(ptr, end, {_InternalParse, msg},
-                               {parser_till_end, object}, size);
 }
 #else  // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
 bool Version::MergePartialFromCodedStream(
@@ -782,104 +759,60 @@ void CodeGeneratorRequest::Clear() {
 }
 
 #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
-const char* CodeGeneratorRequest::_InternalParse(const char* begin, const char* end, void* object,
-                  ::google::protobuf::internal::ParseContext* ctx) {
-  auto msg = static_cast<CodeGeneratorRequest*>(object);
-  ::google::protobuf::int32 size; (void)size;
-  int depth; (void)depth;
-  ::google::protobuf::uint32 tag;
-  ::google::protobuf::internal::ParseFunc parser_till_end; (void)parser_till_end;
-  auto ptr = begin;
-  while (ptr < end) {
-    ptr = ::google::protobuf::io::Parse32(ptr, &tag);
+const char* CodeGeneratorRequest::_InternalParse(const char* ptr, ::google::protobuf::internal::ParseContext* ctx) {
+  while (!ctx->Done(&ptr)) {
+    ::google::protobuf::uint32 tag;
+    ptr = ::google::protobuf::internal::ReadTag(ptr, &tag);
     GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
     switch (tag >> 3) {
       // repeated string file_to_generate = 1;
       case 1: {
         if (static_cast<::google::protobuf::uint8>(tag) != 10) goto handle_unusual;
         do {
-          ptr = ::google::protobuf::io::ReadSize(ptr, &size);
+          ptr = ::google::protobuf::internal::InlineGreedyStringParserUTF8Verify(add_file_to_generate(), ptr, ctx, "google.protobuf.compiler.CodeGeneratorRequest.file_to_generate");
           GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
-          ctx->extra_parse_data().SetFieldName("google.protobuf.compiler.CodeGeneratorRequest.file_to_generate");
-          object = msg->add_file_to_generate();
-          if (size > end - ptr + ::google::protobuf::internal::ParseContext::kSlopBytes) {
-            parser_till_end = ::google::protobuf::internal::GreedyStringParserUTF8Verify;
-            goto string_till_end;
-          }
-          GOOGLE_PROTOBUF_PARSER_ASSERT(::google::protobuf::internal::StringCheckUTF8Verify(ptr, size, ctx));
-          ::google::protobuf::internal::InlineGreedyStringParser(object, ptr, size, ctx);
-          ptr += size;
-          if (ptr >= end) break;
-        } while ((::google::protobuf::io::UnalignedLoad<::google::protobuf::uint64>(ptr) & 255) == 10 && (ptr += 1));
+          if (ctx->Done(&ptr)) return ptr;
+        } while ((::google::protobuf::internal::UnalignedLoad<::google::protobuf::uint64>(ptr) & 255) == 10 && (ptr += 1));
         break;
       }
       // optional string parameter = 2;
       case 2: {
         if (static_cast<::google::protobuf::uint8>(tag) != 18) goto handle_unusual;
-        ptr = ::google::protobuf::io::ReadSize(ptr, &size);
+        ptr = ::google::protobuf::internal::InlineGreedyStringParserUTF8Verify(mutable_parameter(), ptr, ctx, "google.protobuf.compiler.CodeGeneratorRequest.parameter");
         GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
-        ctx->extra_parse_data().SetFieldName("google.protobuf.compiler.CodeGeneratorRequest.parameter");
-        object = msg->mutable_parameter();
-        if (size > end - ptr + ::google::protobuf::internal::ParseContext::kSlopBytes) {
-          parser_till_end = ::google::protobuf::internal::GreedyStringParserUTF8Verify;
-          goto string_till_end;
-        }
-        GOOGLE_PROTOBUF_PARSER_ASSERT(::google::protobuf::internal::StringCheckUTF8Verify(ptr, size, ctx));
-        ::google::protobuf::internal::InlineGreedyStringParser(object, ptr, size, ctx);
-        ptr += size;
         break;
       }
       // optional .google.protobuf.compiler.Version compiler_version = 3;
       case 3: {
         if (static_cast<::google::protobuf::uint8>(tag) != 26) goto handle_unusual;
-        ptr = ::google::protobuf::io::ReadSize(ptr, &size);
+        ptr = ctx->ParseMessage(mutable_compiler_version(), ptr);
         GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
-        parser_till_end = ::google::protobuf::compiler::Version::_InternalParse;
-        object = msg->mutable_compiler_version();
-        if (size > end - ptr) goto len_delim_till_end;
-        ptr += size;
-        GOOGLE_PROTOBUF_PARSER_ASSERT(ctx->ParseExactRange(
-            {parser_till_end, object}, ptr - size, ptr));
         break;
       }
       // repeated .google.protobuf.FileDescriptorProto proto_file = 15;
       case 15: {
         if (static_cast<::google::protobuf::uint8>(tag) != 122) goto handle_unusual;
         do {
-          ptr = ::google::protobuf::io::ReadSize(ptr, &size);
+          ptr = ctx->ParseMessage(add_proto_file(), ptr);
           GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
-          parser_till_end = ::google::protobuf::FileDescriptorProto::_InternalParse;
-          object = msg->add_proto_file();
-          if (size > end - ptr) goto len_delim_till_end;
-          ptr += size;
-          GOOGLE_PROTOBUF_PARSER_ASSERT(ctx->ParseExactRange(
-              {parser_till_end, object}, ptr - size, ptr));
-          if (ptr >= end) break;
-        } while ((::google::protobuf::io::UnalignedLoad<::google::protobuf::uint64>(ptr) & 255) == 122 && (ptr += 1));
+          if (ctx->Done(&ptr)) return ptr;
+        } while ((::google::protobuf::internal::UnalignedLoad<::google::protobuf::uint64>(ptr) & 255) == 122 && (ptr += 1));
         break;
       }
       default: {
       handle_unusual:
         if ((tag & 7) == 4 || tag == 0) {
-          ctx->EndGroup(tag);
+          ctx->SetLastTag(tag);
           return ptr;
         }
-        auto res = UnknownFieldParse(tag, {_InternalParse, msg},
-          ptr, end, msg->_internal_metadata_.mutable_unknown_fields(), ctx);
-        ptr = res.first;
+        ptr = UnknownFieldParse(tag,
+          _internal_metadata_.mutable_unknown_fields(), ptr, ctx);
         GOOGLE_PROTOBUF_PARSER_ASSERT(ptr != nullptr);
-        if (res.second) return ptr;
+        break;
       }
     }  // switch
   }  // while
   return ptr;
-string_till_end:
-  static_cast<::std::string*>(object)->clear();
-  static_cast<::std::string*>(object)->reserve(size);
-  goto len_delim_till_end;
-len_delim_till_end:
-  return ctx->StoreAndTailCall(ptr, end, {_InternalParse, msg},
-                               {parser_till_end, object}, size);
 }
 #else  // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
 bool CodeGeneratorRequest::MergePartialFromCodedStream(
@@ -1295,88 +1228,47 @@ void CodeGeneratorResponse_File::Clear() {
 }
 
 #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
-const char* CodeGeneratorResponse_File::_InternalParse(const char* begin, const char* end, void* object,
-                  ::google::protobuf::internal::ParseContext* ctx) {
-  auto msg = static_cast<CodeGeneratorResponse_File*>(object);
-  ::google::protobuf::int32 size; (void)size;
-  int depth; (void)depth;
-  ::google::protobuf::uint32 tag;
-  ::google::protobuf::internal::ParseFunc parser_till_end; (void)parser_till_end;
-  auto ptr = begin;
-  while (ptr < end) {
-    ptr = ::google::protobuf::io::Parse32(ptr, &tag);
+const char* CodeGeneratorResponse_File::_InternalParse(const char* ptr, ::google::protobuf::internal::ParseContext* ctx) {
+  while (!ctx->Done(&ptr)) {
+    ::google::protobuf::uint32 tag;
+    ptr = ::google::protobuf::internal::ReadTag(ptr, &tag);
     GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
     switch (tag >> 3) {
       // optional string name = 1;
       case 1: {
         if (static_cast<::google::protobuf::uint8>(tag) != 10) goto handle_unusual;
-        ptr = ::google::protobuf::io::ReadSize(ptr, &size);
+        ptr = ::google::protobuf::internal::InlineGreedyStringParserUTF8Verify(mutable_name(), ptr, ctx, "google.protobuf.compiler.CodeGeneratorResponse.File.name");
         GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
-        ctx->extra_parse_data().SetFieldName("google.protobuf.compiler.CodeGeneratorResponse.File.name");
-        object = msg->mutable_name();
-        if (size > end - ptr + ::google::protobuf::internal::ParseContext::kSlopBytes) {
-          parser_till_end = ::google::protobuf::internal::GreedyStringParserUTF8Verify;
-          goto string_till_end;
-        }
-        GOOGLE_PROTOBUF_PARSER_ASSERT(::google::protobuf::internal::StringCheckUTF8Verify(ptr, size, ctx));
-        ::google::protobuf::internal::InlineGreedyStringParser(object, ptr, size, ctx);
-        ptr += size;
         break;
       }
       // optional string insertion_point = 2;
       case 2: {
         if (static_cast<::google::protobuf::uint8>(tag) != 18) goto handle_unusual;
-        ptr = ::google::protobuf::io::ReadSize(ptr, &size);
+        ptr = ::google::protobuf::internal::InlineGreedyStringParserUTF8Verify(mutable_insertion_point(), ptr, ctx, "google.protobuf.compiler.CodeGeneratorResponse.File.insertion_point");
         GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
-        ctx->extra_parse_data().SetFieldName("google.protobuf.compiler.CodeGeneratorResponse.File.insertion_point");
-        object = msg->mutable_insertion_point();
-        if (size > end - ptr + ::google::protobuf::internal::ParseContext::kSlopBytes) {
-          parser_till_end = ::google::protobuf::internal::GreedyStringParserUTF8Verify;
-          goto string_till_end;
-        }
-        GOOGLE_PROTOBUF_PARSER_ASSERT(::google::protobuf::internal::StringCheckUTF8Verify(ptr, size, ctx));
-        ::google::protobuf::internal::InlineGreedyStringParser(object, ptr, size, ctx);
-        ptr += size;
         break;
       }
       // optional string content = 15;
       case 15: {
         if (static_cast<::google::protobuf::uint8>(tag) != 122) goto handle_unusual;
-        ptr = ::google::protobuf::io::ReadSize(ptr, &size);
+        ptr = ::google::protobuf::internal::InlineGreedyStringParserUTF8Verify(mutable_content(), ptr, ctx, "google.protobuf.compiler.CodeGeneratorResponse.File.content");
         GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
-        ctx->extra_parse_data().SetFieldName("google.protobuf.compiler.CodeGeneratorResponse.File.content");
-        object = msg->mutable_content();
-        if (size > end - ptr + ::google::protobuf::internal::ParseContext::kSlopBytes) {
-          parser_till_end = ::google::protobuf::internal::GreedyStringParserUTF8Verify;
-          goto string_till_end;
-        }
-        GOOGLE_PROTOBUF_PARSER_ASSERT(::google::protobuf::internal::StringCheckUTF8Verify(ptr, size, ctx));
-        ::google::protobuf::internal::InlineGreedyStringParser(object, ptr, size, ctx);
-        ptr += size;
         break;
       }
       default: {
       handle_unusual:
         if ((tag & 7) == 4 || tag == 0) {
-          ctx->EndGroup(tag);
+          ctx->SetLastTag(tag);
           return ptr;
         }
-        auto res = UnknownFieldParse(tag, {_InternalParse, msg},
-          ptr, end, msg->_internal_metadata_.mutable_unknown_fields(), ctx);
-        ptr = res.first;
+        ptr = UnknownFieldParse(tag,
+          _internal_metadata_.mutable_unknown_fields(), ptr, ctx);
         GOOGLE_PROTOBUF_PARSER_ASSERT(ptr != nullptr);
-        if (res.second) return ptr;
+        break;
       }
     }  // switch
   }  // while
   return ptr;
-string_till_end:
-  static_cast<::std::string*>(object)->clear();
-  static_cast<::std::string*>(object)->reserve(size);
-  goto len_delim_till_end;
-len_delim_till_end:
-  return ctx->StoreAndTailCall(ptr, end, {_InternalParse, msg},
-                               {parser_till_end, object}, size);
 }
 #else  // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
 bool CodeGeneratorResponse_File::MergePartialFromCodedStream(
@@ -1742,72 +1634,43 @@ void CodeGeneratorResponse::Clear() {
 }
 
 #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
-const char* CodeGeneratorResponse::_InternalParse(const char* begin, const char* end, void* object,
-                  ::google::protobuf::internal::ParseContext* ctx) {
-  auto msg = static_cast<CodeGeneratorResponse*>(object);
-  ::google::protobuf::int32 size; (void)size;
-  int depth; (void)depth;
-  ::google::protobuf::uint32 tag;
-  ::google::protobuf::internal::ParseFunc parser_till_end; (void)parser_till_end;
-  auto ptr = begin;
-  while (ptr < end) {
-    ptr = ::google::protobuf::io::Parse32(ptr, &tag);
+const char* CodeGeneratorResponse::_InternalParse(const char* ptr, ::google::protobuf::internal::ParseContext* ctx) {
+  while (!ctx->Done(&ptr)) {
+    ::google::protobuf::uint32 tag;
+    ptr = ::google::protobuf::internal::ReadTag(ptr, &tag);
     GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
     switch (tag >> 3) {
       // optional string error = 1;
       case 1: {
         if (static_cast<::google::protobuf::uint8>(tag) != 10) goto handle_unusual;
-        ptr = ::google::protobuf::io::ReadSize(ptr, &size);
+        ptr = ::google::protobuf::internal::InlineGreedyStringParserUTF8Verify(mutable_error(), ptr, ctx, "google.protobuf.compiler.CodeGeneratorResponse.error");
         GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
-        ctx->extra_parse_data().SetFieldName("google.protobuf.compiler.CodeGeneratorResponse.error");
-        object = msg->mutable_error();
-        if (size > end - ptr + ::google::protobuf::internal::ParseContext::kSlopBytes) {
-          parser_till_end = ::google::protobuf::internal::GreedyStringParserUTF8Verify;
-          goto string_till_end;
-        }
-        GOOGLE_PROTOBUF_PARSER_ASSERT(::google::protobuf::internal::StringCheckUTF8Verify(ptr, size, ctx));
-        ::google::protobuf::internal::InlineGreedyStringParser(object, ptr, size, ctx);
-        ptr += size;
         break;
       }
       // repeated .google.protobuf.compiler.CodeGeneratorResponse.File file = 15;
       case 15: {
         if (static_cast<::google::protobuf::uint8>(tag) != 122) goto handle_unusual;
         do {
-          ptr = ::google::protobuf::io::ReadSize(ptr, &size);
+          ptr = ctx->ParseMessage(add_file(), ptr);
           GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
-          parser_till_end = ::google::protobuf::compiler::CodeGeneratorResponse_File::_InternalParse;
-          object = msg->add_file();
-          if (size > end - ptr) goto len_delim_till_end;
-          ptr += size;
-          GOOGLE_PROTOBUF_PARSER_ASSERT(ctx->ParseExactRange(
-              {parser_till_end, object}, ptr - size, ptr));
-          if (ptr >= end) break;
-        } while ((::google::protobuf::io::UnalignedLoad<::google::protobuf::uint64>(ptr) & 255) == 122 && (ptr += 1));
+          if (ctx->Done(&ptr)) return ptr;
+        } while ((::google::protobuf::internal::UnalignedLoad<::google::protobuf::uint64>(ptr) & 255) == 122 && (ptr += 1));
         break;
       }
       default: {
       handle_unusual:
         if ((tag & 7) == 4 || tag == 0) {
-          ctx->EndGroup(tag);
+          ctx->SetLastTag(tag);
           return ptr;
         }
-        auto res = UnknownFieldParse(tag, {_InternalParse, msg},
-          ptr, end, msg->_internal_metadata_.mutable_unknown_fields(), ctx);
-        ptr = res.first;
+        ptr = UnknownFieldParse(tag,
+          _internal_metadata_.mutable_unknown_fields(), ptr, ctx);
         GOOGLE_PROTOBUF_PARSER_ASSERT(ptr != nullptr);
-        if (res.second) return ptr;
+        break;
       }
     }  // switch
   }  // while
   return ptr;
-string_till_end:
-  static_cast<::std::string*>(object)->clear();
-  static_cast<::std::string*>(object)->reserve(size);
-  goto len_delim_till_end;
-len_delim_till_end:
-  return ctx->StoreAndTailCall(ptr, end, {_InternalParse, msg},
-                               {parser_till_end, object}, size);
 }
 #else  // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
 bool CodeGeneratorResponse::MergePartialFromCodedStream(

src/google/protobuf/compiler/plugin.pb.h

@@ -31,6 +31,13 @@
 #include <google/protobuf/repeated_field.h>  // IWYU pragma: export
 #include <google/protobuf/extension_set.h>  // IWYU pragma: export
 #include <google/protobuf/unknown_field_set.h>
+namespace google {
+namespace protobuf {
+namespace internal {
+class AnyMetadata;
+}  // namespace internal
+}  // namespace protobuf
+}  // namespace google
 #include <google/protobuf/descriptor.pb.h>
 // @@protoc_insertion_point(includes)
 #include <google/protobuf/port_def.inc>
@@ -153,8 +160,7 @@ class PROTOC_EXPORT Version final :
 
   size_t ByteSizeLong() const final;
   #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
-  static const char* _InternalParse(const char* begin, const char* end, void* object, ::google::protobuf::internal::ParseContext* ctx);
-  ::google::protobuf::internal::ParseFunc _ParseFunc() const final { return _InternalParse; }
+  const char* _InternalParse(const char* ptr, ::google::protobuf::internal::ParseContext* ctx) final;
   #else
   bool MergePartialFromCodedStream(
       ::google::protobuf::io::CodedInputStream* input) final;
@@ -166,10 +172,14 @@ class PROTOC_EXPORT Version final :
   int GetCachedSize() const final { return _cached_size_.Get(); }
 
   private:
-  void SharedCtor();
-  void SharedDtor();
+  inline void SharedCtor();
+  inline void SharedDtor();
   void SetCachedSize(int size) const final;
   void InternalSwap(Version* other);
+  friend class ::google::protobuf::internal::AnyMetadata;
+  static ::google::protobuf::StringPiece FullMessageName() {
+    return "google.protobuf.compiler.Version";
+  }
   private:
   inline ::google::protobuf::Arena* GetArenaNoVirtual() const {
     return nullptr;
@@ -306,8 +316,7 @@ class PROTOC_EXPORT CodeGeneratorRequest final :
 
   size_t ByteSizeLong() const final;
   #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
-  static const char* _InternalParse(const char* begin, const char* end, void* object, ::google::protobuf::internal::ParseContext* ctx);
-  ::google::protobuf::internal::ParseFunc _ParseFunc() const final { return _InternalParse; }
+  const char* _InternalParse(const char* ptr, ::google::protobuf::internal::ParseContext* ctx) final;
   #else
   bool MergePartialFromCodedStream(
       ::google::protobuf::io::CodedInputStream* input) final;
@@ -319,10 +328,14 @@ class PROTOC_EXPORT CodeGeneratorRequest final :
   int GetCachedSize() const final { return _cached_size_.Get(); }
 
   private:
-  void SharedCtor();
-  void SharedDtor();
+  inline void SharedCtor();
+  inline void SharedDtor();
   void SetCachedSize(int size) const final;
   void InternalSwap(CodeGeneratorRequest* other);
+  friend class ::google::protobuf::internal::AnyMetadata;
+  static ::google::protobuf::StringPiece FullMessageName() {
+    return "google.protobuf.compiler.CodeGeneratorRequest";
+  }
   private:
   inline ::google::protobuf::Arena* GetArenaNoVirtual() const {
     return nullptr;
@@ -481,8 +494,7 @@ class PROTOC_EXPORT CodeGeneratorResponse_File final :
 
   size_t ByteSizeLong() const final;
   #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
-  static const char* _InternalParse(const char* begin, const char* end, void* object, ::google::protobuf::internal::ParseContext* ctx);
-  ::google::protobuf::internal::ParseFunc _ParseFunc() const final { return _InternalParse; }
+  const char* _InternalParse(const char* ptr, ::google::protobuf::internal::ParseContext* ctx) final;
   #else
   bool MergePartialFromCodedStream(
       ::google::protobuf::io::CodedInputStream* input) final;
@@ -494,10 +506,14 @@ class PROTOC_EXPORT CodeGeneratorResponse_File final :
   int GetCachedSize() const final { return _cached_size_.Get(); }
 
   private:
-  void SharedCtor();
-  void SharedDtor();
+  inline void SharedCtor();
+  inline void SharedDtor();
   void SetCachedSize(int size) const final;
   void InternalSwap(CodeGeneratorResponse_File* other);
+  friend class ::google::protobuf::internal::AnyMetadata;
+  static ::google::protobuf::StringPiece FullMessageName() {
+    return "google.protobuf.compiler.CodeGeneratorResponse.File";
+  }
   private:
   inline ::google::protobuf::Arena* GetArenaNoVirtual() const {
     return nullptr;
@@ -642,8 +658,7 @@ class PROTOC_EXPORT CodeGeneratorResponse final :
 
   size_t ByteSizeLong() const final;
   #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
-  static const char* _InternalParse(const char* begin, const char* end, void* object, ::google::protobuf::internal::ParseContext* ctx);
-  ::google::protobuf::internal::ParseFunc _ParseFunc() const final { return _InternalParse; }
+  const char* _InternalParse(const char* ptr, ::google::protobuf::internal::ParseContext* ctx) final;
   #else
   bool MergePartialFromCodedStream(
       ::google::protobuf::io::CodedInputStream* input) final;
@@ -655,10 +670,14 @@ class PROTOC_EXPORT CodeGeneratorResponse final :
   int GetCachedSize() const final { return _cached_size_.Get(); }
 
   private:
-  void SharedCtor();
-  void SharedDtor();
+  inline void SharedCtor();
+  inline void SharedDtor();
   void SetCachedSize(int size) const final;
   void InternalSwap(CodeGeneratorResponse* other);
+  friend class ::google::protobuf::internal::AnyMetadata;
+  static ::google::protobuf::StringPiece FullMessageName() {
+    return "google.protobuf.compiler.CodeGeneratorResponse";
+  }
   private:
   inline ::google::protobuf::Arena* GetArenaNoVirtual() const {
     return nullptr;

src/google/protobuf/compiler/python/python_generator.cc

@@ -814,7 +814,7 @@ void Generator::PrintNestedDescriptors(
 void Generator::PrintMessages() const {
   for (int i = 0; i < file_->message_type_count(); ++i) {
     std::vector<string> to_register;
-    PrintMessage(*file_->message_type(i), "", &to_register);
+    PrintMessage(*file_->message_type(i), "", &to_register, false);
     for (int j = 0; j < to_register.size(); ++j) {
       printer_->Print("_sym_db.RegisterMessage($name$)\n", "name",
                       to_register[j]);
@@ -833,25 +833,33 @@ void Generator::PrintMessages() const {
 // Collect nested message names to_register for the symbol_database.
 void Generator::PrintMessage(const Descriptor& message_descriptor,
                              const string& prefix,
-                             std::vector<string>* to_register) const {
+                             std::vector<string>* to_register,
+                             bool is_nested) const {
   string qualified_name(prefix + message_descriptor.name());
   to_register->push_back(qualified_name);
-  printer_->Print(
-      "$name$ = _reflection.GeneratedProtocolMessageType('$name$', "
-      "(_message.Message,), dict(\n",
-      "name", message_descriptor.name());
+  if (is_nested) {
+    printer_->Print(
+        "'$name$' : _reflection.GeneratedProtocolMessageType('$name$', "
+        "(_message.Message,), {\n",
+        "name", message_descriptor.name());
+  } else {
+    printer_->Print(
+        "$name$ = _reflection.GeneratedProtocolMessageType('$name$', "
+        "(_message.Message,), {\n",
+        "name", message_descriptor.name());
+  }
   printer_->Indent();
 
   PrintNestedMessages(message_descriptor, qualified_name + ".", to_register);
   std::map<string, string> m;
   m["descriptor_key"] = kDescriptorKey;
   m["descriptor_name"] = ModuleLevelDescriptorName(message_descriptor);
-  printer_->Print(m, "$descriptor_key$ = $descriptor_name$,\n");
-  printer_->Print("__module__ = '$module_name$'\n",
+  printer_->Print(m, "'$descriptor_key$' : $descriptor_name$,\n");
+  printer_->Print("'__module__' : '$module_name$'\n",
                   "module_name", ModuleName(file_->name()));
   printer_->Print("# @@protoc_insertion_point(class_scope:$full_name$)\n",
                   "full_name", message_descriptor.full_name());
-  printer_->Print("))\n");
+  printer_->Print("})\n");
   printer_->Outdent();
 }
 
@@ -862,7 +870,8 @@ void Generator::PrintNestedMessages(const Descriptor& containing_descriptor,
                                     std::vector<string>* to_register) const {
   for (int i = 0; i < containing_descriptor.nested_type_count(); ++i) {
     printer_->Print("\n");
-    PrintMessage(*containing_descriptor.nested_type(i), prefix, to_register);
+    PrintMessage(*containing_descriptor.nested_type(i), prefix, to_register,
+                 true);
     printer_->Print(",\n");
   }
 }

src/google/protobuf/compiler/python/python_generator.h

@@ -98,7 +98,7 @@ class PROTOC_EXPORT Generator : public CodeGenerator {
 
   void PrintMessages() const;
   void PrintMessage(const Descriptor& message_descriptor, const std::string& prefix,
-                    std::vector<std::string>* to_register) const;
+                    std::vector<std::string>* to_register, bool is_nested) const;
   void PrintNestedMessages(const Descriptor& containing_descriptor,
                            const std::string& prefix,
                            std::vector<std::string>* to_register) const;

src/google/protobuf/descriptor.proto

@@ -486,7 +486,7 @@ message MessageOptions {
   //
   // Implementations may choose not to generate the map_entry=true message, but
   // use a native map in the target language to hold the keys and values.
-  // The reflection APIs in such implementions still need to work as
+  // The reflection APIs in such implementations still need to work as
   // if the field is a repeated message field.
   //
   // NOTE: Do not set the option in .proto files. Always use the maps syntax
@@ -763,7 +763,7 @@ message SourceCodeInfo {
   //   beginning of the "extend" block and is shared by all extensions within
   //   the block.
   // - Just because a location's span is a subset of some other location's span
-  //   does not mean that it is a descendent.  For example, a "group" defines
+  //   does not mean that it is a descendant.  For example, a "group" defines
   //   both a type and a field in a single declaration.  Thus, the locations
   //   corresponding to the type and field and their components will overlap.
   // - Code which tries to interpret locations should probably be designed to

src/google/protobuf/descriptor_unittest.cc

@@ -2910,9 +2910,8 @@ TEST_P(AllowUnknownDependenciesTest,
   ASSERT_TRUE(pool_->FindMessageTypeByName("undeclared.Quux") == NULL);
 }
 
-INSTANTIATE_TEST_CASE_P(DatabaseSource,
-                        AllowUnknownDependenciesTest,
-                        testing::Values(NO_DATABASE, FALLBACK_DATABASE));
+INSTANTIATE_TEST_SUITE_P(DatabaseSource, AllowUnknownDependenciesTest,
+                         testing::Values(NO_DATABASE, FALLBACK_DATABASE));
 
 // ===================================================================
 

src/google/protobuf/duration.pb.cc

@@ -42,9 +42,9 @@ void InitDefaults_google_2fprotobuf_2fduration_2eproto() {
   ::google::protobuf::internal::InitSCC(&scc_info_Duration_google_2fprotobuf_2fduration_2eproto.base);
 }
 
-::google::protobuf::Metadata file_level_metadata_google_2fprotobuf_2fduration_2eproto[1];
-constexpr ::google::protobuf::EnumDescriptor const** file_level_enum_descriptors_google_2fprotobuf_2fduration_2eproto = nullptr;
-constexpr ::google::protobuf::ServiceDescriptor const** file_level_service_descriptors_google_2fprotobuf_2fduration_2eproto = nullptr;
+static ::google::protobuf::Metadata file_level_metadata_google_2fprotobuf_2fduration_2eproto[1];
+static constexpr ::google::protobuf::EnumDescriptor const** file_level_enum_descriptors_google_2fprotobuf_2fduration_2eproto = nullptr;
+static constexpr ::google::protobuf::ServiceDescriptor const** file_level_service_descriptors_google_2fprotobuf_2fduration_2eproto = nullptr;
 
 const ::google::protobuf::uint32 TableStruct_google_2fprotobuf_2fduration_2eproto::offsets[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = {
   ~0u,  // no _has_bits_
@@ -63,7 +63,7 @@ static ::google::protobuf::Message const * const file_default_instances[] = {
   reinterpret_cast<const ::google::protobuf::Message*>(&::google::protobuf::_Duration_default_instance_),
 };
 
-::google::protobuf::internal::AssignDescriptorsTable assign_descriptors_table_google_2fprotobuf_2fduration_2eproto = {
+static ::google::protobuf::internal::AssignDescriptorsTable assign_descriptors_table_google_2fprotobuf_2fduration_2eproto = {
   {}, AddDescriptors_google_2fprotobuf_2fduration_2eproto, "google/protobuf/duration.proto", schemas,
   file_default_instances, TableStruct_google_2fprotobuf_2fduration_2eproto::offsets,
   file_level_metadata_google_2fprotobuf_2fduration_2eproto, 1, file_level_enum_descriptors_google_2fprotobuf_2fduration_2eproto, file_level_service_descriptors_google_2fprotobuf_2fduration_2eproto,
@@ -77,7 +77,7 @@ const char descriptor_table_protodef_google_2fprotobuf_2fduration_2eproto[] =
   "f/ptypes/duration\370\001\001\242\002\003GPB\252\002\036Google.Prot"
   "obuf.WellKnownTypesb\006proto3"
   ;
-::google::protobuf::internal::DescriptorTable descriptor_table_google_2fprotobuf_2fduration_2eproto = {
+static ::google::protobuf::internal::DescriptorTable descriptor_table_google_2fprotobuf_2fduration_2eproto = {
   false, InitDefaults_google_2fprotobuf_2fduration_2eproto, 
   descriptor_table_protodef_google_2fprotobuf_2fduration_2eproto,
   "google/protobuf/duration.proto", &assign_descriptors_table_google_2fprotobuf_2fduration_2eproto, 227,
@@ -173,43 +173,36 @@ void Duration::Clear() {
 }
 
 #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
-const char* Duration::_InternalParse(const char* begin, const char* end, void* object,
-                  ::google::protobuf::internal::ParseContext* ctx) {
-  auto msg = static_cast<Duration*>(object);
-  ::google::protobuf::int32 size; (void)size;
-  int depth; (void)depth;
-  ::google::protobuf::uint32 tag;
-  ::google::protobuf::internal::ParseFunc parser_till_end; (void)parser_till_end;
-  auto ptr = begin;
-  while (ptr < end) {
-    ptr = ::google::protobuf::io::Parse32(ptr, &tag);
+const char* Duration::_InternalParse(const char* ptr, ::google::protobuf::internal::ParseContext* ctx) {
+  while (!ctx->Done(&ptr)) {
+    ::google::protobuf::uint32 tag;
+    ptr = ::google::protobuf::internal::ReadTag(ptr, &tag);
     GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
     switch (tag >> 3) {
       // int64 seconds = 1;
       case 1: {
         if (static_cast<::google::protobuf::uint8>(tag) != 8) goto handle_unusual;
-        msg->set_seconds(::google::protobuf::internal::ReadVarint(&ptr));
+        set_seconds(::google::protobuf::internal::ReadVarint(&ptr));
         GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
         break;
       }
       // int32 nanos = 2;
       case 2: {
         if (static_cast<::google::protobuf::uint8>(tag) != 16) goto handle_unusual;
-        msg->set_nanos(::google::protobuf::internal::ReadVarint(&ptr));
+        set_nanos(::google::protobuf::internal::ReadVarint(&ptr));
         GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
         break;
       }
       default: {
       handle_unusual:
         if ((tag & 7) == 4 || tag == 0) {
-          ctx->EndGroup(tag);
+          ctx->SetLastTag(tag);
           return ptr;
         }
-        auto res = UnknownFieldParse(tag, {_InternalParse, msg},
-          ptr, end, msg->_internal_metadata_.mutable_unknown_fields(), ctx);
-        ptr = res.first;
+        ptr = UnknownFieldParse(tag,
+          _internal_metadata_.mutable_unknown_fields(), ptr, ctx);
         GOOGLE_PROTOBUF_PARSER_ASSERT(ptr != nullptr);
-        if (res.second) return ptr;
+        break;
       }
     }  // switch
   }  // while

src/google/protobuf/duration.pb.h

@@ -31,6 +31,13 @@
 #include <google/protobuf/repeated_field.h>  // IWYU pragma: export
 #include <google/protobuf/extension_set.h>  // IWYU pragma: export
 #include <google/protobuf/unknown_field_set.h>
+namespace google {
+namespace protobuf {
+namespace internal {
+class AnyMetadata;
+}  // namespace internal
+}  // namespace protobuf
+}  // namespace google
 // @@protoc_insertion_point(includes)
 #include <google/protobuf/port_def.inc>
 #define PROTOBUF_INTERNAL_EXPORT_google_2fprotobuf_2fduration_2eproto PROTOBUF_EXPORT
@@ -131,8 +138,7 @@ class PROTOBUF_EXPORT Duration final :
 
   size_t ByteSizeLong() const final;
   #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
-  static const char* _InternalParse(const char* begin, const char* end, void* object, ::google::protobuf::internal::ParseContext* ctx);
-  ::google::protobuf::internal::ParseFunc _ParseFunc() const final { return _InternalParse; }
+  const char* _InternalParse(const char* ptr, ::google::protobuf::internal::ParseContext* ctx) final;
   #else
   bool MergePartialFromCodedStream(
       ::google::protobuf::io::CodedInputStream* input) final;
@@ -144,10 +150,14 @@ class PROTOBUF_EXPORT Duration final :
   int GetCachedSize() const final { return _cached_size_.Get(); }
 
   private:
-  void SharedCtor();
-  void SharedDtor();
+  inline void SharedCtor();
+  inline void SharedDtor();
   void SetCachedSize(int size) const final;
   void InternalSwap(Duration* other);
+  friend class ::google::protobuf::internal::AnyMetadata;
+  static ::google::protobuf::StringPiece FullMessageName() {
+    return "google.protobuf.Duration";
+  }
   protected:
   explicit Duration(::google::protobuf::Arena* arena);
   private:

src/google/protobuf/dynamic_message_unittest.cc

@@ -316,7 +316,7 @@ TEST_F(DynamicMessageTest, Proto3) {
   delete message;
 }
 
-INSTANTIATE_TEST_CASE_P(UseArena, DynamicMessageTest, ::testing::Bool());
+INSTANTIATE_TEST_SUITE_P(UseArena, DynamicMessageTest, ::testing::Bool());
 
 }  // namespace protobuf
 }  // namespace google

src/google/protobuf/empty.pb.cc

@@ -42,9 +42,9 @@ void InitDefaults_google_2fprotobuf_2fempty_2eproto() {
   ::google::protobuf::internal::InitSCC(&scc_info_Empty_google_2fprotobuf_2fempty_2eproto.base);
 }
 
-::google::protobuf::Metadata file_level_metadata_google_2fprotobuf_2fempty_2eproto[1];
-constexpr ::google::protobuf::EnumDescriptor const** file_level_enum_descriptors_google_2fprotobuf_2fempty_2eproto = nullptr;
-constexpr ::google::protobuf::ServiceDescriptor const** file_level_service_descriptors_google_2fprotobuf_2fempty_2eproto = nullptr;
+static ::google::protobuf::Metadata file_level_metadata_google_2fprotobuf_2fempty_2eproto[1];
+static constexpr ::google::protobuf::EnumDescriptor const** file_level_enum_descriptors_google_2fprotobuf_2fempty_2eproto = nullptr;
+static constexpr ::google::protobuf::ServiceDescriptor const** file_level_service_descriptors_google_2fprotobuf_2fempty_2eproto = nullptr;
 
 const ::google::protobuf::uint32 TableStruct_google_2fprotobuf_2fempty_2eproto::offsets[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = {
   ~0u,  // no _has_bits_
@@ -61,7 +61,7 @@ static ::google::protobuf::Message const * const file_default_instances[] = {
   reinterpret_cast<const ::google::protobuf::Message*>(&::google::protobuf::_Empty_default_instance_),
 };
 
-::google::protobuf::internal::AssignDescriptorsTable assign_descriptors_table_google_2fprotobuf_2fempty_2eproto = {
+static ::google::protobuf::internal::AssignDescriptorsTable assign_descriptors_table_google_2fprotobuf_2fempty_2eproto = {
   {}, AddDescriptors_google_2fprotobuf_2fempty_2eproto, "google/protobuf/empty.proto", schemas,
   file_default_instances, TableStruct_google_2fprotobuf_2fempty_2eproto::offsets,
   file_level_metadata_google_2fprotobuf_2fempty_2eproto, 1, file_level_enum_descriptors_google_2fprotobuf_2fempty_2eproto, file_level_service_descriptors_google_2fprotobuf_2fempty_2eproto,
@@ -74,7 +74,7 @@ const char descriptor_table_protodef_google_2fprotobuf_2fempty_2eproto[] =
   "/ptypes/empty\370\001\001\242\002\003GPB\252\002\036Google.Protobuf"
   ".WellKnownTypesb\006proto3"
   ;
-::google::protobuf::internal::DescriptorTable descriptor_table_google_2fprotobuf_2fempty_2eproto = {
+static ::google::protobuf::internal::DescriptorTable descriptor_table_google_2fprotobuf_2fempty_2eproto = {
   false, InitDefaults_google_2fprotobuf_2fempty_2eproto, 
   descriptor_table_protodef_google_2fprotobuf_2fempty_2eproto,
   "google/protobuf/empty.proto", &assign_descriptors_table_google_2fprotobuf_2fempty_2eproto, 183,
@@ -159,28 +159,21 @@ void Empty::Clear() {
 }
 
 #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
-const char* Empty::_InternalParse(const char* begin, const char* end, void* object,
-                  ::google::protobuf::internal::ParseContext* ctx) {
-  auto msg = static_cast<Empty*>(object);
-  ::google::protobuf::int32 size; (void)size;
-  int depth; (void)depth;
-  ::google::protobuf::uint32 tag;
-  ::google::protobuf::internal::ParseFunc parser_till_end; (void)parser_till_end;
-  auto ptr = begin;
-  while (ptr < end) {
-    ptr = ::google::protobuf::io::Parse32(ptr, &tag);
+const char* Empty::_InternalParse(const char* ptr, ::google::protobuf::internal::ParseContext* ctx) {
+  while (!ctx->Done(&ptr)) {
+    ::google::protobuf::uint32 tag;
+    ptr = ::google::protobuf::internal::ReadTag(ptr, &tag);
     GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
     switch (tag >> 3) {
       default: {
         if ((tag & 7) == 4 || tag == 0) {
-          ctx->EndGroup(tag);
+          ctx->SetLastTag(tag);
           return ptr;
         }
-        auto res = UnknownFieldParse(tag, {_InternalParse, msg},
-          ptr, end, msg->_internal_metadata_.mutable_unknown_fields(), ctx);
-        ptr = res.first;
+        ptr = UnknownFieldParse(tag,
+          _internal_metadata_.mutable_unknown_fields(), ptr, ctx);
         GOOGLE_PROTOBUF_PARSER_ASSERT(ptr != nullptr);
-        if (res.second) return ptr;
+        break;
       }
     }  // switch
   }  // while

src/google/protobuf/empty.pb.h

@@ -31,6 +31,13 @@
 #include <google/protobuf/repeated_field.h>  // IWYU pragma: export
 #include <google/protobuf/extension_set.h>  // IWYU pragma: export
 #include <google/protobuf/unknown_field_set.h>
+namespace google {
+namespace protobuf {
+namespace internal {
+class AnyMetadata;
+}  // namespace internal
+}  // namespace protobuf
+}  // namespace google
 // @@protoc_insertion_point(includes)
 #include <google/protobuf/port_def.inc>
 #define PROTOBUF_INTERNAL_EXPORT_google_2fprotobuf_2fempty_2eproto PROTOBUF_EXPORT
@@ -131,8 +138,7 @@ class PROTOBUF_EXPORT Empty final :
 
   size_t ByteSizeLong() const final;
   #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
-  static const char* _InternalParse(const char* begin, const char* end, void* object, ::google::protobuf::internal::ParseContext* ctx);
-  ::google::protobuf::internal::ParseFunc _ParseFunc() const final { return _InternalParse; }
+  const char* _InternalParse(const char* ptr, ::google::protobuf::internal::ParseContext* ctx) final;
   #else
   bool MergePartialFromCodedStream(
       ::google::protobuf::io::CodedInputStream* input) final;
@@ -144,10 +150,14 @@ class PROTOBUF_EXPORT Empty final :
   int GetCachedSize() const final { return _cached_size_.Get(); }
 
   private:
-  void SharedCtor();
-  void SharedDtor();
+  inline void SharedCtor();
+  inline void SharedDtor();
   void SetCachedSize(int size) const final;
   void InternalSwap(Empty* other);
+  friend class ::google::protobuf::internal::AnyMetadata;
+  static ::google::protobuf::StringPiece FullMessageName() {
+    return "google.protobuf.Empty";
+  }
   protected:
   explicit Empty(::google::protobuf::Arena* arena);
   private:

src/google/protobuf/extension_set.cc

@@ -177,11 +177,7 @@ void ExtensionSet::RegisterMessageExtension(const MessageLite* containing_type,
   GOOGLE_CHECK(type == WireFormatLite::TYPE_MESSAGE ||
         type == WireFormatLite::TYPE_GROUP);
   ExtensionInfo info(type, is_repeated, is_packed);
-#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
-  info.message_info = {prototype, prototype->_ParseFunc()};
-#else
   info.message_info = {prototype};
-#endif
   Register(containing_type, number, info);
 }
 
@@ -1204,9 +1200,8 @@ bool ExtensionSet::ParseField(uint32 tag, io::CodedInputStream* input,
 }
 
 #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
-std::pair<const char*, bool> ExtensionSet::ParseField(
-    uint64 tag, ParseClosure parent, const char* begin, const char* end,
-    const MessageLite* containing_type,
+const char* ExtensionSet::ParseField(
+    uint64 tag, const char* ptr, const MessageLite* containing_type,
     internal::InternalMetadataWithArenaLite* metadata,
     internal::ParseContext* ctx) {
   GeneratedExtensionFinder finder(containing_type);
@@ -1215,12 +1210,19 @@ std::pair<const char*, bool> ExtensionSet::ParseField(
   ExtensionInfo extension;
   if (!FindExtensionInfoFromFieldNumber(tag & 7, number, &finder, &extension,
                                         &was_packed_on_wire)) {
-    return UnknownFieldParse(tag, parent, begin, end,
-                             metadata->mutable_unknown_fields(), ctx);
+    return UnknownFieldParse(tag, metadata->mutable_unknown_fields(), ptr, ctx);
   }
   return ParseFieldWithExtensionInfo(number, was_packed_on_wire, extension,
-                                     metadata, parent, begin, end, ctx);
+                                     metadata, ptr, ctx);
 }
+
+const char* ExtensionSet::ParseMessageSetItem(
+    const char* ptr, const MessageLite* containing_type,
+    internal::InternalMetadataWithArenaLite* metadata,
+    internal::ParseContext* ctx) {
+  return ParseMessageSetItemTmpl(ptr, containing_type, metadata, ctx);
+}
+
 #endif
 
 bool ExtensionSet::ParseFieldWithExtensionInfo(

src/google/protobuf/extension_set.h

@@ -121,9 +121,6 @@ struct ExtensionInfo {
 
   struct MessageInfo {
     const MessageLite* prototype;
-#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
-    ParseFunc parse_func;
-#endif
   };
 
   union {
@@ -401,33 +398,45 @@ class PROTOBUF_EXPORT ExtensionSet {
                   io::CodedOutputStream* unknown_fields);
 
 #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
-  template <typename T>
-  std::pair<const char*, bool> ParseFieldWithExtensionInfo(
-      int number, bool was_packed_on_wire, const ExtensionInfo& info,
-      T* metadata, ParseClosure parent, const char* begin, const char* end,
-      internal::ParseContext* ctx);
   // Lite parser
-  std::pair<const char*, bool> ParseField(
-      uint64 tag, ParseClosure parent, const char* begin, const char* end,
-      const MessageLite* containing_type,
-      internal::InternalMetadataWithArenaLite* metadata,
-      internal::ParseContext* ctx);
+  const char* ParseField(uint64 tag, const char* ptr,
+                         const MessageLite* containing_type,
+                         internal::InternalMetadataWithArenaLite* metadata,
+                         internal::ParseContext* ctx);
   // Full parser
-  std::pair<const char*, bool> ParseField(
-      uint64 tag, ParseClosure parent, const char* begin, const char* end,
-      const Message* containing_type,
-      internal::InternalMetadataWithArena* metadata,
-      internal::ParseContext* ctx);
-  std::pair<const char*, bool> ParseFieldMaybeLazily(
-      uint64 tag, ParseClosure parent, const char* begin, const char* end,
-      const Message* containing_type,
-      internal::InternalMetadataWithArena* metadata,
-      internal::ParseContext* ctx);
-  const char* ParseMessageSetItem(ParseClosure parent, const char* begin,
-                                  const char* end,
-                                  const Message* containing_type,
-                                  internal::InternalMetadataWithArena* metadata,
-                                  internal::ParseContext* ctx);
+  const char* ParseField(uint64 tag, const char* ptr,
+                         const Message* containing_type,
+                         internal::InternalMetadataWithArena* metadata,
+                         internal::ParseContext* ctx);
+  template <typename Msg, typename Metadata>
+  const char* ParseMessageSet(const char* ptr, const Msg* containing_type,
+                              Metadata* metadata, internal::ParseContext* ctx) {
+    struct MessageSetItem {
+      const char* _InternalParse(const char* ptr, ParseContext* ctx) {
+        return me->ParseMessageSetItem(ptr, containing_type, metadata, ctx);
+      }
+      ExtensionSet* me;
+      const Msg* containing_type;
+      Metadata* metadata;
+    } item{this, containing_type, metadata};
+    while (!ctx->Done(&ptr)) {
+      uint32 tag;
+      ptr = ReadTag(ptr, &tag);
+      GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
+      if (tag == WireFormatLite::kMessageSetItemStartTag) {
+        ptr = ctx->ParseGroup(&item, ptr, tag);
+        GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
+      } else {
+        if (tag == 0 || (tag & 7) == 4) {
+          ctx->SetLastTag(tag);
+          return ptr;
+        }
+        ptr = ParseField(tag, ptr, containing_type, metadata, ctx);
+        GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
+      }
+    }
+    return ptr;
+  }
 #endif
 
   // Parse an entire message in MessageSet format.  Such messages have no
@@ -527,6 +536,9 @@ class PROTOBUF_EXPORT ExtensionSet {
 
     virtual bool ReadMessage(const MessageLite& prototype,
                              io::CodedInputStream* input) = 0;
+#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
+    virtual const char* _InternalParse(const char* ptr, ParseContext* ctx) = 0;
+#endif
     virtual void WriteMessage(int number,
                               io::CodedOutputStream* output) const = 0;
     virtual uint8* WriteMessageToArray(int number, uint8* target) const = 0;
@@ -697,13 +709,6 @@ class PROTOBUF_EXPORT ExtensionSet {
   // Merges existing Extension from other_extension
   void InternalExtensionMergeFrom(int number, const Extension& other_extension);
 
-#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
-  bool FindExtension(int wire_type, uint32 field,
-                     const Message* containing_type,
-                     const internal::ParseContext* ctx,
-                     ExtensionInfo* extension, bool* was_packed_on_wire);
-#endif
-
   // Returns true and fills field_number and extension if extension is found.
   // Note to support packed repeated field compatibility, it also fills whether
   // the tag on wire is packed, which can be different from
@@ -757,6 +762,53 @@ class PROTOBUF_EXPORT ExtensionSet {
                            ExtensionFinder* extension_finder,
                            MessageSetFieldSkipper* field_skipper);
 
+#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
+  bool FindExtension(int wire_type, uint32 field,
+                     const MessageLite* containing_type,
+                     const internal::ParseContext* ctx,
+                     ExtensionInfo* extension, bool* was_packed_on_wire) {
+    GeneratedExtensionFinder finder(containing_type);
+    return FindExtensionInfoFromFieldNumber(wire_type, field, &finder,
+                                            extension, was_packed_on_wire);
+  }
+  inline bool FindExtension(int wire_type, uint32 field,
+                            const Message* containing_type,
+                            const internal::ParseContext* ctx,
+                            ExtensionInfo* extension, bool* was_packed_on_wire);
+  // Used for MessageSet only
+  const char* ParseFieldMaybeLazily(
+      uint64 tag, const char* ptr, const MessageLite* containing_type,
+      internal::InternalMetadataWithArenaLite* metadata,
+      internal::ParseContext* ctx) {
+    // Lite MessageSet doesn't implement lazy.
+    return ParseField(tag, ptr, containing_type, metadata, ctx);
+  }
+  const char* ParseFieldMaybeLazily(
+      uint64 tag, const char* ptr, const Message* containing_type,
+      internal::InternalMetadataWithArena* metadata,
+      internal::ParseContext* ctx);
+  const char* ParseMessageSetItem(
+      const char* ptr, const MessageLite* containing_type,
+      internal::InternalMetadataWithArenaLite* metadata,
+      internal::ParseContext* ctx);
+  const char* ParseMessageSetItem(const char* ptr,
+                                  const Message* containing_type,
+                                  internal::InternalMetadataWithArena* metadata,
+                                  internal::ParseContext* ctx);
+
+  // Implemented in extension_set_inl.h to keep code out of the header file.
+  template <typename T>
+  const char* ParseFieldWithExtensionInfo(int number, bool was_packed_on_wire,
+                                          const ExtensionInfo& info,
+                                          T* metadata, const char* ptr,
+                                          internal::ParseContext* ctx);
+  template <typename Msg, typename Metadata>
+  const char* ParseMessageSetItemTmpl(const char* ptr,
+                                      const Msg* containing_type,
+                                      Metadata* metadata,
+                                      internal::ParseContext* ctx);
+#endif  // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
+
   // Hack:  RepeatedPtrFieldBase declares ExtensionSet as a friend.  This
   //   friendship should automatically extend to ExtensionSet::Extension, but
   //   unfortunately some older compilers (e.g. GCC 3.4.4) do not implement this
@@ -804,44 +856,6 @@ class PROTOBUF_EXPORT ExtensionSet {
   GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ExtensionSet);
 };
 
-#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
-
-template <typename Msg, typename Metadata>
-const char* ParseMessageSet(const char* begin, const char* end, Msg* msg,
-                            ExtensionSet* ext, Metadata* metadata,
-                            internal::ParseContext* ctx) {
-  auto ptr = begin;
-  int depth = 0;
-  while (ptr < end) {
-    uint32 tag;
-    ptr = io::Parse32(ptr, &tag);
-    GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
-    if (tag == WireFormatLite::kMessageSetItemStartTag) {
-      ctx->extra_parse_data().payload.clear();
-      auto res = ctx->ParseGroup(tag, {Msg::InternalParseMessageSetItem, msg},
-                                 ptr, end, &depth);
-      ptr = res.first;
-      GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
-      if (res.second) {
-        GOOGLE_PROTOBUF_PARSER_ASSERT(ctx->StoreGroup(
-            {Msg::_InternalParse, msg}, {Msg::InternalParseMessageSetItem, msg},
-            depth, tag));
-        return ptr;
-      }
-    } else {
-      auto res =
-          ext->ParseField(tag, {Msg::_InternalParse, msg}, ptr, end,
-                          Msg::internal_default_instance(), metadata, ctx);
-      ptr = res.first;
-      GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
-      if (res.second) break;
-    }
-  }
-  return ptr;
-}
-
-#endif
-
 // These are just for convenience...
 inline void ExtensionSet::SetString(int number, FieldType type,
                                     const std::string& value,

src/google/protobuf/extension_set_heavy.cc

@@ -309,10 +309,6 @@ bool DescriptorPoolExtensionFinder::Find(int number, ExtensionInfo* output) {
       GOOGLE_CHECK(output->message_info.prototype != nullptr)
           << "Extension factory's GetPrototype() returned NULL for extension: "
           << extension->full_name();
-#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
-      output->message_info.parse_func =
-          output->message_info.prototype->_ParseFunc();
-#endif
     } else if (extension->cpp_type() == FieldDescriptor::CPPTYPE_ENUM) {
       output->enum_validity_check.func = ValidateEnumUsingDescriptor;
       output->enum_validity_check.arg = extension->enum_type();
@@ -329,15 +325,14 @@ bool ExtensionSet::FindExtension(int wire_type, uint32 field,
                                  const internal::ParseContext* ctx,
                                  ExtensionInfo* extension,
                                  bool* was_packed_on_wire) {
-  if (ctx->extra_parse_data().pool == nullptr) {
+  if (ctx->data().pool == nullptr) {
     GeneratedExtensionFinder finder(containing_type);
     if (!FindExtensionInfoFromFieldNumber(wire_type, field, &finder, extension,
                                           was_packed_on_wire)) {
       return false;
     }
   } else {
-    DescriptorPoolExtensionFinder finder(ctx->extra_parse_data().pool,
-                                         ctx->extra_parse_data().factory,
+    DescriptorPoolExtensionFinder finder(ctx->data().pool, ctx->data().factory,
                                          containing_type->GetDescriptor());
     if (!FindExtensionInfoFromFieldNumber(wire_type, field, &finder, extension,
                                           was_packed_on_wire)) {
@@ -347,9 +342,8 @@ bool ExtensionSet::FindExtension(int wire_type, uint32 field,
   return true;
 }
 
-std::pair<const char*, bool> ExtensionSet::ParseField(
-    uint64 tag, ParseClosure parent, const char* begin, const char* end,
-    const Message* containing_type,
+const char* ExtensionSet::ParseField(
+    uint64 tag, const char* ptr, const Message* containing_type,
     internal::InternalMetadataWithArena* metadata,
     internal::ParseContext* ctx) {
   int number = tag >> 3;
@@ -357,20 +351,26 @@ std::pair<const char*, bool> ExtensionSet::ParseField(
   ExtensionInfo extension;
   if (!FindExtension(tag & 7, number, containing_type, ctx, &extension,
                      &was_packed_on_wire)) {
-    return UnknownFieldParse(tag, parent, begin, end,
-                             metadata->mutable_unknown_fields(), ctx);
+    return UnknownFieldParse(tag, metadata->mutable_unknown_fields(), ptr, ctx);
   }
   return ParseFieldWithExtensionInfo(number, was_packed_on_wire, extension,
-                                     metadata, parent, begin, end, ctx);
+                                     metadata, ptr, ctx);
+}
+
+const char* ExtensionSet::ParseFieldMaybeLazily(
+    uint64 tag, const char* ptr, const Message* containing_type,
+    internal::InternalMetadataWithArena* metadata,
+    internal::ParseContext* ctx) {
+  return ParseField(tag, ptr, containing_type, metadata, ctx);
 }
 
-std::pair<const char*, bool> ExtensionSet::ParseFieldMaybeLazily(
-    uint64 tag, ParseClosure parent, const char* begin, const char* end,
-    const Message* containing_type,
+const char* ExtensionSet::ParseMessageSetItem(
+    const char* ptr, const Message* containing_type,
     internal::InternalMetadataWithArena* metadata,
     internal::ParseContext* ctx) {
-  return ParseField(tag, parent, begin, end, containing_type, metadata, ctx);
+  return ParseMessageSetItemTmpl(ptr, containing_type, metadata, ctx);
 }
+
 #endif  // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
 
 bool ExtensionSet::ParseField(uint32 tag, io::CodedInputStream* input,
@@ -388,83 +388,6 @@ bool ExtensionSet::ParseField(uint32 tag, io::CodedInputStream* input,
   }
 }
 
-#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
-const char* ExtensionSet::ParseMessageSetItem(
-    ParseClosure parent, const char* begin, const char* end,
-    const Message* containing_type,
-    internal::InternalMetadataWithArena* metadata,
-    internal::ParseContext* ctx) {
-  auto ptr = begin;
-  while (ptr < end) {
-    uint32 tag = *ptr++;
-    if (tag == WireFormatLite::kMessageSetTypeIdTag) {
-      uint32 type_id;
-      ptr = io::Parse32(ptr, &type_id);
-      GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
-
-      if (ctx->extra_parse_data().payload.empty()) {
-        tag = *ptr++;
-        GOOGLE_PROTOBUF_PARSER_ASSERT(tag ==
-                                       WireFormatLite::kMessageSetMessageTag);
-        auto res =
-            ParseFieldMaybeLazily(static_cast<uint64>(type_id) * 8 + 2, parent,
-                                  ptr, end, containing_type, metadata, ctx);
-        ptr = res.first;
-        GOOGLE_PROTOBUF_PARSER_ASSERT(ptr != nullptr);
-        if (res.second) break;
-      } else {
-        ExtensionInfo extension;
-        bool was_packed_on_wire;
-        if (!FindExtension(2, type_id, containing_type, ctx, &extension,
-                           &was_packed_on_wire)) {
-          metadata->mutable_unknown_fields()->AddLengthDelimited(
-              type_id, ctx->extra_parse_data().payload);
-          continue;
-        }
-        MessageLite* value =
-            extension.is_repeated
-                ? AddMessage(type_id, WireFormatLite::TYPE_MESSAGE,
-                             *extension.message_info.prototype,
-                             extension.descriptor)
-                : MutableMessage(type_id, WireFormatLite::TYPE_MESSAGE,
-                                 *extension.message_info.prototype,
-                                 extension.descriptor);
-        ParseClosure parser = {extension.message_info.parse_func, value};
-        StringPiece chunk(ctx->extra_parse_data().payload);
-        bool ok = ctx->ParseExactRange(parser, chunk.begin(), chunk.end());
-        GOOGLE_PROTOBUF_PARSER_ASSERT(ok);
-      }
-    } else if (tag == WireFormatLite::kMessageSetItemEndTag) {
-      ctx->EndGroup(tag);
-      break;
-    } else if (tag == WireFormatLite::kMessageSetMessageTag) {
-      uint32 size;
-      ptr = io::Parse32(ptr, &size);
-      GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
-      ParseClosure child = {internal::StringParser,
-                            &ctx->extra_parse_data().payload};
-      if (size > end - ptr + internal::ParseContext::kSlopBytes) {
-        ctx->extra_parse_data().payload.clear();
-        return ctx->StoreAndTailCall(ptr, end, parent, child, size);
-      } else {
-        ctx->extra_parse_data().payload.assign(ptr, size);
-        ptr += size;
-      }
-    } else {
-      ptr--;
-      ptr = io::Parse32(ptr, &tag);
-      GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
-      auto res =
-          ParseField(tag, parent, ptr, end, containing_type, metadata, ctx);
-      ptr = res.first;
-      GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
-      if (res.second) break;
-    }
-  }
-  return ptr;
-}
-#endif  // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
-
 bool ExtensionSet::ParseMessageSet(io::CodedInputStream* input,
                                    const Message* containing_type,
                                    UnknownFieldSet* unknown_fields) {

src/google/protobuf/extension_set_inl.h

@@ -31,6 +31,7 @@
 #ifndef GOOGLE_PROTOBUF_EXTENSION_SET_INL_H__
 #define GOOGLE_PROTOBUF_EXTENSION_SET_INL_H__
 
+#include <google/protobuf/parse_context.h>
 #include <google/protobuf/extension_set.h>
 
 namespace google {
@@ -39,22 +40,17 @@ namespace internal {
 
 #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
 template <typename T>
-std::pair<const char*, bool> ExtensionSet::ParseFieldWithExtensionInfo(
+const char* ExtensionSet::ParseFieldWithExtensionInfo(
     int number, bool was_packed_on_wire, const ExtensionInfo& extension,
-    T* metadata, ParseClosure parent, const char* begin, const char* end,
-    internal::ParseContext* ctx) {
-  auto ptr = begin;
-  ParseClosure child;
-  int depth;
+    T* metadata, const char* ptr, internal::ParseContext* ctx) {
   if (was_packed_on_wire) {
     switch (extension.type) {
 #define HANDLE_TYPE(UPPERCASE, CPP_CAMELCASE)                                \
   case WireFormatLite::TYPE_##UPPERCASE:                                     \
-    child = {                                                                \
-        internal::Packed##CPP_CAMELCASE##Parser,                             \
+    return internal::Packed##CPP_CAMELCASE##Parser(                          \
         MutableRawRepeatedField(number, extension.type, extension.is_packed, \
-                                extension.descriptor)};                      \
-    goto length_delim
+                                extension.descriptor),                       \
+        ptr, ctx);
       HANDLE_TYPE(INT32, Int32);
       HANDLE_TYPE(INT64, Int64);
       HANDLE_TYPE(UINT32, UInt32);
@@ -71,15 +67,12 @@ std::pair<const char*, bool> ExtensionSet::ParseFieldWithExtensionInfo(
 #undef HANDLE_TYPE
 
       case WireFormatLite::TYPE_ENUM:
-        ctx->extra_parse_data().SetEnumValidatorArg(
-            extension.enum_validity_check.func,
+        return internal::PackedEnumParserArg(
+            MutableRawRepeatedField(number, extension.type, extension.is_packed,
+                                    extension.descriptor),
+            ptr, ctx, extension.enum_validity_check.func,
             extension.enum_validity_check.arg,
             metadata->mutable_unknown_fields(), number);
-        child = {
-            internal::PackedValidEnumParserLiteArg,
-            MutableRawRepeatedField(number, extension.type, extension.is_packed,
-                                    extension.descriptor)};
-        goto length_delim;
       case WireFormatLite::TYPE_STRING:
       case WireFormatLite::TYPE_BYTES:
       case WireFormatLite::TYPE_GROUP:
@@ -92,8 +85,8 @@ std::pair<const char*, bool> ExtensionSet::ParseFieldWithExtensionInfo(
 #define HANDLE_VARINT_TYPE(UPPERCASE, CPP_CAMELCASE)                        \
   case WireFormatLite::TYPE_##UPPERCASE: {                                  \
     uint64 value;                                                           \
-    ptr = io::Parse64(ptr, &value);                                     \
-    GOOGLE_PROTOBUF_ASSERT_RETURN(ptr, std::make_pair(nullptr, true));     \
+    ptr = ParseVarint64(ptr, &value);                                       \
+    GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);                                    \
     if (extension.is_repeated) {                                            \
       Add##CPP_CAMELCASE(number, WireFormatLite::TYPE_##UPPERCASE,          \
                          extension.is_packed, value, extension.descriptor); \
@@ -111,8 +104,8 @@ std::pair<const char*, bool> ExtensionSet::ParseFieldWithExtensionInfo(
 #define HANDLE_SVARINT_TYPE(UPPERCASE, CPP_CAMELCASE, SIZE)                 \
   case WireFormatLite::TYPE_##UPPERCASE: {                                  \
     uint64 val;                                                             \
-    ptr = io::Parse64(ptr, &val);                                       \
-    GOOGLE_PROTOBUF_ASSERT_RETURN(ptr, std::make_pair(nullptr, true));     \
+    ptr = ParseVarint64(ptr, &val);                                         \
+    GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);                                    \
     auto value = WireFormatLite::ZigZagDecode##SIZE(val);                   \
     if (extension.is_repeated) {                                            \
       Add##CPP_CAMELCASE(number, WireFormatLite::TYPE_##UPPERCASE,          \
@@ -151,8 +144,8 @@ std::pair<const char*, bool> ExtensionSet::ParseFieldWithExtensionInfo(
 
       case WireFormatLite::TYPE_ENUM: {
         uint64 val;
-        ptr = io::Parse64(ptr, &val);
-        GOOGLE_PROTOBUF_ASSERT_RETURN(ptr, std::make_pair(nullptr, true));
+        ptr = ParseVarint64(ptr, &val);
+        GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
         int value = val;
 
         if (!extension.enum_validity_check.func(
@@ -175,8 +168,9 @@ std::pair<const char*, bool> ExtensionSet::ParseFieldWithExtensionInfo(
                                         extension.descriptor)
                             : MutableString(number, WireFormatLite::TYPE_STRING,
                                             extension.descriptor);
-        child = {StringParser, value};
-        goto length_delim;
+        int size = ReadSize(&ptr);
+        GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
+        return ctx->ReadString(ptr, size, value);
       }
 
       case WireFormatLite::TYPE_GROUP: {
@@ -188,18 +182,8 @@ std::pair<const char*, bool> ExtensionSet::ParseFieldWithExtensionInfo(
                 : MutableMessage(number, WireFormatLite::TYPE_GROUP,
                                  *extension.message_info.prototype,
                                  extension.descriptor);
-        child = {extension.message_info.parse_func, value};
         uint32 tag = (number << 3) + WireFormatLite::WIRETYPE_START_GROUP;
-        auto res = ctx->ParseGroup(tag, child, ptr, end, &depth);
-        ptr = res.first;
-        GOOGLE_PROTOBUF_ASSERT_RETURN(ptr, std::make_pair(nullptr, true));
-        if (res.second) {
-          GOOGLE_PROTOBUF_ASSERT_RETURN(
-              ctx->StoreGroup(parent, child, depth, tag),
-              std::make_pair(nullptr, true));
-          return std::make_pair(ptr, true);
-        }
-        break;
+        return ctx->ParseGroup(value, ptr, tag);
       }
 
       case WireFormatLite::TYPE_MESSAGE: {
@@ -211,31 +195,77 @@ std::pair<const char*, bool> ExtensionSet::ParseFieldWithExtensionInfo(
                 : MutableMessage(number, WireFormatLite::TYPE_MESSAGE,
                                  *extension.message_info.prototype,
                                  extension.descriptor);
-        child = {extension.message_info.parse_func, value};
-        goto length_delim;
+        return ctx->ParseMessage(value, ptr);
       }
     }
   }
+  return ptr;
+}
+
+template <typename Msg, typename Metadata>
+const char* ExtensionSet::ParseMessageSetItemTmpl(const char* ptr,
+                                                  const Msg* containing_type,
+                                                  Metadata* metadata,
+                                                  internal::ParseContext* ctx) {
+  std::string payload;
+  uint32 type_id = 0;
+  while (!ctx->Done(&ptr)) {
+    uint32 tag;
+    ptr = ReadTag(ptr, &tag);
+    GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
+    if (tag == WireFormatLite::kMessageSetTypeIdTag) {
+      ptr = _Parse32(ptr, &type_id);
+      GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
+      if (!payload.empty()) {
+        ExtensionInfo extension;
+        bool was_packed_on_wire;
+        if (!FindExtension(2, type_id, containing_type, ctx, &extension,
+                           &was_packed_on_wire)) {
+          WriteLengthDelimited(type_id, payload,
+                               metadata->mutable_unknown_fields());
+        } else {
+          MessageLite* value =
+              extension.is_repeated
+                  ? AddMessage(type_id, WireFormatLite::TYPE_MESSAGE,
+                               *extension.message_info.prototype,
+                               extension.descriptor)
+                  : MutableMessage(type_id, WireFormatLite::TYPE_MESSAGE,
+                                   *extension.message_info.prototype,
+                                   extension.descriptor);
 
-  return std::make_pair(ptr, false);
-
-length_delim:
-  uint32 size;
-  ptr = io::Parse32(ptr, &size);
-  GOOGLE_PROTOBUF_ASSERT_RETURN(ptr, std::make_pair(nullptr, true));
-  if (size > end - ptr) goto len_delim_till_end;
-  {
-    auto newend = ptr + size;
-    bool ok = ctx->ParseExactRange(child, ptr, newend);
-    GOOGLE_PROTOBUF_ASSERT_RETURN(ok, std::make_pair(nullptr, true));
-    ptr = newend;
+          const char* p;
+          ParseContext tmp_ctx(ctx->depth(), false, &p, payload);
+          tmp_ctx.data().pool = ctx->data().pool;
+          tmp_ctx.data().factory = ctx->data().factory;
+          GOOGLE_PROTOBUF_PARSER_ASSERT(
+              tmp_ctx.AtLegitimateEnd(value->_InternalParse(p, &tmp_ctx)));
+        }
+        type_id = 0;
+      }
+    } else if (tag == WireFormatLite::kMessageSetMessageTag) {
+      if (type_id != 0) {
+        ptr = ParseFieldMaybeLazily(static_cast<uint64>(type_id) * 8 + 2, ptr,
+                                    containing_type, metadata, ctx);
+        GOOGLE_PROTOBUF_PARSER_ASSERT(ptr != nullptr);
+        type_id = 0;
+      } else {
+        int32 size = ReadSize(&ptr);
+        GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
+        ptr = ctx->ReadString(ptr, size, &payload);
+        GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
+      }
+    } else {
+      if (tag == 0 || (tag & 7) == 4) {
+        ctx->SetLastTag(tag);
+        return ptr;
+      }
+      ptr = ParseField(tag, ptr, containing_type, metadata, ctx);
+      GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
+    }
   }
-  return std::make_pair(ptr, false);
-len_delim_till_end:
-  return std::make_pair(ctx->StoreAndTailCall(ptr, end, parent, child, size),
-                        true);
+  return ptr;
 }
-#endif
+#endif  // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
 
 }  // namespace internal
 }  // namespace protobuf

src/google/protobuf/field_mask.pb.cc

@@ -42,9 +42,9 @@ void InitDefaults_google_2fprotobuf_2ffield_5fmask_2eproto() {
   ::google::protobuf::internal::InitSCC(&scc_info_FieldMask_google_2fprotobuf_2ffield_5fmask_2eproto.base);
 }
 
-::google::protobuf::Metadata file_level_metadata_google_2fprotobuf_2ffield_5fmask_2eproto[1];
-constexpr ::google::protobuf::EnumDescriptor const** file_level_enum_descriptors_google_2fprotobuf_2ffield_5fmask_2eproto = nullptr;
-constexpr ::google::protobuf::ServiceDescriptor const** file_level_service_descriptors_google_2fprotobuf_2ffield_5fmask_2eproto = nullptr;
+static ::google::protobuf::Metadata file_level_metadata_google_2fprotobuf_2ffield_5fmask_2eproto[1];
+static constexpr ::google::protobuf::EnumDescriptor const** file_level_enum_descriptors_google_2fprotobuf_2ffield_5fmask_2eproto = nullptr;
+static constexpr ::google::protobuf::ServiceDescriptor const** file_level_service_descriptors_google_2fprotobuf_2ffield_5fmask_2eproto = nullptr;
 
 const ::google::protobuf::uint32 TableStruct_google_2fprotobuf_2ffield_5fmask_2eproto::offsets[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = {
   ~0u,  // no _has_bits_
@@ -62,7 +62,7 @@ static ::google::protobuf::Message const * const file_default_instances[] = {
   reinterpret_cast<const ::google::protobuf::Message*>(&::google::protobuf::_FieldMask_default_instance_),
 };
 
-::google::protobuf::internal::AssignDescriptorsTable assign_descriptors_table_google_2fprotobuf_2ffield_5fmask_2eproto = {
+static ::google::protobuf::internal::AssignDescriptorsTable assign_descriptors_table_google_2fprotobuf_2ffield_5fmask_2eproto = {
   {}, AddDescriptors_google_2fprotobuf_2ffield_5fmask_2eproto, "google/protobuf/field_mask.proto", schemas,
   file_default_instances, TableStruct_google_2fprotobuf_2ffield_5fmask_2eproto::offsets,
   file_level_metadata_google_2fprotobuf_2ffield_5fmask_2eproto, 1, file_level_enum_descriptors_google_2fprotobuf_2ffield_5fmask_2eproto, file_level_service_descriptors_google_2fprotobuf_2ffield_5fmask_2eproto,
@@ -76,7 +76,7 @@ const char descriptor_table_protodef_google_2fprotobuf_2ffield_5fmask_2eproto[]
   "ield_mask;field_mask\370\001\001\242\002\003GPB\252\002\036Google.P"
   "rotobuf.WellKnownTypesb\006proto3"
   ;
-::google::protobuf::internal::DescriptorTable descriptor_table_google_2fprotobuf_2ffield_5fmask_2eproto = {
+static ::google::protobuf::internal::DescriptorTable descriptor_table_google_2fprotobuf_2ffield_5fmask_2eproto = {
   false, InitDefaults_google_2fprotobuf_2ffield_5fmask_2eproto, 
   descriptor_table_protodef_google_2fprotobuf_2ffield_5fmask_2eproto,
   "google/protobuf/field_mask.proto", &assign_descriptors_table_google_2fprotobuf_2ffield_5fmask_2eproto, 230,
@@ -167,59 +167,36 @@ void FieldMask::Clear() {
 }
 
 #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
-const char* FieldMask::_InternalParse(const char* begin, const char* end, void* object,
-                  ::google::protobuf::internal::ParseContext* ctx) {
-  auto msg = static_cast<FieldMask*>(object);
-  ::google::protobuf::int32 size; (void)size;
-  int depth; (void)depth;
-  ::google::protobuf::uint32 tag;
-  ::google::protobuf::internal::ParseFunc parser_till_end; (void)parser_till_end;
-  auto ptr = begin;
-  while (ptr < end) {
-    ptr = ::google::protobuf::io::Parse32(ptr, &tag);
+const char* FieldMask::_InternalParse(const char* ptr, ::google::protobuf::internal::ParseContext* ctx) {
+  while (!ctx->Done(&ptr)) {
+    ::google::protobuf::uint32 tag;
+    ptr = ::google::protobuf::internal::ReadTag(ptr, &tag);
     GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
     switch (tag >> 3) {
       // repeated string paths = 1;
       case 1: {
         if (static_cast<::google::protobuf::uint8>(tag) != 10) goto handle_unusual;
         do {
-          ptr = ::google::protobuf::io::ReadSize(ptr, &size);
+          ptr = ::google::protobuf::internal::InlineGreedyStringParserUTF8(add_paths(), ptr, ctx, "google.protobuf.FieldMask.paths");
           GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
-          ctx->extra_parse_data().SetFieldName("google.protobuf.FieldMask.paths");
-          object = msg->add_paths();
-          if (size > end - ptr + ::google::protobuf::internal::ParseContext::kSlopBytes) {
-            parser_till_end = ::google::protobuf::internal::GreedyStringParserUTF8;
-            goto string_till_end;
-          }
-          GOOGLE_PROTOBUF_PARSER_ASSERT(::google::protobuf::internal::StringCheckUTF8(ptr, size, ctx));
-          ::google::protobuf::internal::InlineGreedyStringParser(object, ptr, size, ctx);
-          ptr += size;
-          if (ptr >= end) break;
-        } while ((::google::protobuf::io::UnalignedLoad<::google::protobuf::uint64>(ptr) & 255) == 10 && (ptr += 1));
+          if (ctx->Done(&ptr)) return ptr;
+        } while ((::google::protobuf::internal::UnalignedLoad<::google::protobuf::uint64>(ptr) & 255) == 10 && (ptr += 1));
         break;
       }
       default: {
       handle_unusual:
         if ((tag & 7) == 4 || tag == 0) {
-          ctx->EndGroup(tag);
+          ctx->SetLastTag(tag);
           return ptr;
         }
-        auto res = UnknownFieldParse(tag, {_InternalParse, msg},
-          ptr, end, msg->_internal_metadata_.mutable_unknown_fields(), ctx);
-        ptr = res.first;
+        ptr = UnknownFieldParse(tag,
+          _internal_metadata_.mutable_unknown_fields(), ptr, ctx);
         GOOGLE_PROTOBUF_PARSER_ASSERT(ptr != nullptr);
-        if (res.second) return ptr;
+        break;
       }
     }  // switch
   }  // while
   return ptr;
-string_till_end:
-  static_cast<::std::string*>(object)->clear();
-  static_cast<::std::string*>(object)->reserve(size);
-  goto len_delim_till_end;
-len_delim_till_end:
-  return ctx->StoreAndTailCall(ptr, end, {_InternalParse, msg},
-                               {parser_till_end, object}, size);
 }
 #else  // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
 bool FieldMask::MergePartialFromCodedStream(

src/google/protobuf/field_mask.pb.h

@@ -31,6 +31,13 @@
 #include <google/protobuf/repeated_field.h>  // IWYU pragma: export
 #include <google/protobuf/extension_set.h>  // IWYU pragma: export
 #include <google/protobuf/unknown_field_set.h>
+namespace google {
+namespace protobuf {
+namespace internal {
+class AnyMetadata;
+}  // namespace internal
+}  // namespace protobuf
+}  // namespace google
 // @@protoc_insertion_point(includes)
 #include <google/protobuf/port_def.inc>
 #define PROTOBUF_INTERNAL_EXPORT_google_2fprotobuf_2ffield_5fmask_2eproto PROTOBUF_EXPORT
@@ -131,8 +138,7 @@ class PROTOBUF_EXPORT FieldMask final :
 
   size_t ByteSizeLong() const final;
   #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
-  static const char* _InternalParse(const char* begin, const char* end, void* object, ::google::protobuf::internal::ParseContext* ctx);
-  ::google::protobuf::internal::ParseFunc _ParseFunc() const final { return _InternalParse; }
+  const char* _InternalParse(const char* ptr, ::google::protobuf::internal::ParseContext* ctx) final;
   #else
   bool MergePartialFromCodedStream(
       ::google::protobuf::io::CodedInputStream* input) final;
@@ -144,10 +150,14 @@ class PROTOBUF_EXPORT FieldMask final :
   int GetCachedSize() const final { return _cached_size_.Get(); }
 
   private:
-  void SharedCtor();
-  void SharedDtor();
+  inline void SharedCtor();
+  inline void SharedDtor();
   void SetCachedSize(int size) const final;
   void InternalSwap(FieldMask* other);
+  friend class ::google::protobuf::internal::AnyMetadata;
+  static ::google::protobuf::StringPiece FullMessageName() {
+    return "google.protobuf.FieldMask";
+  }
   protected:
   explicit FieldMask(::google::protobuf::Arena* arena);
   private:

src/google/protobuf/generated_message_table_driven.h

@@ -209,15 +209,12 @@ static_assert(sizeof(ParseTableField) <= 16, "ParseTableField is too large");
 // The tables must be composed of POD components to ensure link-time
 // initialization.
 static_assert(std::is_pod<ParseTableField>::value, "");
+static_assert(std::is_pod<AuxillaryParseTableField>::value, "");
 static_assert(std::is_pod<AuxillaryParseTableField::enum_aux>::value, "");
 static_assert(std::is_pod<AuxillaryParseTableField::message_aux>::value, "");
 static_assert(std::is_pod<AuxillaryParseTableField::string_aux>::value, "");
 static_assert(std::is_pod<ParseTable>::value, "");
 
-#ifndef __NVCC__  // This assertion currently fails under NVCC.
-static_assert(std::is_pod<AuxillaryParseTableField>::value, "");
-#endif
-
 // TODO(ckennelly): Consolidate these implementations into a single one, using
 // dynamic dispatch to the appropriate unknown field handler.
 bool MergePartialFromCodedStream(MessageLite* msg, const ParseTable& table,

src/google/protobuf/generated_message_util.h

@@ -44,9 +44,9 @@
 #include <string>
 #include <vector>
 
-#include <google/protobuf/stubs/logging.h>
 #include <google/protobuf/stubs/common.h>
 #include <google/protobuf/parse_context.h>
+#include <google/protobuf/arenastring.h>
 #include <google/protobuf/has_bits.h>
 #include <google/protobuf/implicit_weak_message.h>
 #include <google/protobuf/message_lite.h>
@@ -218,31 +218,6 @@ inline void OnShutdownDestroyString(const ::std::string* ptr) {
   OnShutdownRun(DestroyString, ptr);
 }
 
-#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
-// To simplify generation of the parse loop code we take objects by void ptr.
-inline void InlineGreedyStringParser(void* str, const char* begin, int size,
-                                     ParseContext*) {
-  static_cast<std::string*>(str)->assign(begin, size);
-}
-
-
-inline bool StringCheck(const char* begin, int size, ParseContext* ctx) {
-  return true;
-}
-
-inline bool StringCheckUTF8(const char* begin, int size, ParseContext* ctx) {
-  return VerifyUTF8(StringPiece(begin, size), ctx);
-}
-
-inline bool StringCheckUTF8Verify(const char* begin, int size,
-                                  ParseContext* ctx) {
-#ifndef NDEBUG
-  VerifyUTF8(StringPiece(begin, size), ctx);
-#endif
-  return true;
-}
-
-#endif  // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
 
 }  // namespace internal
 }  // namespace protobuf

src/google/protobuf/implicit_weak_message.cc

@@ -48,11 +48,9 @@ bool ImplicitWeakMessage::MergePartialFromCodedStream(io::CodedInputStream* inpu
 }
 
 #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
-const char* ImplicitWeakMessage::_InternalParse(const char* begin,
-                                                const char* end, void* object,
+const char* ImplicitWeakMessage::_InternalParse(const char* ptr,
                                                 ParseContext* ctx) {
-  return internal::StringParser(
-      begin, end, &(static_cast<ImplicitWeakMessage*>(object)->data_), ctx);
+  return ctx->AppendString(ptr, &data_);
 }
 #endif
 

src/google/protobuf/implicit_weak_message.h

@@ -79,10 +79,7 @@ class PROTOBUF_EXPORT ImplicitWeakMessage : public MessageLite {
   bool MergePartialFromCodedStream(io::CodedInputStream* input) override;
 
 #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
-  ParseFunc _ParseFunc() const override { return _InternalParse; }
-
-  static const char* _InternalParse(const char* begin, const char* end,
-                                    void* object, ParseContext* ctx);
+  const char* _InternalParse(const char* ptr, ParseContext* ctx) final;
 #endif
 
   size_t ByteSizeLong() const override { return data_.size(); }

src/google/protobuf/io/coded_stream.h

@@ -144,8 +144,12 @@ namespace protobuf {
 
 class DescriptorPool;
 class MessageFactory;
+class ZeroCopyCodedInputStream;
 
-namespace internal { void MapTestForceDeterministic(); }
+namespace internal {
+void MapTestForceDeterministic();
+class EpsCopyByteStream;
+}  // namespace internal
 
 namespace io {
 
@@ -157,64 +161,6 @@ class CodedOutputStream;
 class ZeroCopyInputStream;           // zero_copy_stream.h
 class ZeroCopyOutputStream;          // zero_copy_stream.h
 
-template <typename T>
-T UnalignedLoad(const void* p) {
-  T res;
-  memcpy(&res, p, sizeof(T));
-  return res;
-}
-
-// TODO(gerbens) Experiment with best implementation.
-// Clang unrolls loop and generating pretty good code on O2, gcc doesn't.
-// Unclear if we want 64 bit parse loop unrolled, inlined or opaque function
-// call. Hence experimentation is needed.
-// Important guarantee is that it doesn't read more than size bytes from p.
-template <int size, typename T>
-const char* VarintParse(const char* p, T* out) {
-  T res = 0;
-  T extra = 0;
-  for (int i = 0; i < size; i++) {
-    T byte = static_cast<uint8>(p[i]);
-    res += byte << (i * 7);
-    int j = i + 1;
-    if (PROTOBUF_PREDICT_TRUE(byte < 128)) {
-      *out = res - extra;
-      return p + j;
-    }
-    extra += 128ull << (i * 7);
-  }
-  *out = 0;
-  return nullptr;
-}
-
-inline const char* Parse32(const char* p, uint32* out) {
-  return VarintParse<5>(p, out);
-}
-inline const char* Parse64(const char* p, uint64* out) {
-  return VarintParse<10>(p, out);
-}
-
-inline const char* ReadSize(const char* p, int32* out) {
-  int32 res = 0;
-  int32 extra = 0;
-  for (int i = 0; i < 4; i++) {
-    uint32 byte = static_cast<uint8>(p[i]);
-    res += byte << (i * 7);
-    int j = i + 1;
-    if (PROTOBUF_PREDICT_TRUE(byte < 128)) {
-      *out = res - extra;
-      return p + j;
-    }
-    extra += 128ull << (i * 7);
-  }
-  uint32 byte = static_cast<uint8>(p[4]);
-  // size may not be negative, so only the lowest 3 bits can be set.
-  if (byte >= 8) return nullptr;
-  res += byte << (4 * 7);
-  *out = res - extra;
-  return p + 5;
-}
-
 // Class which reads and decodes binary data which is composed of varint-
 // encoded integers and fixed-width pieces.  Wraps a ZeroCopyInputStream.
 // Most users will not need to deal with CodedInputStream.
@@ -693,6 +639,9 @@ class PROTOBUF_EXPORT CodedInputStream {
   static const int kDefaultTotalBytesLimit = INT_MAX;
 
   static int default_recursion_limit_;  // 100 by default.
+
+  friend class google::protobuf::ZeroCopyCodedInputStream;
+  friend class google::protobuf::internal::EpsCopyByteStream;
 };
 
 // Class which encodes and writes binary data which is composed of varint-

src/google/protobuf/io/zero_copy_stream_impl.cc

@@ -78,7 +78,6 @@ int close_no_eintr(int fd) {
 
 }  // namespace
 
-
 // ===================================================================
 
 FileInputStream::FileInputStream(int file_descriptor, int block_size)

src/google/protobuf/io/zero_copy_stream_impl.h

@@ -53,7 +53,6 @@ namespace google {
 namespace protobuf {
 namespace io {
 
-
 // ===================================================================
 
 // A ZeroCopyInputStream which reads from a file descriptor.

src/google/protobuf/map.h

@@ -414,7 +414,7 @@ class Map {
           : node_(NodePtrFromKeyPtr(*tree_it)), m_(m), bucket_index_(index) {
         // Invariant: iterators that use buckets with trees have an even
         // bucket_index_.
-        GOOGLE_DCHECK_EQ(bucket_index_ % 2, 0);
+        GOOGLE_DCHECK_EQ(bucket_index_ % 2, 0u);
       }
 
       // Advance through buckets, looking for the first that isn't empty.
@@ -454,7 +454,7 @@ class Map {
           if (is_list) {
             SearchFrom(bucket_index_ + 1);
           } else {
-            GOOGLE_DCHECK_EQ(bucket_index_ & 1, 0);
+            GOOGLE_DCHECK_EQ(bucket_index_ & 1, 0u);
             Tree* tree = static_cast<Tree*>(m_->table_[bucket_index_]);
             if (++tree_it == tree->end()) {
               SearchFrom(bucket_index_ + 2);

src/google/protobuf/map_entry_lite.h

@@ -36,6 +36,7 @@
 
 #include <google/protobuf/stubs/casts.h>
 #include <google/protobuf/parse_context.h>
+#include <google/protobuf/io/coded_stream.h>
 #include <google/protobuf/arena.h>
 #include <google/protobuf/arenastring.h>
 #include <google/protobuf/map.h>
@@ -193,6 +194,31 @@ class MapEntryImpl : public Base {
     MergeFromInternal(*::google::protobuf::down_cast<const Derived*>(&other));
   }
 
+#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
+  const char* _InternalParse(const char* ptr, ParseContext* ctx) final {
+    while (!ctx->Done(&ptr)) {
+      uint32 tag;
+      ptr = ReadTag(ptr, &tag);
+      GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
+      if (tag == kKeyTag) {
+        set_has_key();
+        ptr = KeyTypeHandler::Read(ptr, ctx, mutable_key());
+      } else if (tag == kValueTag) {
+        set_has_value();
+        ptr = ValueTypeHandler::Read(ptr, ctx, mutable_value());
+      } else {
+        if (tag == 0 || WireFormatLite::GetTagWireType(tag) ==
+                            WireFormatLite::WIRETYPE_END_GROUP) {
+          ctx->SetLastTag(tag);
+          return ptr;
+        }
+        ptr = UnknownFieldParse(tag, nullptr, ptr, ctx);
+      }
+      GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
+    }
+    return ptr;
+  }
+#else
   bool MergePartialFromCodedStream(io::CodedInputStream* input) override {
     uint32 tag;
 
@@ -233,6 +259,7 @@ class MapEntryImpl : public Base {
       }
     }
   }
+#endif
 
   size_t ByteSizeLong() const override {
     size_t size = 0;
@@ -386,35 +413,12 @@ class MapEntryImpl : public Base {
       return result;
     }
 
-#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
-    bool ParseMap(const char* begin, const char* end) {
-      io::CodedInputStream input(reinterpret_cast<const uint8*>(begin),
-                                 end - begin);
-      return MergePartialFromCodedStream(&input) &&
-             input.ConsumedEntireMessage();
-    }
-    template <typename Metadata>
-    bool ParseMapEnumValidation(const char* begin, const char* end, uint32 num,
-                                Metadata* metadata,
-                                bool (*validate_enum)(int)) {
-      io::CodedInputStream input(reinterpret_cast<const uint8*>(begin),
-                                 static_cast<int>(end - begin));
+    const char* _InternalParse(const char* ptr, ParseContext* ctx) {
       auto entry = NewEntry();
-      // TODO(gerbens) implement _InternalParse for maps. We can't use
-      // ParseFromString as this will call _InternalParse
-      if (!(entry->MergePartialFromCodedStream(&input) &&
-            input.ConsumedEntireMessage()))
-        return false;
-      if (!validate_enum(entry->value())) {
-        auto unknown_fields = metadata->mutable_unknown_fields();
-        WriteLengthDelimited(num, StringPiece(begin, end - begin),
-                             unknown_fields);
-        return true;
-      }
-      (*map_)[entry->key()] = static_cast<Value>(entry->value());
-      return true;
+      ptr = entry->_InternalParse(ptr, ctx);
+      UseKeyAndValueFromEntry();
+      return ptr;
     }
-#endif
 
     MapEntryImpl* NewEntry() { return entry_ = mf_->NewEntry(); }
 
@@ -425,7 +429,7 @@ class MapEntryImpl : public Base {
     const Value& entry_value() const { return entry_->value(); }
 
    private:
-    void UseKeyAndValueFromEntry() PROTOBUF_COLD {
+    void UseKeyAndValueFromEntry() {
       // Update key_ in case we need it later (because key() is called).
       // This is potentially inefficient, especially if the key is
       // expensive to copy (e.g., a long string), but this is a cold
@@ -477,7 +481,7 @@ class MapEntryImpl : public Base {
   bool has_value() const { return (_has_bits_[0] & 0x00000002u) != 0; }
   void clear_has_value() { _has_bits_[0] &= ~0x00000002u; }
 
- private:
+ public:
   // Serializing a generated message containing map field involves serializing
   // key-value pairs from Map. The wire format of each key-value pair
   // after serialization should be the same as that of a MapEntry message

src/google/protobuf/map_field.h

@@ -288,6 +288,10 @@ class MapField : public TypeDefinedMapFieldBase<Key, T> {
     return impl_.NewEntryWrapper(key, t);
   }
 
+  const char* _InternalParse(const char* ptr, ParseContext* ctx) {
+    return impl_._InternalParse(ptr, ctx);
+  }
+
  private:
   MapFieldLiteType impl_;
 

src/google/protobuf/map_field_lite.h

@@ -32,6 +32,7 @@
 #define GOOGLE_PROTOBUF_MAP_FIELD_LITE_H__
 
 #include <type_traits>
+#include <google/protobuf/parse_context.h>
 #include <google/protobuf/io/coded_stream.h>
 #include <google/protobuf/map.h>
 #include <google/protobuf/map_entry_lite.h>
@@ -108,6 +109,11 @@ class MapFieldLite {
     return EntryType::Wrap(key, t, arena_);
   }
 
+  const char* _InternalParse(const char* ptr, ParseContext* ctx) {
+    typename Derived::template Parser<MapFieldLite, Map<Key, T>> parser(this);
+    return parser._InternalParse(ptr, ctx);
+  }
+
  private:
   typedef void DestructorSkippable_;
 

src/google/protobuf/map_field_test.cc

@@ -309,8 +309,8 @@ class MapFieldStateTest
   State state_;
 };
 
-INSTANTIATE_TEST_CASE_P(MapFieldStateTestInstance, MapFieldStateTest,
-                        ::testing::Values(CLEAN, MAP_DIRTY, REPEATED_DIRTY));
+INSTANTIATE_TEST_SUITE_P(MapFieldStateTestInstance, MapFieldStateTest,
+                         ::testing::Values(CLEAN, MAP_DIRTY, REPEATED_DIRTY));
 
 TEST_P(MapFieldStateTest, GetMap) {
   map_field_->GetMap();

src/google/protobuf/map_type_handler.h

@@ -31,7 +31,10 @@
 #ifndef GOOGLE_PROTOBUF_TYPE_HANDLER_H__
 #define GOOGLE_PROTOBUF_TYPE_HANDLER_H__
 
+#include <google/protobuf/parse_context.h>
+#include <google/protobuf/io/coded_stream.h>
 #include <google/protobuf/arena.h>
+#include <google/protobuf/wire_format_lite.h>
 #include <google/protobuf/wire_format_lite_inl.h>
 
 #ifdef SWIG
@@ -165,6 +168,9 @@ class MapTypeHandler<WireFormatLite::TYPE_MESSAGE, Type> {
   static inline int GetCachedSize(const MapEntryAccessorType& value);
   static inline bool Read(io::CodedInputStream* input,
                           MapEntryAccessorType* value);
+  static inline const char* Read(const char* ptr, ParseContext* ctx,
+                                 MapEntryAccessorType* value);
+
   static inline void Write(int field, const MapEntryAccessorType& value,
                            io::CodedOutputStream* output);
   static inline uint8* WriteToArray(int field,
@@ -221,6 +227,8 @@ class MapTypeHandler<WireFormatLite::TYPE_MESSAGE, Type> {
     static inline int GetCachedSize(const MapEntryAccessorType& value);       \
     static inline bool Read(io::CodedInputStream* input,                      \
                             MapEntryAccessorType* value);                     \
+    static inline const char* Read(const char* begin, ParseContext* ctx,      \
+                                   MapEntryAccessorType* value);              \
     static inline void Write(int field, const MapEntryAccessorType& value,    \
                              io::CodedOutputStream* output);                  \
     static inline uint8* WriteToArray(int field,                              \
@@ -422,6 +430,96 @@ inline bool MapTypeHandler<WireFormatLite::TYPE_BYTES, Type>::Read(
   return WireFormatLite::ReadBytes(input, value);
 }
 
+template <typename Type>
+const char* MapTypeHandler<WireFormatLite::TYPE_MESSAGE, Type>::Read(
+    const char* ptr, ParseContext* ctx, MapEntryAccessorType* value) {
+  return ctx->ParseMessage(value, ptr);
+}
+
+template <typename Type>
+const char* MapTypeHandler<WireFormatLite::TYPE_STRING, Type>::Read(
+    const char* ptr, ParseContext* ctx, MapEntryAccessorType* value) {
+  int size = ReadSize(&ptr);
+  GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
+  return ctx->ReadString(ptr, size, value);
+}
+
+template <typename Type>
+const char* MapTypeHandler<WireFormatLite::TYPE_BYTES, Type>::Read(
+    const char* ptr, ParseContext* ctx, MapEntryAccessorType* value) {
+  int size = ReadSize(&ptr);
+  GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
+  return ctx->ReadString(ptr, size, value);
+}
+
+inline const char* ReadINT64(const char* ptr, int64* value) {
+  return ParseVarint64(ptr, reinterpret_cast<uint64*>(value));
+}
+inline const char* ReadUINT64(const char* ptr, uint64* value) {
+  return ParseVarint64(ptr, value);
+}
+inline const char* ReadINT32(const char* ptr, int32* value) {
+  uint64 tmp;
+  auto res = ParseVarint64(ptr, &tmp);
+  *value = static_cast<uint32>(tmp);
+  return res;
+}
+inline const char* ReadUINT32(const char* ptr, uint32* value) {
+  uint64 tmp;
+  auto res = ParseVarint64(ptr, &tmp);
+  *value = static_cast<uint32>(tmp);
+  return res;
+}
+inline const char* ReadSINT64(const char* ptr, int64* value) {
+  uint64 tmp;
+  auto res = ParseVarint64(ptr, &tmp);
+  *value = WireFormatLite::ZigZagDecode64(tmp);
+  return res;
+}
+inline const char* ReadSINT32(const char* ptr, int32* value) {
+  uint64 tmp;
+  auto res = ParseVarint64(ptr, &tmp);
+  *value = WireFormatLite::ZigZagDecode32(static_cast<uint32>(tmp));
+  return res;
+}
+template <typename E>
+inline const char* ReadENUM(const char* ptr, E* value) {
+  uint64 tmp;
+  auto res = ParseVarint64(ptr, &tmp);
+  *value = static_cast<E>(tmp);
+  return res;
+}
+inline const char* ReadBOOL(const char* ptr, bool* value) {
+  uint64 tmp;
+  auto res = ParseVarint64(ptr, &tmp);
+  *value = static_cast<bool>(tmp);
+  return res;
+}
+
+template <typename F>
+inline const char* ReadUnaligned(const char* ptr, F* value) {
+  *value = UnalignedLoad<F>(ptr);
+  return ptr + sizeof(F);
+}
+inline const char* ReadFLOAT(const char* ptr, float* value) {
+  return ReadUnaligned(ptr, value);
+}
+inline const char* ReadDOUBLE(const char* ptr, double* value) {
+  return ReadUnaligned(ptr, value);
+}
+inline const char* ReadFIXED64(const char* ptr, uint64* value) {
+  return ReadUnaligned(ptr, value);
+}
+inline const char* ReadFIXED32(const char* ptr, uint32* value) {
+  return ReadUnaligned(ptr, value);
+}
+inline const char* ReadSFIXED64(const char* ptr, int64* value) {
+  return ReadUnaligned(ptr, value);
+}
+inline const char* ReadSFIXED32(const char* ptr, int32* value) {
+  return ReadUnaligned(ptr, value);
+}
+
 #define READ_METHOD(FieldType)                                              \
   template <typename Type>                                                  \
   inline bool MapTypeHandler<WireFormatLite::TYPE_##FieldType, Type>::Read( \
@@ -429,6 +527,11 @@ inline bool MapTypeHandler<WireFormatLite::TYPE_BYTES, Type>::Read(
     return WireFormatLite::ReadPrimitive<TypeOnMemory,                      \
                                          WireFormatLite::TYPE_##FieldType>( \
         input, value);                                                      \
+  }                                                                         \
+  template <typename Type>                                                  \
+  const char* MapTypeHandler<WireFormatLite::TYPE_##FieldType, Type>::Read( \
+      const char* begin, ParseContext* ctx, MapEntryAccessorType* value) {  \
+    return Read##FieldType(begin, value);                                   \
   }
 
 READ_METHOD(INT64)