Integrate internal changes

conformance/Makefile.am

@@ -251,31 +251,31 @@ conformance-csharp: $(other_language_protoc_outputs)
 
 # Targets for actually running tests.
 test_cpp: protoc_middleman conformance-test-runner conformance-cpp
-	./conformance-test-runner --failure_list failure_list_cpp.txt ./conformance-cpp
+	./conformance-test-runner --enforce_recommended --failure_list failure_list_cpp.txt ./conformance-cpp
 
 test_java: protoc_middleman conformance-test-runner conformance-java
-	./conformance-test-runner --failure_list failure_list_java.txt ./conformance-java
+	./conformance-test-runner --enforce_recommended --failure_list failure_list_java.txt ./conformance-java
 
 test_java_lite: protoc_middleman conformance-test-runner conformance-java-lite
-	./conformance-test-runner ./conformance-java-lite
+	./conformance-test-runner --enforce_recommended ./conformance-java-lite
 
 test_csharp: protoc_middleman conformance-test-runner conformance-csharp
-	./conformance-test-runner --failure_list failure_list_csharp.txt ./conformance-csharp
+	./conformance-test-runner --enforce_recommended --failure_list failure_list_csharp.txt ./conformance-csharp
 
 test_ruby: protoc_middleman conformance-test-runner $(other_language_protoc_outputs)
-	RUBYLIB=../ruby/lib:. ./conformance-test-runner --failure_list failure_list_ruby.txt ./conformance_ruby.rb
+	RUBYLIB=../ruby/lib:. ./conformance-test-runner --enforce_recommended --failure_list failure_list_ruby.txt ./conformance_ruby.rb
 
 # These depend on library paths being properly set up.  The easiest way to
 # run them is to just use "tox" from the python dir.
 test_python: protoc_middleman conformance-test-runner
-	./conformance-test-runner --failure_list failure_list_python.txt ./conformance_python.py
+	./conformance-test-runner --enforce_recommended --failure_list failure_list_python.txt ./conformance_python.py
 
 test_python_cpp: protoc_middleman conformance-test-runner
-	./conformance-test-runner --failure_list failure_list_python_cpp.txt ./conformance_python.py
+	./conformance-test-runner --enforce_recommended --failure_list failure_list_python_cpp.txt ./conformance_python.py
 
 if OBJC_CONFORMANCE_TEST
 
 test_objc: protoc_middleman conformance-test-runner conformance-objc
-	./conformance-test-runner --failure_list failure_list_objc.txt ./conformance-objc
+	./conformance-test-runner --enforce_recomemnded --failure_list failure_list_objc.txt ./conformance-objc
 
 endif

conformance/conformance_python.py

@@ -67,7 +67,7 @@ def do_test(request):
     elif request.WhichOneof('payload') == 'json_payload':
       try:
         json_format.Parse(request.json_payload, test_message)
-      except json_format.ParseError as e:
+      except Exception as e:
         response.parse_error = str(e)
         return response
 
@@ -81,7 +81,11 @@ def do_test(request):
       response.protobuf_payload = test_message.SerializeToString()
 
     elif request.requested_output_format == conformance_pb2.JSON:
-      response.json_payload = json_format.MessageToJson(test_message)
+      try:
+        response.json_payload = json_format.MessageToJson(test_message)
+      except Exception as e:
+        response.serialize_error = str(e)
+        return response
 
   except Exception as e:
     response.runtime_error = str(e)

conformance/conformance_test.h

@@ -91,7 +91,7 @@ class ConformanceTestRunner {
 //
 class ConformanceTestSuite {
  public:
-  ConformanceTestSuite() : verbose_(false) {}
+  ConformanceTestSuite() : verbose_(false), enforce_recommended_(false) {}
 
   void SetVerbose(bool verbose) { verbose_ = verbose; }
 
@@ -104,6 +104,18 @@ class ConformanceTestSuite {
   void SetFailureList(const std::string& filename,
                       const std::vector<std::string>& failure_list);
 
+  // Whether to require the testee to pass RECOMMENDED tests. By default failing
+  // a RECOMMENDED test case will not fail the entire suite but will only
+  // generated a warning. If this flag is set to true, RECOMMENDED tests will
+  // be treated the same way as REQUIRED tests and failing a RECOMMENDED test
+  // case will cause the entire test suite to fail as well. An implementation
+  // can enable this if it wants to be strictly conforming to protobuf spec.
+  // See the comments about ConformanceLevel below to learn more about the
+  // difference between REQUIRED and RECOMMENDED test cases.
+  void SetEnforceRecommended(bool value) {
+    enforce_recommended_ = value;
+  }
+
   // Run all the conformance tests against the given test runner.
   // Test output will be stored in "output".
   //
@@ -113,8 +125,27 @@ class ConformanceTestSuite {
   bool RunSuite(ConformanceTestRunner* runner, std::string* output);
 
  private:
+  // Test cases are classified into a few categories:
+  //   REQUIRED: the test case must be passed for an implementation to be
+  //             interoperable with other implementations. For example, a
+  //             parser implementaiton must accept both packed and unpacked
+  //             form of repeated primitive fields.
+  //   RECOMMENDED: the test case is not required for the implementation to
+  //                be interoperable with other implementations, but is
+  //                recommended for best performance and compatibility. For
+  //                example, a proto3 serializer should serialize repeated
+  //                primitive fields in packed form, but an implementation
+  //                failing to do so will still be able to communicate with
+  //                other implementations.
+  enum ConformanceLevel {
+    REQUIRED = 0,
+    RECOMMENDED = 1,
+  };
+  string ConformanceLevelToString(ConformanceLevel level);
+
   void ReportSuccess(const std::string& test_name);
   void ReportFailure(const string& test_name,
+                     ConformanceLevel level,
                      const conformance::ConformanceRequest& request,
                      const conformance::ConformanceResponse& response,
                      const char* fmt, ...);
@@ -124,13 +155,18 @@ class ConformanceTestSuite {
   void RunTest(const std::string& test_name,
                const conformance::ConformanceRequest& request,
                conformance::ConformanceResponse* response);
-  void RunValidInputTest(const string& test_name, const string& input,
+  void RunValidInputTest(const string& test_name,
+                         ConformanceLevel level,
+                         const string& input,
                          conformance::WireFormat input_format,
                          const string& equivalent_text_format,
                          conformance::WireFormat requested_output);
-  void RunValidJsonTest(const string& test_name, const string& input_json,
+  void RunValidJsonTest(const string& test_name,
+                        ConformanceLevel level,
+                        const string& input_json,
                         const string& equivalent_text_format);
   void RunValidJsonTestWithProtobufInput(const string& test_name,
+                                         ConformanceLevel level,
                                          const conformance::TestAllTypes& input,
                                          const string& equivalent_text_format);
   void RunValidProtobufTest(const string& test_name,
@@ -139,16 +175,21 @@ class ConformanceTestSuite {
 
   typedef std::function<bool(const Json::Value&)> Validator;
   void RunValidJsonTestWithValidator(const string& test_name,
+                                     ConformanceLevel level,
                                      const string& input_json,
                                      const Validator& validator);
   void ExpectParseFailureForJson(const string& test_name,
+                                 ConformanceLevel level,
                                  const string& input_json);
   void ExpectSerializeFailureForJson(const string& test_name,
+                                     ConformanceLevel level,
                                      const string& text_format);
   void ExpectParseFailureForProto(const std::string& proto,
-                                  const std::string& test_name);
+                                  const std::string& test_name,
+                                  ConformanceLevel level);
   void ExpectHardParseFailureForProto(const std::string& proto,
-                                      const std::string& test_name);
+                                      const std::string& test_name,
+                                      ConformanceLevel level);
   void TestPrematureEOFForType(google::protobuf::FieldDescriptor::Type type);
   bool CheckSetEmpty(const set<string>& set_to_check,
                      const std::string& write_to_file, const std::string& msg);
@@ -156,6 +197,7 @@ class ConformanceTestSuite {
   int successes_;
   int expected_failures_;
   bool verbose_;
+  bool enforce_recommended_;
   std::string output_;
   std::string failure_list_filename_;
 

conformance/conformance_test_runner.cc

@@ -251,6 +251,16 @@ void UsageError() {
           "                              should contain one test name per\n");
   fprintf(stderr,
           "                              line.  Use '#' for comments.\n");
+  fprintf(stderr,
+          "  --enforce_recommended       Enforce that recommended test\n");
+  fprintf(stderr,
+          "                              cases are also passing. Specify\n");
+  fprintf(stderr,
+          "                              this flag if you want to be\n");
+  fprintf(stderr,
+          "                              strictly conforming to protobuf\n");
+  fprintf(stderr,
+          "                              spec.\n");
   exit(1);
 }
 
@@ -290,6 +300,8 @@ int main(int argc, char *argv[]) {
       ParseFailureList(argv[arg], &failure_list);
     } else if (strcmp(argv[arg], "--verbose") == 0) {
       suite.SetVerbose(true);
+    } else if (strcmp(argv[arg], "--enforce_recommended") == 0) {
+      suite.SetEnforceRecommended(true);
     } else if (argv[arg][0] == '-') {
       fprintf(stderr, "Unknown option: %s\n", argv[arg]);
       UsageError();

conformance/failure_list_cpp.txt

@@ -7,48 +7,41 @@
 # TODO(haberman): insert links to corresponding bugs tracking the issue.
 # Should we use GitHub issues or the Google-internal bug tracker?
 
-FieldMaskNumbersDontRoundTrip.JsonOutput
-FieldMaskPathsDontRoundTrip.JsonOutput
-FieldMaskTooManyUnderscore.JsonOutput
-JsonInput.AnyUnorderedTypeTag.JsonOutput
-JsonInput.AnyUnorderedTypeTag.ProtobufOutput
-JsonInput.BoolFieldDoubleQuotedFalse
-JsonInput.BoolFieldDoubleQuotedTrue
-JsonInput.BytesFieldNoPadding
-JsonInput.DoubleFieldTooSmall
-JsonInput.DurationHasZeroFractionalDigit.Validator
-JsonInput.EnumFieldUnknownValue.Validator
-JsonInput.FieldMaskInvalidCharacter
-JsonInput.FieldNameDuplicate
-JsonInput.FieldNameDuplicateDifferentCasing1
-JsonInput.FieldNameDuplicateDifferentCasing2
-JsonInput.FieldNameNotQuoted
-JsonInput.MapFieldValueIsNull
-JsonInput.RepeatedFieldMessageElementIsNull
-JsonInput.RepeatedFieldPrimitiveElementIsNull
-JsonInput.RepeatedFieldTrailingComma
-JsonInput.RepeatedFieldTrailingCommaWithNewlines
-JsonInput.RepeatedFieldTrailingCommaWithSpace
-JsonInput.RepeatedFieldTrailingCommaWithSpaceCommaSpace
-JsonInput.StringFieldSingleQuoteBoth
-JsonInput.StringFieldSingleQuoteKey
-JsonInput.StringFieldSingleQuoteValue
-JsonInput.StringFieldUppercaseEscapeLetter
-JsonInput.TrailingCommaInAnObject
-JsonInput.TrailingCommaInAnObjectWithNewlines
-JsonInput.TrailingCommaInAnObjectWithSpace
-JsonInput.TrailingCommaInAnObjectWithSpaceCommaSpace
-JsonInput.WrapperTypesWithNullValue.JsonOutput
-JsonInput.WrapperTypesWithNullValue.ProtobufOutput
-ProtobufInput.PrematureEofBeforeKnownRepeatedValue.MESSAGE
-ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE
-ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE
-ProtobufInput.PrematureEofInPackedField.BOOL
-ProtobufInput.PrematureEofInPackedField.ENUM
-ProtobufInput.PrematureEofInPackedField.INT32
-ProtobufInput.PrematureEofInPackedField.INT64
-ProtobufInput.PrematureEofInPackedField.SINT32
-ProtobufInput.PrematureEofInPackedField.SINT64
-ProtobufInput.PrematureEofInPackedField.UINT32
-ProtobufInput.PrematureEofInPackedField.UINT64
-ProtobufInput.PrematureEofInsideKnownRepeatedValue.MESSAGE
+Recommended.FieldMaskNumbersDontRoundTrip.JsonOutput
+Recommended.FieldMaskPathsDontRoundTrip.JsonOutput
+Recommended.FieldMaskTooManyUnderscore.JsonOutput
+Recommended.JsonInput.BoolFieldDoubleQuotedFalse
+Recommended.JsonInput.BoolFieldDoubleQuotedTrue
+Recommended.JsonInput.FieldMaskInvalidCharacter
+Recommended.JsonInput.FieldNameDuplicate
+Recommended.JsonInput.FieldNameDuplicateDifferentCasing1
+Recommended.JsonInput.FieldNameDuplicateDifferentCasing2
+Recommended.JsonInput.FieldNameNotQuoted
+Recommended.JsonInput.MapFieldValueIsNull
+Recommended.JsonInput.RepeatedFieldMessageElementIsNull
+Recommended.JsonInput.RepeatedFieldPrimitiveElementIsNull
+Recommended.JsonInput.RepeatedFieldTrailingComma
+Recommended.JsonInput.RepeatedFieldTrailingCommaWithNewlines
+Recommended.JsonInput.RepeatedFieldTrailingCommaWithSpace
+Recommended.JsonInput.RepeatedFieldTrailingCommaWithSpaceCommaSpace
+Recommended.JsonInput.StringFieldSingleQuoteBoth
+Recommended.JsonInput.StringFieldSingleQuoteKey
+Recommended.JsonInput.StringFieldSingleQuoteValue
+Recommended.JsonInput.StringFieldUppercaseEscapeLetter
+Recommended.JsonInput.TrailingCommaInAnObject
+Recommended.JsonInput.TrailingCommaInAnObjectWithNewlines
+Recommended.JsonInput.TrailingCommaInAnObjectWithSpace
+Recommended.JsonInput.TrailingCommaInAnObjectWithSpaceCommaSpace
+Required.JsonInput.DoubleFieldTooSmall
+Required.ProtobufInput.PrematureEofBeforeKnownRepeatedValue.MESSAGE
+Required.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE
+Required.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE
+Required.ProtobufInput.PrematureEofInPackedField.BOOL
+Required.ProtobufInput.PrematureEofInPackedField.ENUM
+Required.ProtobufInput.PrematureEofInPackedField.INT32
+Required.ProtobufInput.PrematureEofInPackedField.INT64
+Required.ProtobufInput.PrematureEofInPackedField.SINT32
+Required.ProtobufInput.PrematureEofInPackedField.SINT64
+Required.ProtobufInput.PrematureEofInPackedField.UINT32
+Required.ProtobufInput.PrematureEofInPackedField.UINT64
+Required.ProtobufInput.PrematureEofInsideKnownRepeatedValue.MESSAGE

conformance/failure_list_java.txt

@@ -4,48 +4,46 @@
 # By listing them here we can keep tabs on which ones are failing and be sure
 # that we don't introduce regressions in other tests.
 
-FieldMaskNumbersDontRoundTrip.JsonOutput
-FieldMaskPathsDontRoundTrip.JsonOutput
-FieldMaskTooManyUnderscore.JsonOutput
-JsonInput.BoolFieldAllCapitalFalse
-JsonInput.BoolFieldAllCapitalTrue
-JsonInput.BoolFieldCamelCaseFalse
-JsonInput.BoolFieldCamelCaseTrue
-JsonInput.BoolFieldDoubleQuotedFalse
-JsonInput.BoolFieldDoubleQuotedTrue
-JsonInput.BoolMapFieldKeyNotQuoted
-JsonInput.DoubleFieldInfinityNotQuoted
-JsonInput.DoubleFieldNanNotQuoted
-JsonInput.DoubleFieldNegativeInfinityNotQuoted
-JsonInput.EnumFieldNotQuoted
-JsonInput.FieldMaskInvalidCharacter
-JsonInput.FieldNameDuplicate
-JsonInput.FieldNameInLowerCamelCase.Validator
-JsonInput.FieldNameInSnakeCase.JsonOutput
-JsonInput.FieldNameInSnakeCase.ProtobufOutput
-JsonInput.FieldNameNotQuoted
-JsonInput.FieldNameWithDoubleUnderscores.JsonOutput
-JsonInput.FieldNameWithDoubleUnderscores.ProtobufOutput
-JsonInput.FieldNameWithDoubleUnderscores.Validator
-JsonInput.FloatFieldInfinityNotQuoted
-JsonInput.FloatFieldNanNotQuoted
-JsonInput.FloatFieldNegativeInfinityNotQuoted
-JsonInput.Int32FieldLeadingZero
-JsonInput.Int32FieldNegativeWithLeadingZero
-JsonInput.Int32FieldPlusSign
-JsonInput.Int32MapFieldKeyNotQuoted
-JsonInput.Int64MapFieldKeyNotQuoted
-JsonInput.JsonWithComments
-JsonInput.OriginalProtoFieldName.JsonOutput
-JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotBool
-JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotInt
-JsonInput.StringFieldNotAString
-JsonInput.StringFieldSingleQuoteBoth
-JsonInput.StringFieldSingleQuoteKey
-JsonInput.StringFieldSingleQuoteValue
-JsonInput.StringFieldSurrogateInWrongOrder
-JsonInput.StringFieldUnpairedHighSurrogate
-JsonInput.StringFieldUnpairedLowSurrogate
-JsonInput.StringFieldUppercaseEscapeLetter
-JsonInput.Uint32MapFieldKeyNotQuoted
-JsonInput.Uint64MapFieldKeyNotQuoted
+Recommended.FieldMaskNumbersDontRoundTrip.JsonOutput
+Recommended.FieldMaskPathsDontRoundTrip.JsonOutput
+Recommended.FieldMaskTooManyUnderscore.JsonOutput
+Recommended.JsonInput.BoolFieldAllCapitalFalse
+Recommended.JsonInput.BoolFieldAllCapitalTrue
+Recommended.JsonInput.BoolFieldCamelCaseFalse
+Recommended.JsonInput.BoolFieldCamelCaseTrue
+Recommended.JsonInput.BoolFieldDoubleQuotedFalse
+Recommended.JsonInput.BoolFieldDoubleQuotedTrue
+Recommended.JsonInput.BoolMapFieldKeyNotQuoted
+Recommended.JsonInput.DoubleFieldInfinityNotQuoted
+Recommended.JsonInput.DoubleFieldNanNotQuoted
+Recommended.JsonInput.DoubleFieldNegativeInfinityNotQuoted
+Recommended.JsonInput.FieldMaskInvalidCharacter
+Recommended.JsonInput.FieldNameDuplicate
+Recommended.JsonInput.FieldNameInLowerCamelCase.Validator
+Recommended.JsonInput.FieldNameInSnakeCase.ProtobufOutput
+Recommended.JsonInput.FieldNameNotQuoted
+Recommended.JsonInput.FieldNameWithDoubleUnderscores.JsonOutput
+Recommended.JsonInput.FieldNameWithDoubleUnderscores.ProtobufOutput
+Recommended.JsonInput.FieldNameWithDoubleUnderscores.Validator
+Recommended.JsonInput.FloatFieldInfinityNotQuoted
+Recommended.JsonInput.FloatFieldNanNotQuoted
+Recommended.JsonInput.FloatFieldNegativeInfinityNotQuoted
+Recommended.JsonInput.Int32MapFieldKeyNotQuoted
+Recommended.JsonInput.Int64MapFieldKeyNotQuoted
+Recommended.JsonInput.JsonWithComments
+Recommended.JsonInput.StringFieldSingleQuoteBoth
+Recommended.JsonInput.StringFieldSingleQuoteKey
+Recommended.JsonInput.StringFieldSingleQuoteValue
+Recommended.JsonInput.StringFieldSurrogateInWrongOrder
+Recommended.JsonInput.StringFieldUnpairedHighSurrogate
+Recommended.JsonInput.StringFieldUnpairedLowSurrogate
+Recommended.JsonInput.StringFieldUppercaseEscapeLetter
+Recommended.JsonInput.Uint32MapFieldKeyNotQuoted
+Recommended.JsonInput.Uint64MapFieldKeyNotQuoted
+Required.JsonInput.EnumFieldNotQuoted
+Required.JsonInput.Int32FieldLeadingZero
+Required.JsonInput.Int32FieldNegativeWithLeadingZero
+Required.JsonInput.Int32FieldPlusSign
+Required.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotBool
+Required.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotInt
+Required.JsonInput.StringFieldNotAString

conformance/failure_list_python.txt

@@ -1,46 +1,15 @@
-DurationProtoInputTooLarge.JsonOutput
-DurationProtoInputTooSmall.JsonOutput
-FieldMaskNumbersDontRoundTrip.JsonOutput
-FieldMaskPathsDontRoundTrip.JsonOutput
-FieldMaskTooManyUnderscore.JsonOutput
-JsonInput.AnyWithFieldMask.ProtobufOutput
-JsonInput.BytesFieldInvalidBase64Characters
-JsonInput.DoubleFieldInfinityNotQuoted
-JsonInput.DoubleFieldNanNotQuoted
-JsonInput.DoubleFieldNegativeInfinityNotQuoted
-JsonInput.DoubleFieldTooSmall
-JsonInput.DurationJsonInputTooLarge
-JsonInput.DurationJsonInputTooSmall
-JsonInput.DurationMissingS
-JsonInput.EnumFieldNumericValueNonZero.JsonOutput
-JsonInput.EnumFieldNumericValueNonZero.ProtobufOutput
-JsonInput.EnumFieldNumericValueZero.JsonOutput
-JsonInput.EnumFieldNumericValueZero.ProtobufOutput
-JsonInput.EnumFieldUnknownValue.Validator
-JsonInput.FieldMask.ProtobufOutput
-JsonInput.FieldMaskInvalidCharacter
-JsonInput.FloatFieldInfinityNotQuoted
-JsonInput.FloatFieldNanNotQuoted
-JsonInput.FloatFieldNegativeInfinityNotQuoted
-JsonInput.FloatFieldTooLarge
-JsonInput.FloatFieldTooSmall
-JsonInput.Int32FieldExponentialFormat.JsonOutput
-JsonInput.Int32FieldExponentialFormat.ProtobufOutput
-JsonInput.Int32FieldFloatTrailingZero.JsonOutput
-JsonInput.Int32FieldFloatTrailingZero.ProtobufOutput
-JsonInput.Int32FieldMaxFloatValue.JsonOutput
-JsonInput.Int32FieldMaxFloatValue.ProtobufOutput
-JsonInput.Int32FieldMinFloatValue.JsonOutput
-JsonInput.Int32FieldMinFloatValue.ProtobufOutput
-JsonInput.OneofZeroMessage.JsonOutput
-JsonInput.OneofZeroMessage.ProtobufOutput
-JsonInput.OriginalProtoFieldName.JsonOutput
-JsonInput.OriginalProtoFieldName.ProtobufOutput
-JsonInput.RepeatedFieldWrongElementTypeExpectingIntegersGotBool
-JsonInput.TimestampJsonInputLowercaseT
-JsonInput.Uint32FieldMaxFloatValue.JsonOutput
-JsonInput.Uint32FieldMaxFloatValue.ProtobufOutput
-JsonInput.ValueAcceptNull.JsonOutput
-JsonInput.ValueAcceptNull.ProtobufOutput
-TimestampProtoInputTooLarge.JsonOutput
-TimestampProtoInputTooSmall.JsonOutput
+Recommended.JsonInput.DoubleFieldInfinityNotQuoted
+Recommended.JsonInput.DoubleFieldNanNotQuoted
+Recommended.JsonInput.DoubleFieldNegativeInfinityNotQuoted
+Recommended.JsonInput.FloatFieldInfinityNotQuoted
+Recommended.JsonInput.FloatFieldNanNotQuoted
+Recommended.JsonInput.FloatFieldNegativeInfinityNotQuoted
+Recommended.JsonInput.OneofZeroMessage.JsonOutput
+Recommended.JsonInput.OneofZeroMessage.ProtobufOutput
+Required.JsonInput.BytesFieldInvalidBase64Characters
+Required.JsonInput.DoubleFieldTooSmall
+Required.JsonInput.EnumFieldUnknownValue.Validator
+Required.JsonInput.FloatFieldTooLarge
+Required.JsonInput.FloatFieldTooSmall
+Required.JsonInput.RepeatedFieldWrongElementTypeExpectingIntegersGotBool
+Required.JsonInput.TimestampJsonInputLowercaseT

conformance/failure_list_python_cpp.txt

@@ -7,65 +7,34 @@
 # TODO(haberman): insert links to corresponding bugs tracking the issue.
 # Should we use GitHub issues or the Google-internal bug tracker?
 
-DurationProtoInputTooLarge.JsonOutput
-DurationProtoInputTooSmall.JsonOutput
-FieldMaskNumbersDontRoundTrip.JsonOutput
-FieldMaskPathsDontRoundTrip.JsonOutput
-FieldMaskTooManyUnderscore.JsonOutput
-JsonInput.AnyWithFieldMask.ProtobufOutput
-JsonInput.BytesFieldInvalidBase64Characters
-JsonInput.DoubleFieldInfinityNotQuoted
-JsonInput.DoubleFieldNanNotQuoted
-JsonInput.DoubleFieldNegativeInfinityNotQuoted
-JsonInput.DoubleFieldTooSmall
-JsonInput.DurationJsonInputTooLarge
-JsonInput.DurationJsonInputTooSmall
-JsonInput.DurationMissingS
-JsonInput.EnumFieldNumericValueNonZero.JsonOutput
-JsonInput.EnumFieldNumericValueNonZero.ProtobufOutput
-JsonInput.EnumFieldNumericValueZero.JsonOutput
-JsonInput.EnumFieldNumericValueZero.ProtobufOutput
-JsonInput.EnumFieldUnknownValue.Validator
-JsonInput.FieldMask.ProtobufOutput
-JsonInput.FieldMaskInvalidCharacter
-JsonInput.FloatFieldInfinityNotQuoted
-JsonInput.FloatFieldNanNotQuoted
-JsonInput.FloatFieldNegativeInfinityNotQuoted
-JsonInput.FloatFieldTooLarge
-JsonInput.FloatFieldTooSmall
-JsonInput.Int32FieldExponentialFormat.JsonOutput
-JsonInput.Int32FieldExponentialFormat.ProtobufOutput
-JsonInput.Int32FieldFloatTrailingZero.JsonOutput
-JsonInput.Int32FieldFloatTrailingZero.ProtobufOutput
-JsonInput.Int32FieldMaxFloatValue.JsonOutput
-JsonInput.Int32FieldMaxFloatValue.ProtobufOutput
-JsonInput.Int32FieldMinFloatValue.JsonOutput
-JsonInput.Int32FieldMinFloatValue.ProtobufOutput
-JsonInput.OneofZeroMessage.JsonOutput
-JsonInput.OneofZeroMessage.ProtobufOutput
-JsonInput.OriginalProtoFieldName.JsonOutput
-JsonInput.OriginalProtoFieldName.ProtobufOutput
-JsonInput.RepeatedFieldWrongElementTypeExpectingIntegersGotBool
-JsonInput.TimestampJsonInputLowercaseT
-JsonInput.Uint32FieldMaxFloatValue.JsonOutput
-JsonInput.Uint32FieldMaxFloatValue.ProtobufOutput
-JsonInput.ValueAcceptNull.JsonOutput
-JsonInput.ValueAcceptNull.ProtobufOutput
-ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE
-ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE
-ProtobufInput.PrematureEofInPackedField.BOOL
-ProtobufInput.PrematureEofInPackedField.DOUBLE
-ProtobufInput.PrematureEofInPackedField.ENUM
-ProtobufInput.PrematureEofInPackedField.FIXED32
-ProtobufInput.PrematureEofInPackedField.FIXED64
-ProtobufInput.PrematureEofInPackedField.FLOAT
-ProtobufInput.PrematureEofInPackedField.INT32
-ProtobufInput.PrematureEofInPackedField.INT64
-ProtobufInput.PrematureEofInPackedField.SFIXED32
-ProtobufInput.PrematureEofInPackedField.SFIXED64
-ProtobufInput.PrematureEofInPackedField.SINT32
-ProtobufInput.PrematureEofInPackedField.SINT64
-ProtobufInput.PrematureEofInPackedField.UINT32
-ProtobufInput.PrematureEofInPackedField.UINT64
-TimestampProtoInputTooLarge.JsonOutput
-TimestampProtoInputTooSmall.JsonOutput
+Recommended.JsonInput.DoubleFieldInfinityNotQuoted
+Recommended.JsonInput.DoubleFieldNanNotQuoted
+Recommended.JsonInput.DoubleFieldNegativeInfinityNotQuoted
+Recommended.JsonInput.FloatFieldInfinityNotQuoted
+Recommended.JsonInput.FloatFieldNanNotQuoted
+Recommended.JsonInput.FloatFieldNegativeInfinityNotQuoted
+Recommended.JsonInput.OneofZeroMessage.JsonOutput
+Recommended.JsonInput.OneofZeroMessage.ProtobufOutput
+Required.JsonInput.BytesFieldInvalidBase64Characters
+Required.JsonInput.DoubleFieldTooSmall
+Required.JsonInput.EnumFieldUnknownValue.Validator
+Required.JsonInput.FloatFieldTooLarge
+Required.JsonInput.FloatFieldTooSmall
+Required.JsonInput.RepeatedFieldWrongElementTypeExpectingIntegersGotBool
+Required.JsonInput.TimestampJsonInputLowercaseT
+Required.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE
+Required.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE
+Required.ProtobufInput.PrematureEofInPackedField.BOOL
+Required.ProtobufInput.PrematureEofInPackedField.DOUBLE
+Required.ProtobufInput.PrematureEofInPackedField.ENUM
+Required.ProtobufInput.PrematureEofInPackedField.FIXED32
+Required.ProtobufInput.PrematureEofInPackedField.FIXED64
+Required.ProtobufInput.PrematureEofInPackedField.FLOAT
+Required.ProtobufInput.PrematureEofInPackedField.INT32
+Required.ProtobufInput.PrematureEofInPackedField.INT64
+Required.ProtobufInput.PrematureEofInPackedField.SFIXED32
+Required.ProtobufInput.PrematureEofInPackedField.SFIXED64
+Required.ProtobufInput.PrematureEofInPackedField.SINT32
+Required.ProtobufInput.PrematureEofInPackedField.SINT64
+Required.ProtobufInput.PrematureEofInPackedField.UINT32
+Required.ProtobufInput.PrematureEofInPackedField.UINT64

java/core/src/main/java/com/google/protobuf/ByteString.java

@@ -310,6 +310,18 @@ public abstract class ByteString implements Iterable<Byte>, Serializable {
     return copyFrom(bytes, 0, bytes.length);
   }
 
+  /**
+   * Wraps the given bytes into a {@code ByteString}. Intended for internal only usage.
+   */
+  static ByteString wrap(ByteBuffer buffer) {
+    if (buffer.hasArray()) {
+      final int offset = buffer.arrayOffset();
+      return ByteString.wrap(buffer.array(), offset + buffer.position(), buffer.remaining());
+    } else {
+      return new NioByteString(buffer);
+    }
+  }
+
   /**
    * Wraps the given bytes into a {@code ByteString}. Intended for internal only
    * usage to force a classload of ByteString before LiteralByteString.
@@ -679,6 +691,7 @@ public abstract class ByteString implements Iterable<Byte>, Serializable {
    */
   abstract void writeTo(ByteOutput byteOutput) throws IOException;
 
+
   /**
    * Constructs a read-only {@code java.nio.ByteBuffer} whose content
    * is equal to the contents of this byte string.
@@ -820,6 +833,7 @@ public abstract class ByteString implements Iterable<Byte>, Serializable {
       return true;
     }
 
+
     /**
      * Check equality of the substring of given length of this object starting at
      * zero with another {@code ByteString} substring starting at offset.

java/core/src/main/java/com/google/protobuf/CodedOutputStream.java

@@ -30,10 +30,12 @@
 
 package com.google.protobuf;
 
+import static com.google.protobuf.WireFormat.FIXED_32_SIZE;
+import static com.google.protobuf.WireFormat.FIXED_64_SIZE;
+import static com.google.protobuf.WireFormat.MAX_VARINT_SIZE;
 import static java.lang.Math.max;
 
 import com.google.protobuf.Utf8.UnpairedSurrogateException;
-
 import java.io.IOException;
 import java.io.OutputStream;
 import java.nio.BufferOverflowException;
@@ -59,10 +61,6 @@ public abstract class CodedOutputStream extends ByteOutput {
   private static final boolean HAS_UNSAFE_ARRAY_OPERATIONS = UnsafeUtil.hasUnsafeArrayOperations();
   private static final long ARRAY_BASE_OFFSET = UnsafeUtil.getArrayBaseOffset();
 
-  private static final int FIXED_32_SIZE = 4;
-  private static final int FIXED_64_SIZE = 8;
-  private static final int MAX_VARINT_SIZE = 10;
-
   /**
    * @deprecated Use {@link #computeFixed32SizeNoTag(int)} instead.
    */
@@ -134,14 +132,27 @@ public abstract class CodedOutputStream extends ByteOutput {
     return new ArrayEncoder(flatArray, offset, length);
   }
 
-  /**
-   * Create a new {@code CodedOutputStream} that writes to the given {@link ByteBuffer}.
-   */
-  public static CodedOutputStream newInstance(ByteBuffer byteBuffer) {
-    if (byteBuffer.hasArray()) {
-      return new NioHeapEncoder(byteBuffer);
+  /** Create a new {@code CodedOutputStream} that writes to the given {@link ByteBuffer}. */
+  public static CodedOutputStream newInstance(ByteBuffer buffer) {
+    if (buffer.hasArray()) {
+      return new HeapNioEncoder(buffer);
     }
-    return new NioEncoder(byteBuffer);
+    if (buffer.isDirect() && !buffer.isReadOnly()) {
+      return UnsafeDirectNioEncoder.isSupported()
+          ? newUnsafeInstance(buffer)
+          : newSafeInstance(buffer);
+    }
+    throw new IllegalArgumentException("ByteBuffer is read-only");
+  }
+
+  /** For testing purposes only. */
+  static CodedOutputStream newUnsafeInstance(ByteBuffer buffer) {
+    return new UnsafeDirectNioEncoder(buffer);
+  }
+
+  /** For testing purposes only. */
+  static CodedOutputStream newSafeInstance(ByteBuffer buffer) {
+    return new SafeDirectNioEncoder(buffer);
   }
 
   /**
@@ -979,6 +990,10 @@ public abstract class CodedOutputStream extends ByteOutput {
       super(MESSAGE);
     }
 
+    OutOfSpaceException(String explanationMessage) {
+      super(MESSAGE + ": " + explanationMessage);
+    }
+
     OutOfSpaceException(Throwable cause) {
       super(MESSAGE, cause);
     }
@@ -1486,11 +1501,11 @@ public abstract class CodedOutputStream extends ByteOutput {
    * A {@link CodedOutputStream} that writes directly to a heap {@link ByteBuffer}. Writes are
    * done directly to the underlying array. The buffer position is only updated after a flush.
    */
-  private static final class NioHeapEncoder extends ArrayEncoder {
+  private static final class HeapNioEncoder extends ArrayEncoder {
     private final ByteBuffer byteBuffer;
     private int initialPosition;
 
-    NioHeapEncoder(ByteBuffer byteBuffer) {
+    HeapNioEncoder(ByteBuffer byteBuffer) {
       super(byteBuffer.array(), byteBuffer.arrayOffset() + byteBuffer.position(),
           byteBuffer.remaining());
       this.byteBuffer = byteBuffer;
@@ -1505,14 +1520,15 @@ public abstract class CodedOutputStream extends ByteOutput {
   }
 
   /**
-   * A {@link CodedOutputStream} that writes directly to a {@link ByteBuffer}.
+   * A {@link CodedOutputStream} that writes directly to a direct {@link ByteBuffer}, using only
+   * safe operations..
    */
-  private static final class NioEncoder extends CodedOutputStream {
+  private static final class SafeDirectNioEncoder extends CodedOutputStream {
     private final ByteBuffer originalBuffer;
     private final ByteBuffer buffer;
     private final int initialPosition;
 
-    NioEncoder(ByteBuffer buffer) {
+    SafeDirectNioEncoder(ByteBuffer buffer) {
       this.originalBuffer = buffer;
       this.buffer = buffer.duplicate().order(ByteOrder.LITTLE_ENDIAN);
       initialPosition = buffer.position();
@@ -1814,6 +1830,356 @@ public abstract class CodedOutputStream extends ByteOutput {
     }
   }
 
+  /**
+   * A {@link CodedOutputStream} that writes directly to a direct {@link ByteBuffer} using {@code
+   * sun.misc.Unsafe}.
+   */
+  private static final class UnsafeDirectNioEncoder extends CodedOutputStream {
+    private final ByteBuffer originalBuffer;
+    private final ByteBuffer buffer;
+    private final long address;
+    private final long initialPosition;
+    private final long limit;
+    private final long oneVarintLimit;
+    private long position;
+
+    UnsafeDirectNioEncoder(ByteBuffer buffer) {
+      this.originalBuffer = buffer;
+      this.buffer = buffer.duplicate().order(ByteOrder.LITTLE_ENDIAN);
+      address = UnsafeUtil.addressOffset(buffer);
+      initialPosition = address + buffer.position();
+      limit = address + buffer.limit();
+      oneVarintLimit = limit - MAX_VARINT_SIZE;
+      position = initialPosition;
+    }
+
+    static boolean isSupported() {
+      return UnsafeUtil.hasUnsafeByteBufferOperations();
+    }
+
+    @Override
+    public void writeTag(int fieldNumber, int wireType) throws IOException {
+      writeUInt32NoTag(WireFormat.makeTag(fieldNumber, wireType));
+    }
+
+    @Override
+    public void writeInt32(int fieldNumber, int value) throws IOException {
+      writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT);
+      writeInt32NoTag(value);
+    }
+
+    @Override
+    public void writeUInt32(int fieldNumber, int value) throws IOException {
+      writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT);
+      writeUInt32NoTag(value);
+    }
+
+    @Override
+    public void writeFixed32(int fieldNumber, int value) throws IOException {
+      writeTag(fieldNumber, WireFormat.WIRETYPE_FIXED32);
+      writeFixed32NoTag(value);
+    }
+
+    @Override
+    public void writeUInt64(int fieldNumber, long value) throws IOException {
+      writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT);
+      writeUInt64NoTag(value);
+    }
+
+    @Override
+    public void writeFixed64(int fieldNumber, long value) throws IOException {
+      writeTag(fieldNumber, WireFormat.WIRETYPE_FIXED64);
+      writeFixed64NoTag(value);
+    }
+
+    @Override
+    public void writeBool(int fieldNumber, boolean value) throws IOException {
+      writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT);
+      write((byte) (value ? 1 : 0));
+    }
+
+    @Override
+    public void writeString(int fieldNumber, String value) throws IOException {
+      writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED);
+      writeStringNoTag(value);
+    }
+
+    @Override
+    public void writeBytes(int fieldNumber, ByteString value) throws IOException {
+      writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED);
+      writeBytesNoTag(value);
+    }
+
+    @Override
+    public void writeByteArray(int fieldNumber, byte[] value) throws IOException {
+      writeByteArray(fieldNumber, value, 0, value.length);
+    }
+
+    @Override
+    public void writeByteArray(int fieldNumber, byte[] value, int offset, int length)
+        throws IOException {
+      writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED);
+      writeByteArrayNoTag(value, offset, length);
+    }
+
+    @Override
+    public void writeByteBuffer(int fieldNumber, ByteBuffer value) throws IOException {
+      writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED);
+      writeUInt32NoTag(value.capacity());
+      writeRawBytes(value);
+    }
+
+    @Override
+    public void writeMessage(int fieldNumber, MessageLite value) throws IOException {
+      writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED);
+      writeMessageNoTag(value);
+    }
+
+    @Override
+    public void writeMessageSetExtension(int fieldNumber, MessageLite value) throws IOException {
+      writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_START_GROUP);
+      writeUInt32(WireFormat.MESSAGE_SET_TYPE_ID, fieldNumber);
+      writeMessage(WireFormat.MESSAGE_SET_MESSAGE, value);
+      writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_END_GROUP);
+    }
+
+    @Override
+    public void writeRawMessageSetExtension(int fieldNumber, ByteString value) throws IOException {
+      writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_START_GROUP);
+      writeUInt32(WireFormat.MESSAGE_SET_TYPE_ID, fieldNumber);
+      writeBytes(WireFormat.MESSAGE_SET_MESSAGE, value);
+      writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_END_GROUP);
+    }
+
+    @Override
+    public void writeMessageNoTag(MessageLite value) throws IOException {
+      writeUInt32NoTag(value.getSerializedSize());
+      value.writeTo(this);
+    }
+
+    @Override
+    public void write(byte value) throws IOException {
+      if (position >= limit) {
+        throw new OutOfSpaceException(
+            String.format("Pos: %d, limit: %d, len: %d", position, limit, 1));
+      }
+      UnsafeUtil.putByte(position++, value);
+    }
+
+    @Override
+    public void writeBytesNoTag(ByteString value) throws IOException {
+      writeUInt32NoTag(value.size());
+      value.writeTo(this);
+    }
+
+    @Override
+    public void writeByteArrayNoTag(byte[] value, int offset, int length) throws IOException {
+      writeUInt32NoTag(length);
+      write(value, offset, length);
+    }
+
+    @Override
+    public void writeRawBytes(ByteBuffer value) throws IOException {
+      if (value.hasArray()) {
+        write(value.array(), value.arrayOffset(), value.capacity());
+      } else {
+        ByteBuffer duplicated = value.duplicate();
+        duplicated.clear();
+        write(duplicated);
+      }
+    }
+
+    @Override
+    public void writeInt32NoTag(int value) throws IOException {
+      if (value >= 0) {
+        writeUInt32NoTag(value);
+      } else {
+        // Must sign-extend.
+        writeUInt64NoTag(value);
+      }
+    }
+
+    @Override
+    public void writeUInt32NoTag(int value) throws IOException {
+      if (position <= oneVarintLimit) {
+        // Optimization to avoid bounds checks on each iteration.
+        while (true) {
+          if ((value & ~0x7F) == 0) {
+            UnsafeUtil.putByte(position++, (byte) value);
+            return;
+          } else {
+            UnsafeUtil.putByte(position++, (byte) ((value & 0x7F) | 0x80));
+            value >>>= 7;
+          }
+        }
+      } else {
+        while (position < limit) {
+          if ((value & ~0x7F) == 0) {
+            UnsafeUtil.putByte(position++, (byte) value);
+            return;
+          } else {
+            UnsafeUtil.putByte(position++, (byte) ((value & 0x7F) | 0x80));
+            value >>>= 7;
+          }
+        }
+        throw new OutOfSpaceException(
+            String.format("Pos: %d, limit: %d, len: %d", position, limit, 1));
+      }
+    }
+
+    @Override
+    public void writeFixed32NoTag(int value) throws IOException {
+      buffer.putInt(bufferPos(position), value);
+      position += FIXED_32_SIZE;
+    }
+
+    @Override
+    public void writeUInt64NoTag(long value) throws IOException {
+      if (position <= oneVarintLimit) {
+        // Optimization to avoid bounds checks on each iteration.
+        while (true) {
+          if ((value & ~0x7FL) == 0) {
+            UnsafeUtil.putByte(position++, (byte) value);
+            return;
+          } else {
+            UnsafeUtil.putByte(position++, (byte) (((int) value & 0x7F) | 0x80));
+            value >>>= 7;
+          }
+        }
+      } else {
+        while (position < limit) {
+          if ((value & ~0x7FL) == 0) {
+            UnsafeUtil.putByte(position++, (byte) value);
+            return;
+          } else {
+            UnsafeUtil.putByte(position++, (byte) (((int) value & 0x7F) | 0x80));
+            value >>>= 7;
+          }
+        }
+        throw new OutOfSpaceException(
+            String.format("Pos: %d, limit: %d, len: %d", position, limit, 1));
+      }
+    }
+
+    @Override
+    public void writeFixed64NoTag(long value) throws IOException {
+      buffer.putLong(bufferPos(position), value);
+      position += FIXED_64_SIZE;
+    }
+
+    @Override
+    public void write(byte[] value, int offset, int length) throws IOException {
+      if (value == null
+          || offset < 0
+          || length < 0
+          || (value.length - length) < offset
+          || (limit - length) < position) {
+        if (value == null) {
+          throw new NullPointerException("value");
+        }
+        throw new OutOfSpaceException(
+            String.format("Pos: %d, limit: %d, len: %d", position, limit, length));
+      }
+
+      UnsafeUtil.copyMemory(
+          value, UnsafeUtil.getArrayBaseOffset() + offset, null, position, length);
+      position += length;
+    }
+
+    @Override
+    public void writeLazy(byte[] value, int offset, int length) throws IOException {
+      write(value, offset, length);
+    }
+
+    @Override
+    public void write(ByteBuffer value) throws IOException {
+      try {
+        int length = value.remaining();
+        repositionBuffer(position);
+        buffer.put(value);
+        position += length;
+      } catch (BufferOverflowException e) {
+        throw new OutOfSpaceException(e);
+      }
+    }
+
+    @Override
+    public void writeLazy(ByteBuffer value) throws IOException {
+      write(value);
+    }
+
+    @Override
+    public void writeStringNoTag(String value) throws IOException {
+      long prevPos = position;
+      try {
+        // UTF-8 byte length of the string is at least its UTF-16 code unit length (value.length()),
+        // and at most 3 times of it. We take advantage of this in both branches below.
+        int maxEncodedSize = value.length() * Utf8.MAX_BYTES_PER_CHAR;
+        int maxLengthVarIntSize = computeUInt32SizeNoTag(maxEncodedSize);
+        int minLengthVarIntSize = computeUInt32SizeNoTag(value.length());
+        if (minLengthVarIntSize == maxLengthVarIntSize) {
+          // Save the current position and increment past the length field. We'll come back
+          // and write the length field after the encoding is complete.
+          int stringStart = bufferPos(position) + minLengthVarIntSize;
+          buffer.position(stringStart);
+
+          // Encode the string.
+          Utf8.encodeUtf8(value, buffer);
+
+          // Write the length and advance the position.
+          int length = buffer.position() - stringStart;
+          writeUInt32NoTag(length);
+          position += length;
+        } else {
+          // Calculate and write the encoded length.
+          int length = Utf8.encodedLength(value);
+          writeUInt32NoTag(length);
+
+          // Write the string and advance the position.
+          repositionBuffer(position);
+          Utf8.encodeUtf8(value, buffer);
+          position += length;
+        }
+      } catch (UnpairedSurrogateException e) {
+        // Roll back the change and convert to an IOException.
+        position = prevPos;
+        repositionBuffer(position);
+
+        // TODO(nathanmittler): We should throw an IOException here instead.
+        inefficientWriteStringNoTag(value, e);
+      } catch (IllegalArgumentException e) {
+        // Thrown by buffer.position() if out of range.
+        throw new OutOfSpaceException(e);
+      } catch (IndexOutOfBoundsException e) {
+        throw new OutOfSpaceException(e);
+      }
+    }
+
+    @Override
+    public void flush() {
+      // Update the position of the original buffer.
+      originalBuffer.position(bufferPos(position));
+    }
+
+    @Override
+    public int spaceLeft() {
+      return (int) (limit - position);
+    }
+
+    @Override
+    public int getTotalBytesWritten() {
+      return (int) (position - initialPosition);
+    }
+
+    private void repositionBuffer(long pos) {
+      buffer.position(bufferPos(pos));
+    }
+
+    private int bufferPos(long pos) {
+      return (int) (pos - address);
+    }
+  }
+
   /**
    * Abstract base class for buffered encoders.
    */

java/core/src/main/java/com/google/protobuf/Descriptors.java

@@ -1212,33 +1212,20 @@ public final class Descriptors {
       private final Object defaultDefault;
     }
 
-    // TODO(xiaofeng): Implement it consistently across different languages. See b/24751348.
-    private static String fieldNameToLowerCamelCase(String name) {
+    // This method should match exactly with the ToJsonName() function in C++
+    // descriptor.cc.
+    private static String fieldNameToJsonName(String name) {
       StringBuilder result = new StringBuilder(name.length());
       boolean isNextUpperCase = false;
       for (int i = 0; i < name.length(); i++) {
         Character ch = name.charAt(i);
-        if (Character.isLowerCase(ch)) {
-          if (isNextUpperCase) {
-            result.append(Character.toUpperCase(ch));
-          } else {
-            result.append(ch);
-          }
-          isNextUpperCase = false;
-        } else if (Character.isUpperCase(ch)) {
-          if (i == 0) {
-            // Force first letter to lower-case.
-            result.append(Character.toLowerCase(ch));
-          } else {
-            // Capital letters after the first are left as-is.
-            result.append(ch);
-          }
-          isNextUpperCase = false;
-        } else if (Character.isDigit(ch)) {
-          result.append(ch);
+        if (ch == '_') {
+          isNextUpperCase = true;
+        } else if (isNextUpperCase) {
+          result.append(Character.toUpperCase(ch));
           isNextUpperCase = false;
         } else {
-          isNextUpperCase = true;
+          result.append(ch);
         }
       }
       return result.toString();
@@ -1257,7 +1244,7 @@ public final class Descriptors {
       if (proto.hasJsonName()) {
         jsonName = proto.getJsonName();
       } else {
-        jsonName = fieldNameToLowerCamelCase(proto.getName());
+        jsonName = fieldNameToJsonName(proto.getName());
       }
 
       if (proto.hasType()) {

java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java

@@ -36,7 +36,6 @@ import com.google.protobuf.Descriptors.EnumValueDescriptor;
 import com.google.protobuf.Descriptors.FieldDescriptor;
 import com.google.protobuf.Descriptors.FileDescriptor;
 import com.google.protobuf.Descriptors.OneofDescriptor;
-import com.google.protobuf.GeneratedMessage.GeneratedExtension;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -45,6 +44,7 @@ import java.io.Serializable;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
@@ -1609,6 +1609,43 @@ public abstract class GeneratedMessageV3 extends AbstractMessage
     FieldDescriptor getDescriptor();
   }
 
+  /** For use by generated code only. */
+  public static <ContainingType extends Message, Type>
+      GeneratedExtension<ContainingType, Type>
+      newMessageScopedGeneratedExtension(final Message scope,
+                                         final int descriptorIndex,
+                                         final Class singularType,
+                                         final Message defaultInstance) {
+    // For extensions scoped within a Message, we use the Message to resolve
+    // the outer class's descriptor, from which the extension descriptor is
+    // obtained.
+    return new GeneratedExtension<ContainingType, Type>(
+        new CachedDescriptorRetriever() {
+          @Override
+          public FieldDescriptor loadDescriptor() {
+            return scope.getDescriptorForType().getExtensions().get(descriptorIndex);
+          }
+        },
+        singularType,
+        defaultInstance,
+        Extension.ExtensionType.IMMUTABLE);
+  }
+
+  /** For use by generated code only. */
+  public static <ContainingType extends Message, Type>
+     GeneratedExtension<ContainingType, Type>
+     newFileScopedGeneratedExtension(final Class singularType,
+                                     final Message defaultInstance) {
+    // For extensions scoped within a file, we rely on the outer class's
+    // static initializer to call internalInit() on the extension when the
+    // descriptor is available.
+    return new GeneratedExtension<ContainingType, Type>(
+        null,  // ExtensionDescriptorRetriever is initialized in internalInit();
+        singularType,
+        defaultInstance,
+        Extension.ExtensionType.IMMUTABLE);
+  }
+
   private abstract static class CachedDescriptorRetriever
       implements ExtensionDescriptorRetriever {
     private volatile FieldDescriptor descriptor;
@@ -1627,6 +1664,301 @@ public abstract class GeneratedMessageV3 extends AbstractMessage
     }
   }
 
+  /**
+   * Used in proto1 generated code only.
+   *
+   * After enabling bridge, we can define proto2 extensions (the extended type
+   * is a proto2 mutable message) in a proto1 .proto file. For these extensions
+   * we should generate proto2 GeneratedExtensions.
+   */
+  public static <ContainingType extends Message, Type>
+      GeneratedExtension<ContainingType, Type>
+      newMessageScopedGeneratedExtension(
+          final Message scope, final String name,
+          final Class singularType, final Message defaultInstance) {
+    // For extensions scoped within a Message, we use the Message to resolve
+    // the outer class's descriptor, from which the extension descriptor is
+    // obtained.
+    return new GeneratedExtension<ContainingType, Type>(
+        new CachedDescriptorRetriever() {
+          @Override
+          protected FieldDescriptor loadDescriptor() {
+            return scope.getDescriptorForType().findFieldByName(name);
+          }
+        },
+        singularType,
+        defaultInstance,
+        Extension.ExtensionType.MUTABLE);
+  }
+
+  /**
+   * Used in proto1 generated code only.
+   *
+   * After enabling bridge, we can define proto2 extensions (the extended type
+   * is a proto2 mutable message) in a proto1 .proto file. For these extensions
+   * we should generate proto2 GeneratedExtensions.
+   */
+  public static <ContainingType extends Message, Type>
+     GeneratedExtension<ContainingType, Type>
+     newFileScopedGeneratedExtension(
+         final Class singularType, final Message defaultInstance,
+         final String descriptorOuterClass, final String extensionName) {
+    // For extensions scoped within a file, we load the descriptor outer
+    // class and rely on it to get the FileDescriptor which then can be
+    // used to obtain the extension's FieldDescriptor.
+    return new GeneratedExtension<ContainingType, Type>(
+        new CachedDescriptorRetriever() {
+          @Override
+          protected FieldDescriptor loadDescriptor() {
+            try {
+              Class clazz = singularType.getClassLoader().loadClass(descriptorOuterClass);
+              FileDescriptor file = (FileDescriptor) clazz.getField("descriptor").get(null);
+              return file.findExtensionByName(extensionName);
+            } catch (Exception e) {
+              throw new RuntimeException(
+                  "Cannot load descriptors: "
+                      + descriptorOuterClass
+                      + " is not a valid descriptor class name",
+                  e);
+            }
+          }
+        },
+        singularType,
+        defaultInstance,
+        Extension.ExtensionType.MUTABLE);
+  }
+
+  /**
+   * Type used to represent generated extensions.  The protocol compiler
+   * generates a static singleton instance of this class for each extension.
+   *
+   * <p>For example, imagine you have the {@code .proto} file:
+   *
+   * <pre>
+   * option java_class = "MyProto";
+   *
+   * message Foo {
+   *   extensions 1000 to max;
+   * }
+   *
+   * extend Foo {
+   *   optional int32 bar;
+   * }
+   * </pre>
+   *
+   * <p>Then, {@code MyProto.Foo.bar} has type
+   * {@code GeneratedExtension<MyProto.Foo, Integer>}.
+   *
+   * <p>In general, users should ignore the details of this type, and simply use
+   * these static singletons as parameters to the extension accessors defined
+   * in {@link ExtendableMessage} and {@link ExtendableBuilder}.
+   */
+  public static class GeneratedExtension<
+      ContainingType extends Message, Type> extends
+      Extension<ContainingType, Type> {
+    // TODO(kenton):  Find ways to avoid using Java reflection within this
+    //   class.  Also try to avoid suppressing unchecked warnings.
+
+    // We can't always initialize the descriptor of a GeneratedExtension when
+    // we first construct it due to initialization order difficulties (namely,
+    // the descriptor may not have been constructed yet, since it is often
+    // constructed by the initializer of a separate module).
+    //
+    // In the case of nested extensions, we initialize the
+    // ExtensionDescriptorRetriever with an instance that uses the scoping
+    // Message's default instance to retrieve the extension's descriptor.
+    //
+    // In the case of non-nested extensions, we initialize the
+    // ExtensionDescriptorRetriever to null and rely on the outer class's static
+    // initializer to call internalInit() after the descriptor has been parsed.
+    GeneratedExtension(ExtensionDescriptorRetriever descriptorRetriever,
+        Class singularType,
+        Message messageDefaultInstance,
+        ExtensionType extensionType) {
+      if (Message.class.isAssignableFrom(singularType) &&
+          !singularType.isInstance(messageDefaultInstance)) {
+        throw new IllegalArgumentException(
+            "Bad messageDefaultInstance for " + singularType.getName());
+      }
+      this.descriptorRetriever = descriptorRetriever;
+      this.singularType = singularType;
+      this.messageDefaultInstance = messageDefaultInstance;
+
+      if (ProtocolMessageEnum.class.isAssignableFrom(singularType)) {
+        this.enumValueOf = getMethodOrDie(singularType, "valueOf",
+                                          EnumValueDescriptor.class);
+        this.enumGetValueDescriptor =
+            getMethodOrDie(singularType, "getValueDescriptor");
+      } else {
+        this.enumValueOf = null;
+        this.enumGetValueDescriptor = null;
+      }
+      this.extensionType = extensionType;
+    }
+
+    /** For use by generated code only. */
+    public void internalInit(final FieldDescriptor descriptor) {
+      if (descriptorRetriever != null) {
+        throw new IllegalStateException("Already initialized.");
+      }
+      descriptorRetriever =
+          new ExtensionDescriptorRetriever() {
+            @Override
+            public FieldDescriptor getDescriptor() {
+              return descriptor;
+            }
+          };
+    }
+
+    private ExtensionDescriptorRetriever descriptorRetriever;
+    private final Class singularType;
+    private final Message messageDefaultInstance;
+    private final Method enumValueOf;
+    private final Method enumGetValueDescriptor;
+    private final ExtensionType extensionType;
+
+    @Override
+    public FieldDescriptor getDescriptor() {
+      if (descriptorRetriever == null) {
+        throw new IllegalStateException(
+            "getDescriptor() called before internalInit()");
+      }
+      return descriptorRetriever.getDescriptor();
+    }
+
+    /**
+     * If the extension is an embedded message or group, returns the default
+     * instance of the message.
+     */
+    @Override
+    public Message getMessageDefaultInstance() {
+      return messageDefaultInstance;
+    }
+
+    @Override
+    protected ExtensionType getExtensionType() {
+      return extensionType;
+    }
+
+    /**
+     * Convert from the type used by the reflection accessors to the type used
+     * by native accessors.  E.g., for enums, the reflection accessors use
+     * EnumValueDescriptors but the native accessors use the generated enum
+     * type.
+     */
+    @Override
+    @SuppressWarnings("unchecked")
+    protected Object fromReflectionType(final Object value) {
+      FieldDescriptor descriptor = getDescriptor();
+      if (descriptor.isRepeated()) {
+        if (descriptor.getJavaType() == FieldDescriptor.JavaType.MESSAGE ||
+            descriptor.getJavaType() == FieldDescriptor.JavaType.ENUM) {
+          // Must convert the whole list.
+          final List result = new ArrayList();
+          for (final Object element : (List) value) {
+            result.add(singularFromReflectionType(element));
+          }
+          return result;
+        } else {
+          return value;
+        }
+      } else {
+        return singularFromReflectionType(value);
+      }
+    }
+
+    /**
+     * Like {@link #fromReflectionType(Object)}, but if the type is a repeated
+     * type, this converts a single element.
+     */
+    @Override
+    protected Object singularFromReflectionType(final Object value) {
+      FieldDescriptor descriptor = getDescriptor();
+      switch (descriptor.getJavaType()) {
+        case MESSAGE:
+          if (singularType.isInstance(value)) {
+            return value;
+          } else {
+            return messageDefaultInstance.newBuilderForType()
+                .mergeFrom((Message) value).build();
+          }
+        case ENUM:
+          return invokeOrDie(enumValueOf, null, (EnumValueDescriptor) value);
+        default:
+          return value;
+      }
+    }
+
+    /**
+     * Convert from the type used by the native accessors to the type used
+     * by reflection accessors.  E.g., for enums, the reflection accessors use
+     * EnumValueDescriptors but the native accessors use the generated enum
+     * type.
+     */
+    @Override
+    @SuppressWarnings("unchecked")
+    protected Object toReflectionType(final Object value) {
+      FieldDescriptor descriptor = getDescriptor();
+      if (descriptor.isRepeated()) {
+        if (descriptor.getJavaType() == FieldDescriptor.JavaType.ENUM) {
+          // Must convert the whole list.
+          final List result = new ArrayList();
+          for (final Object element : (List) value) {
+            result.add(singularToReflectionType(element));
+          }
+          return result;
+        } else {
+          return value;
+        }
+      } else {
+        return singularToReflectionType(value);
+      }
+    }
+
+    /**
+     * Like {@link #toReflectionType(Object)}, but if the type is a repeated
+     * type, this converts a single element.
+     */
+    @Override
+    protected Object singularToReflectionType(final Object value) {
+      FieldDescriptor descriptor = getDescriptor();
+      switch (descriptor.getJavaType()) {
+        case ENUM:
+          return invokeOrDie(enumGetValueDescriptor, value);
+        default:
+          return value;
+      }
+    }
+
+    @Override
+    public int getNumber() {
+      return getDescriptor().getNumber();
+    }
+
+    @Override
+    public WireFormat.FieldType getLiteType() {
+      return getDescriptor().getLiteType();
+    }
+
+    @Override
+    public boolean isRepeated() {
+      return getDescriptor().isRepeated();
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    public Type getDefaultValue() {
+      if (isRepeated()) {
+        return (Type) Collections.emptyList();
+      }
+      if (getDescriptor().getJavaType() == FieldDescriptor.JavaType.MESSAGE) {
+        return (Type) messageDefaultInstance;
+      }
+      return (Type) singularFromReflectionType(
+          getDescriptor().getDefaultValue());
+    }
+  }
+
   // =================================================================
 
   /** Calls Class.getMethod and throws a RuntimeException if it fails. */
@@ -2223,6 +2555,20 @@ public abstract class GeneratedMessageV3 extends AbstractMessage
             field.getNumber());
       }
 
+      private Message coerceType(Message value) {
+        if (value == null) {
+          return null;
+        }
+        if (mapEntryMessageDefaultInstance.getClass().isInstance(value)) {
+          return value;
+        }
+        // The value is not the exact right message type.  However, if it
+        // is an alternative implementation of the same type -- e.g. a
+        // DynamicMessage -- we should accept it.  In this case we can make
+        // a copy of the message.
+        return mapEntryMessageDefaultInstance.toBuilder().mergeFrom(value).build();
+      }
+
       @Override
       public Object get(GeneratedMessageV3 message) {
         List result = new ArrayList();
@@ -2278,15 +2624,15 @@ public abstract class GeneratedMessageV3 extends AbstractMessage
       public Object getRepeatedRaw(Builder builder, int index) {
         return getRepeated(builder, index);
       }
-
+      
       @Override
       public void setRepeated(Builder builder, int index, Object value) {
-        getMutableMapField(builder).getMutableList().set(index, (Message) value);
+        getMutableMapField(builder).getMutableList().set(index, coerceType((Message) value));
       }
 
       @Override
       public void addRepeated(Builder builder, Object value) {
-        getMutableMapField(builder).getMutableList().add((Message) value);
+        getMutableMapField(builder).getMutableList().add(coerceType((Message) value));
       }
 
       @Override
@@ -2713,4 +3059,131 @@ public abstract class GeneratedMessageV3 extends AbstractMessage
       output.writeBytesNoTag((ByteString) value);
     }
   }
+
+  protected static <V> void serializeIntegerMapTo(
+      CodedOutputStream out,
+      MapField<Integer, V> field,
+      MapEntry<Integer, V> defaultEntry,
+      int fieldNumber) throws IOException {
+    Map<Integer, V> m = field.getMap();
+    if (!out.isSerializationDeterministic()) {
+      serializeMapTo(out, m, defaultEntry, fieldNumber);
+      return;
+    }
+    // Sorting the unboxed keys and then look up the values during serialziation is 2x faster
+    // than sorting map entries with a custom comparator directly.
+    int[] keys = new int[m.size()];
+    int index = 0;
+    for (int k : m.keySet()) {
+      keys[index++] = k;
+    }
+    Arrays.sort(keys);
+    for (int key : keys) {
+      out.writeMessage(fieldNumber,
+          defaultEntry.newBuilderForType()
+              .setKey(key)
+              .setValue(m.get(key))
+              .build());
+    }
+  }
+
+  protected static <V> void serializeLongMapTo(
+      CodedOutputStream out,
+      MapField<Long, V> field,
+      MapEntry<Long, V> defaultEntry,
+      int fieldNumber)
+      throws IOException {
+    Map<Long, V> m = field.getMap();
+    if (!out.isSerializationDeterministic()) {
+      serializeMapTo(out, m, defaultEntry, fieldNumber);
+      return;
+    }
+
+    long[] keys = new long[m.size()];
+    int index = 0;
+    for (long k : m.keySet()) {
+      keys[index++] = k;
+    }
+    Arrays.sort(keys);
+    for (long key : keys) {
+      out.writeMessage(fieldNumber,
+          defaultEntry.newBuilderForType()
+              .setKey(key)
+              .setValue(m.get(key))
+              .build());
+    }
+  }
+
+  protected static <V> void serializeStringMapTo(
+      CodedOutputStream out,
+      MapField<String, V> field,
+      MapEntry<String, V> defaultEntry,
+      int fieldNumber)
+      throws IOException {
+    Map<String, V> m = field.getMap();
+    if (!out.isSerializationDeterministic()) {
+      serializeMapTo(out, m, defaultEntry, fieldNumber);
+      return;
+    }
+
+    // Sorting the String keys and then look up the values during serialziation is 25% faster than
+    // sorting map entries with a custom comparator directly.
+    String[] keys = new String[m.size()];
+    keys = m.keySet().toArray(keys);
+    Arrays.sort(keys);
+    for (String key : keys) {
+      out.writeMessage(fieldNumber,
+          defaultEntry.newBuilderForType()
+              .setKey(key)
+              .setValue(m.get(key))
+              .build());
+    }
+  }
+
+  protected static <V> void serializeBooleanMapTo(
+      CodedOutputStream out,
+      MapField<Boolean, V> field,
+      MapEntry<Boolean, V> defaultEntry,
+      int fieldNumber)
+      throws IOException {
+    Map<Boolean, V> m = field.getMap();
+    if (!out.isSerializationDeterministic()) {
+      serializeMapTo(out, m, defaultEntry, fieldNumber);
+      return;
+    }
+    maybeSerializeBooleanEntryTo(out, m, defaultEntry, fieldNumber, false);
+    maybeSerializeBooleanEntryTo(out, m, defaultEntry, fieldNumber, true);
+  }
+
+  private static <V> void maybeSerializeBooleanEntryTo(
+      CodedOutputStream out,
+      Map<Boolean, V> m,
+      MapEntry<Boolean, V> defaultEntry,
+      int fieldNumber,
+      boolean key)
+      throws IOException {
+    if (m.containsKey(key)) {
+      out.writeMessage(fieldNumber,
+          defaultEntry.newBuilderForType()
+              .setKey(key)
+              .setValue(m.get(key))
+              .build());
+    }
+  }
+
+  /** Serialize the map using the iteration order. */
+  private static <K, V> void serializeMapTo(
+      CodedOutputStream out,
+      Map<K, V> m,
+      MapEntry<K, V> defaultEntry,
+      int fieldNumber)
+      throws IOException {
+    for (Map.Entry<K, V> entry : m.entrySet()) {
+      out.writeMessage(fieldNumber,
+          defaultEntry.newBuilderForType()
+              .setKey(entry.getKey())
+              .setValue(entry.getValue())
+              .build());
+    }
+  }
 }

java/core/src/main/java/com/google/protobuf/Internal.java

@@ -59,6 +59,16 @@ public final class Internal {
   static final Charset UTF_8 = Charset.forName("UTF-8");
   static final Charset ISO_8859_1 = Charset.forName("ISO-8859-1");
 
+  /**
+   * Throws an appropriate {@link NullPointerException} if the given objects is {@code null}.
+   */
+  static <T> T checkNotNull(T obj, String message) {
+    if (obj == null) {
+      throw new NullPointerException(message);
+    }
+    return obj;
+  }
+
   /**
    * Helper called by generated code to construct default values for string
    * fields.

java/core/src/main/java/com/google/protobuf/InvalidProtocolBufferException.java

@@ -107,11 +107,23 @@ public class InvalidProtocolBufferException extends IOException {
       "Protocol message end-group tag did not match expected tag.");
   }
 
-  static InvalidProtocolBufferException invalidWireType() {
-    return new InvalidProtocolBufferException(
+  static InvalidWireTypeException invalidWireType() {
+    return new InvalidWireTypeException(
       "Protocol message tag had invalid wire type.");
   }
 
+  /**
+   * Exception indicating that and unexpected wire type was encountered for a field.
+   */
+  @ExperimentalApi
+  public static class InvalidWireTypeException extends InvalidProtocolBufferException {
+    private static final long serialVersionUID = 3283890091615336259L;
+
+    public InvalidWireTypeException(String description) {
+      super(description);
+    }
+  }
+
   static InvalidProtocolBufferException recursionLimitExceeded() {
     return new InvalidProtocolBufferException(
       "Protocol message had too many levels of nesting.  May be malicious.  " +

java/core/src/main/java/com/google/protobuf/MapEntry.java

@@ -334,6 +334,15 @@ public final class MapEntry<K, V> extends AbstractMessage {
       } else {
         if (field.getType() == FieldDescriptor.Type.ENUM) {
           value = ((EnumValueDescriptor) value).getNumber();
+        } else if (field.getType() == FieldDescriptor.Type.MESSAGE) {
+          if (value != null && !metadata.defaultValue.getClass().isInstance(value)) {
+            // The value is not the exact right message type.  However, if it
+            // is an alternative implementation of the same type -- e.g. a
+            // DynamicMessage -- we should accept it.  In this case we can make
+            // a copy of the message.
+            value =
+                ((Message) metadata.defaultValue).toBuilder().mergeFrom((Message) value).build();
+          }
         }
         setValue((V) value);
       }

java/core/src/main/java/com/google/protobuf/NioByteString.java

@@ -30,6 +30,8 @@
 
 package com.google.protobuf;
 
+import static com.google.protobuf.Internal.checkNotNull;
+
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InvalidObjectException;
@@ -49,10 +51,9 @@ final class NioByteString extends ByteString.LeafByteString {
   private final ByteBuffer buffer;
 
   NioByteString(ByteBuffer buffer) {
-    if (buffer == null) {
-      throw new NullPointerException("buffer");
-    }
+    checkNotNull(buffer, "buffer");
 
+    // Use native byte order for fast fixed32/64 operations.
     this.buffer = buffer.slice().order(ByteOrder.nativeOrder());
   }
 
@@ -266,7 +267,7 @@ final class NioByteString extends ByteString.LeafByteString {
 
   @Override
   public CodedInputStream newCodedInput() {
-    return CodedInputStream.newInstance(buffer);
+    return CodedInputStream.newInstance(buffer, true);
   }
 
   /**

java/core/src/main/java/com/google/protobuf/RopeByteString.java

@@ -406,6 +406,7 @@ final class RopeByteString extends ByteString {
     right.writeTo(output);
   }
 
+
   @Override
   protected String toStringInternal(Charset charset) {
     return new String(toByteArray(), charset);

java/core/src/main/java/com/google/protobuf/TextFormat.java

@@ -1576,14 +1576,22 @@ public final class TextFormat {
       // Support specifying repeated field values as a comma-separated list.
       // Ex."foo: [1, 2, 3]"
       if (field.isRepeated() && tokenizer.tryConsume("[")) {
-        while (true) {
-          consumeFieldValue(tokenizer, extensionRegistry, target, field, extension,
-              parseTreeBuilder, unknownFields);
-          if (tokenizer.tryConsume("]")) {
-            // End of list.
-            break;
+        if (!tokenizer.tryConsume("]")) {  // Allow "foo: []" to be treated as empty.
+          while (true) {
+            consumeFieldValue(
+                tokenizer,
+                extensionRegistry,
+                target,
+                field,
+                extension,
+                parseTreeBuilder,
+                unknownFields);
+            if (tokenizer.tryConsume("]")) {
+              // End of list.
+              break;
+            }
+            tokenizer.consume(",");
           }
-          tokenizer.consume(",");
         }
       } else {
         consumeFieldValue(tokenizer, extensionRegistry, target, field,

java/core/src/main/java/com/google/protobuf/TextFormatParseInfoTree.java

@@ -45,7 +45,8 @@ import java.util.Map.Entry;
  *
  * <p>The locations of primary fields values are retrieved by {@code getLocation} or
  * {@code getLocations}.  The locations of sub message values are within nested
- * {@code TextFormatParseInfoTree}s and are retrieve by {@code getNestedTree} or {@code getNestedTrees}.
+ * {@code TextFormatParseInfoTree}s and are retrieve by {@code getNestedTree} or
+ * {@code getNestedTrees}.
  *
  * <p>The {@code TextFormatParseInfoTree} is created by a Builder.
  */

java/core/src/main/java/com/google/protobuf/UnsafeByteOperations.java

@@ -64,6 +64,29 @@ import java.nio.ByteBuffer;
 public final class UnsafeByteOperations {
   private UnsafeByteOperations() {}
 
+  /**
+   * An unsafe operation that returns a {@link ByteString} that is backed by the provided buffer.
+   *
+   * @param buffer the buffer to be wrapped
+   * @return a {@link ByteString} backed by the provided buffer
+   */
+  public static ByteString unsafeWrap(byte[] buffer) {
+    return ByteString.wrap(buffer);
+  }
+
+  /**
+   * An unsafe operation that returns a {@link ByteString} that is backed by a subregion of the
+   * provided buffer.
+   *
+   * @param buffer the buffer to be wrapped
+   * @param offset the offset of the wrapped region
+   * @param length the number of bytes of the wrapped region
+   * @return a {@link ByteString} backed by the provided buffer
+   */
+  public static ByteString unsafeWrap(byte[] buffer, int offset, int length) {
+    return ByteString.wrap(buffer, offset, length);
+  }
+
   /**
    * An unsafe operation that returns a {@link ByteString} that is backed by the provided buffer.
    *
@@ -71,12 +94,7 @@ public final class UnsafeByteOperations {
    * @return a {@link ByteString} backed by the provided buffer
    */
   public static ByteString unsafeWrap(ByteBuffer buffer) {
-    if (buffer.hasArray()) {
-      final int offset = buffer.arrayOffset();
-      return ByteString.wrap(buffer.array(), offset + buffer.position(), buffer.remaining());
-    } else {
-      return new NioByteString(buffer);
-    }
+    return ByteString.wrap(buffer);
   }
 
   /**
@@ -98,4 +116,5 @@ public final class UnsafeByteOperations {
   public static void unsafeWriteTo(ByteString bytes, ByteOutput output) throws IOException {
     bytes.writeTo(output);
   }
+
 }

java/core/src/main/java/com/google/protobuf/UnsafeUtil.java

@@ -30,17 +30,14 @@
 
 package com.google.protobuf;
 
-import sun.misc.Unsafe;
-
 import java.lang.reflect.Field;
 import java.nio.Buffer;
 import java.nio.ByteBuffer;
 import java.security.AccessController;
 import java.security.PrivilegedExceptionAction;
+import sun.misc.Unsafe;
 
-/**
- * Utility class for working with unsafe operations.
- */
+/** Utility class for working with unsafe operations. */
 // TODO(nathanmittler): Add support for Android Memory/MemoryBlock
 final class UnsafeUtil {
   private static final sun.misc.Unsafe UNSAFE = getUnsafe();
@@ -50,8 +47,7 @@ final class UnsafeUtil {
   private static final long ARRAY_BASE_OFFSET = byteArrayBaseOffset();
   private static final long BUFFER_ADDRESS_OFFSET = fieldOffset(field(Buffer.class, "address"));
 
-  private UnsafeUtil() {
-  }
+  private UnsafeUtil() {}
 
   static boolean hasUnsafeArrayOperations() {
     return HAS_UNSAFE_ARRAY_OPERATIONS;
@@ -61,27 +57,83 @@ final class UnsafeUtil {
     return HAS_UNSAFE_BYTEBUFFER_OPERATIONS;
   }
 
+  static Object allocateInstance(Class<?> clazz) {
+    try {
+      return UNSAFE.allocateInstance(clazz);
+    } catch (InstantiationException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  static long objectFieldOffset(Field field) {
+    return UNSAFE.objectFieldOffset(field);
+  }
+
   static long getArrayBaseOffset() {
     return ARRAY_BASE_OFFSET;
   }
 
-  static byte getByte(byte[] target, long offset) {
+  static byte getByte(Object target, long offset) {
     return UNSAFE.getByte(target, offset);
   }
 
-  static void putByte(byte[] target, long offset, byte value) {
+  static void putByte(Object target, long offset, byte value) {
     UNSAFE.putByte(target, offset, value);
   }
 
-  static void copyMemory(
-      byte[] src, long srcOffset, byte[] target, long targetOffset, long length) {
-    UNSAFE.copyMemory(src, srcOffset, target, targetOffset, length);
+  static int getInt(Object target, long offset) {
+    return UNSAFE.getInt(target, offset);
   }
 
-  static long getLong(byte[] target, long offset) {
+  static void putInt(Object target, long offset, int value) {
+    UNSAFE.putInt(target, offset, value);
+  }
+
+  static long getLong(Object target, long offset) {
     return UNSAFE.getLong(target, offset);
   }
 
+  static void putLong(Object target, long offset, long value) {
+    UNSAFE.putLong(target, offset, value);
+  }
+
+  static boolean getBoolean(Object target, long offset) {
+    return UNSAFE.getBoolean(target, offset);
+  }
+
+  static void putBoolean(Object target, long offset, boolean value) {
+    UNSAFE.putBoolean(target, offset, value);
+  }
+
+  static float getFloat(Object target, long offset) {
+    return UNSAFE.getFloat(target, offset);
+  }
+
+  static void putFloat(Object target, long offset, float value) {
+    UNSAFE.putFloat(target, offset, value);
+  }
+
+  static double getDouble(Object target, long offset) {
+    return UNSAFE.getDouble(target, offset);
+  }
+
+  static void putDouble(Object target, long offset, double value) {
+    UNSAFE.putDouble(target, offset, value);
+  }
+
+  static Object getObject(Object target, long offset) {
+    return UNSAFE.getObject(target, offset);
+  }
+
+  static void putObject(Object target, long offset, Object value) {
+    UNSAFE.putObject(target, offset, value);
+  }
+
+  static void copyMemory(
+      Object src, long srcOffset, Object target, long targetOffset, long length) {
+    UNSAFE.copyMemory(src, srcOffset, target, targetOffset, length);
+  }
+
   static byte getByte(long address) {
     return UNSAFE.getByte(address);
   }
@@ -90,14 +142,30 @@ final class UnsafeUtil {
     UNSAFE.putByte(address, value);
   }
 
+  static int getInt(long address) {
+    return UNSAFE.getInt(address);
+  }
+
+  static void putInt(long address, int value) {
+    UNSAFE.putInt(address, value);
+  }
+
   static long getLong(long address) {
     return UNSAFE.getLong(address);
   }
 
+  static void putLong(long address, long value) {
+    UNSAFE.putLong(address, value);
+  }
+
   static void copyMemory(long srcAddress, long targetAddress, long length) {
     UNSAFE.copyMemory(srcAddress, targetAddress, length);
   }
 
+  static void setMemory(long address, long numBytes, byte value) {
+    UNSAFE.setMemory(address, numBytes, value);
+  }
+
   /**
    * Gets the offset of the {@code address} field of the given direct {@link ByteBuffer}.
    */
@@ -136,18 +204,29 @@ final class UnsafeUtil {
     return unsafe;
   }
 
-  /**
-   * Indicates whether or not unsafe array operations are supported on this platform.
-   */
+  /** Indicates whether or not unsafe array operations are supported on this platform. */
   private static boolean supportsUnsafeArrayOperations() {
     boolean supported = false;
     if (UNSAFE != null) {
       try {
         Class<?> clazz = UNSAFE.getClass();
+        clazz.getMethod("objectFieldOffset", Field.class);
+        clazz.getMethod("allocateInstance", Class.class);
         clazz.getMethod("arrayBaseOffset", Class.class);
         clazz.getMethod("getByte", Object.class, long.class);
         clazz.getMethod("putByte", Object.class, long.class, byte.class);
+        clazz.getMethod("getBoolean", Object.class, long.class);
+        clazz.getMethod("putBoolean", Object.class, long.class, boolean.class);
+        clazz.getMethod("getInt", Object.class, long.class);
+        clazz.getMethod("putInt", Object.class, long.class, int.class);
         clazz.getMethod("getLong", Object.class, long.class);
+        clazz.getMethod("putLong", Object.class, long.class, long.class);
+        clazz.getMethod("getFloat", Object.class, long.class);
+        clazz.getMethod("putFloat", Object.class, long.class, float.class);
+        clazz.getMethod("getDouble", Object.class, long.class);
+        clazz.getMethod("putDouble", Object.class, long.class, double.class);
+        clazz.getMethod("getObject", Object.class, long.class);
+        clazz.getMethod("putObject", Object.class, long.class, Object.class);
         clazz.getMethod(
             "copyMemory", Object.class, long.class, Object.class, long.class, long.class);
         supported = true;
@@ -163,11 +242,17 @@ final class UnsafeUtil {
     if (UNSAFE != null) {
       try {
         Class<?> clazz = UNSAFE.getClass();
+        // Methods for getting direct buffer address.
         clazz.getMethod("objectFieldOffset", Field.class);
-        clazz.getMethod("getByte", long.class);
         clazz.getMethod("getLong", Object.class, long.class);
+
+        clazz.getMethod("getByte", long.class);
         clazz.getMethod("putByte", long.class, byte.class);
+        clazz.getMethod("getInt", long.class);
+        clazz.getMethod("putInt", long.class, int.class);
         clazz.getMethod("getLong", long.class);
+        clazz.getMethod("putLong", long.class, long.class);
+        clazz.getMethod("setMemory", long.class, long.class, byte.class);
         clazz.getMethod("copyMemory", long.class, long.class, long.class);
         supported = true;
       } catch (Throwable e) {

java/core/src/main/java/com/google/protobuf/WireFormat.java

@@ -47,6 +47,10 @@ public final class WireFormat {
   // Do not allow instantiation.
   private WireFormat() {}
 
+  static final int FIXED_32_SIZE = 4;
+  static final int FIXED_64_SIZE = 8;
+  static final int MAX_VARINT_SIZE = 10;
+
   public static final int WIRETYPE_VARINT           = 0;
   public static final int WIRETYPE_FIXED64          = 1;
   public static final int WIRETYPE_LENGTH_DELIMITED = 2;

java/core/src/test/java/com/google/protobuf/AbstractMessageTest.java

@@ -40,9 +40,8 @@ import protobuf_unittest.UnittestProto.TestPackedTypes;
 import protobuf_unittest.UnittestProto.TestRequired;
 import protobuf_unittest.UnittestProto.TestRequiredForeign;
 import protobuf_unittest.UnittestProto.TestUnpackedTypes;
-import junit.framework.TestCase;
-
 import java.util.Map;
+import junit.framework.TestCase;
 
 /**
  * Unit test for {@link AbstractMessage}.
@@ -492,7 +491,6 @@ public class AbstractMessageTest extends TestCase {
     checkEqualsIsConsistent(eUnknownFields, eUnknownFields2);
   }
 
-
   /**
    * Asserts that the given proto has symmetric equals and hashCode methods.
    */

java/core/src/test/java/com/google/protobuf/BooleanArrayListTest.java

@@ -32,11 +32,10 @@ package com.google.protobuf;
 
 import static java.util.Arrays.asList;
 
-import junit.framework.TestCase;
-
 import java.util.Collections;
 import java.util.ConcurrentModificationException;
 import java.util.Iterator;
+import junit.framework.TestCase;
 
 /**
  * Tests for {@link BooleanArrayList}.

java/core/src/test/java/com/google/protobuf/ByteBufferWriterTest.java

@@ -30,13 +30,12 @@
 
 package com.google.protobuf;
 
-import junit.framework.TestCase;
-
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.Random;
+import junit.framework.TestCase;
 
 /**
  * Tests for {@link ByteBufferWriter}.

java/core/src/test/java/com/google/protobuf/ByteStringTest.java

@@ -31,9 +31,6 @@
 package com.google.protobuf;
 
 import com.google.protobuf.ByteString.Output;
-
-import junit.framework.TestCase;
-
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -48,6 +45,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.NoSuchElementException;
 import java.util.Random;
+import junit.framework.TestCase;
 
 /**
  * Test methods with implementations in {@link ByteString}, plus do some top-level "integration"

java/core/src/test/java/com/google/protobuf/CodedInputStreamTest.java

@@ -30,20 +30,20 @@
 
 package com.google.protobuf;
 
+import static org.junit.Assert.assertArrayEquals;
+
 import protobuf_unittest.UnittestProto.BoolMessage;
 import protobuf_unittest.UnittestProto.Int32Message;
 import protobuf_unittest.UnittestProto.Int64Message;
 import protobuf_unittest.UnittestProto.TestAllTypes;
 import protobuf_unittest.UnittestProto.TestRecursiveMessage;
-
-import junit.framework.TestCase;
-
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.FilterInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.ByteBuffer;
+import junit.framework.TestCase;
 
 /**
  * Unit test for {@link CodedInputStream}.
@@ -51,10 +51,45 @@ import java.nio.ByteBuffer;
  * @author kenton@google.com Kenton Varda
  */
 public class CodedInputStreamTest extends TestCase {
+  private enum InputType {
+    ARRAY {
+      @Override
+      CodedInputStream newDecoder(byte[] data, int blockSize) {
+        return CodedInputStream.newInstance(data);
+      }
+    },
+    NIO_HEAP {
+      @Override
+      CodedInputStream newDecoder(byte[] data, int blockSize) {
+        return CodedInputStream.newInstance(ByteBuffer.wrap(data));
+      }
+    },
+    NIO_DIRECT {
+      @Override
+      CodedInputStream newDecoder(byte[] data, int blockSize) {
+        ByteBuffer buffer = ByteBuffer.allocateDirect(data.length);
+        buffer.put(data);
+        buffer.flip();
+        return CodedInputStream.newInstance(buffer);
+      }
+    },
+    STREAM {
+      @Override
+      CodedInputStream newDecoder(byte[] data, int blockSize) {
+        return CodedInputStream.newInstance(new SmallBlockInputStream(data, blockSize));
+      }
+    };
+
+    CodedInputStream newDecoder(byte[] data) {
+      return newDecoder(data, data.length);
+    }
+
+    abstract CodedInputStream newDecoder(byte[] data, int blockSize);
+  }
+
   /**
-   * Helper to construct a byte array from a bunch of bytes.  The inputs are
-   * actually ints so that I can use hex notation and not get stupid errors
-   * about precision.
+   * Helper to construct a byte array from a bunch of bytes. The inputs are actually ints so that I
+   * can use hex notation and not get stupid errors about precision.
    */
   private byte[] bytes(int... bytesAsInts) {
     byte[] bytes = new byte[bytesAsInts.length];
@@ -65,19 +100,14 @@ public class CodedInputStreamTest extends TestCase {
   }
 
   /**
-   * An InputStream which limits the number of bytes it reads at a time.
-   * We use this to make sure that CodedInputStream doesn't screw up when
-   * reading in small blocks.
+   * An InputStream which limits the number of bytes it reads at a time. We use this to make sure
+   * that CodedInputStream doesn't screw up when reading in small blocks.
    */
   private static final class SmallBlockInputStream extends FilterInputStream {
     private final int blockSize;
 
     public SmallBlockInputStream(byte[] data, int blockSize) {
-      this(new ByteArrayInputStream(data), blockSize);
-    }
-
-    public SmallBlockInputStream(InputStream in, int blockSize) {
-      super(in);
+      super(new ByteArrayInputStream(data));
       this.blockSize = blockSize;
     }
 
@@ -92,54 +122,36 @@ public class CodedInputStreamTest extends TestCase {
     }
   }
 
-  private void assertDataConsumed(byte[] data, CodedInputStream input)
+  private void assertDataConsumed(String msg, byte[] data, CodedInputStream input)
       throws IOException {
-    assertEquals(data.length, input.getTotalBytesRead());
-    assertTrue(input.isAtEnd());
+    assertEquals(msg, data.length, input.getTotalBytesRead());
+    assertTrue(msg, input.isAtEnd());
   }
 
   /**
-   * Parses the given bytes using readRawVarint32() and readRawVarint64() and
-   * checks that the result matches the given value.
+   * Parses the given bytes using readRawVarint32() and readRawVarint64() and checks that the result
+   * matches the given value.
    */
   private void assertReadVarint(byte[] data, long value) throws Exception {
-    CodedInputStream input = CodedInputStream.newInstance(data);
-    assertEquals((int) value, input.readRawVarint32());
-    assertDataConsumed(data, input);
-
-    input = CodedInputStream.newInstance(data);
-    assertEquals(value, input.readRawVarint64());
-    assertDataConsumed(data, input);
-
-    input = CodedInputStream.newInstance(data);
-    assertEquals(value, input.readRawVarint64SlowPath());
-    assertDataConsumed(data, input);
-
-    input = CodedInputStream.newInstance(data);
-    assertTrue(input.skipField(WireFormat.WIRETYPE_VARINT));
-    assertDataConsumed(data, input);
-
-    // Try different block sizes.
-    for (int blockSize = 1; blockSize <= 16; blockSize *= 2) {
-      input = CodedInputStream.newInstance(
-        new SmallBlockInputStream(data, blockSize));
-      assertEquals((int) value, input.readRawVarint32());
-      assertDataConsumed(data, input);
-
-      input = CodedInputStream.newInstance(
-        new SmallBlockInputStream(data, blockSize));
-      assertEquals(value, input.readRawVarint64());
-      assertDataConsumed(data, input);
-
-      input = CodedInputStream.newInstance(
-        new SmallBlockInputStream(data, blockSize));
-      assertEquals(value, input.readRawVarint64SlowPath());
-      assertDataConsumed(data, input);
-
-      input = CodedInputStream.newInstance(
-        new SmallBlockInputStream(data, blockSize));
-      assertTrue(input.skipField(WireFormat.WIRETYPE_VARINT));
-      assertDataConsumed(data, input);
+    for (InputType inputType : InputType.values()) {
+      // Try different block sizes.
+      for (int blockSize = 1; blockSize <= 16; blockSize *= 2) {
+        CodedInputStream input = inputType.newDecoder(data, blockSize);
+        assertEquals(inputType.name(), (int) value, input.readRawVarint32());
+        assertDataConsumed(inputType.name(), data, input);
+
+        input = inputType.newDecoder(data, blockSize);
+        assertEquals(inputType.name(), value, input.readRawVarint64());
+        assertDataConsumed(inputType.name(), data, input);
+
+        input = inputType.newDecoder(data, blockSize);
+        assertEquals(inputType.name(), value, input.readRawVarint64SlowPath());
+        assertDataConsumed(inputType.name(), data, input);
+
+        input = inputType.newDecoder(data, blockSize);
+        assertTrue(inputType.name(), input.skipField(WireFormat.WIRETYPE_VARINT));
+        assertDataConsumed(inputType.name(), data, input);
+      }
     }
 
     // Try reading direct from an InputStream.  We want to verify that it
@@ -153,35 +165,26 @@ public class CodedInputStreamTest extends TestCase {
   }
 
   /**
-   * Parses the given bytes using readRawVarint32() and readRawVarint64() and
-   * expects them to fail with an InvalidProtocolBufferException whose
-   * description matches the given one.
+   * Parses the given bytes using readRawVarint32() and readRawVarint64() and expects them to fail
+   * with an InvalidProtocolBufferException whose description matches the given one.
    */
-  private void assertReadVarintFailure(
-      InvalidProtocolBufferException expected, byte[] data)
+  private void assertReadVarintFailure(InvalidProtocolBufferException expected, byte[] data)
       throws Exception {
-    CodedInputStream input = CodedInputStream.newInstance(data);
-    try {
-      input.readRawVarint32();
-      fail("Should have thrown an exception.");
-    } catch (InvalidProtocolBufferException e) {
-      assertEquals(expected.getMessage(), e.getMessage());
-    }
-
-    input = CodedInputStream.newInstance(data);
-    try {
-      input.readRawVarint64();
-      fail("Should have thrown an exception.");
-    } catch (InvalidProtocolBufferException e) {
-      assertEquals(expected.getMessage(), e.getMessage());
-    }
-
-    input = CodedInputStream.newInstance(data);
-    try {
-      input.readRawVarint64SlowPath();
-      fail("Should have thrown an exception.");
-    } catch (InvalidProtocolBufferException e) {
-      assertEquals(expected.getMessage(), e.getMessage());
+    for (InputType inputType : InputType.values()) {
+      try {
+        CodedInputStream input = inputType.newDecoder(data);
+        input.readRawVarint32();
+        fail(inputType.name() + ": Should have thrown an exception.");
+      } catch (InvalidProtocolBufferException e) {
+        assertEquals(inputType.name(), expected.getMessage(), e.getMessage());
+      }
+      try {
+        CodedInputStream input = inputType.newDecoder(data);
+        input.readRawVarint64();
+        fail(inputType.name() + ": Should have thrown an exception.");
+      } catch (InvalidProtocolBufferException e) {
+        assertEquals(inputType.name(), expected.getMessage(), e.getMessage());
+      }
     }
 
     // Make sure we get the same error when reading direct from an InputStream.
@@ -201,72 +204,74 @@ public class CodedInputStreamTest extends TestCase {
     // 14882
     assertReadVarint(bytes(0xa2, 0x74), (0x22 << 0) | (0x74 << 7));
     // 2961488830
-    assertReadVarint(bytes(0xbe, 0xf7, 0x92, 0x84, 0x0b),
-      (0x3e << 0) | (0x77 << 7) | (0x12 << 14) | (0x04 << 21) |
-      (0x0bL << 28));
+    assertReadVarint(
+        bytes(0xbe, 0xf7, 0x92, 0x84, 0x0b),
+        (0x3e << 0) | (0x77 << 7) | (0x12 << 14) | (0x04 << 21) | (0x0bL << 28));
 
     // 64-bit
     // 7256456126
-    assertReadVarint(bytes(0xbe, 0xf7, 0x92, 0x84, 0x1b),
-      (0x3e << 0) | (0x77 << 7) | (0x12 << 14) | (0x04 << 21) |
-      (0x1bL << 28));
+    assertReadVarint(
+        bytes(0xbe, 0xf7, 0x92, 0x84, 0x1b),
+        (0x3e << 0) | (0x77 << 7) | (0x12 << 14) | (0x04 << 21) | (0x1bL << 28));
     // 41256202580718336
     assertReadVarint(
-      bytes(0x80, 0xe6, 0xeb, 0x9c, 0xc3, 0xc9, 0xa4, 0x49),
-      (0x00 << 0) | (0x66 << 7) | (0x6b << 14) | (0x1c << 21) |
-      (0x43L << 28) | (0x49L << 35) | (0x24L << 42) | (0x49L << 49));
+        bytes(0x80, 0xe6, 0xeb, 0x9c, 0xc3, 0xc9, 0xa4, 0x49),
+        (0x00 << 0)
+            | (0x66 << 7)
+            | (0x6b << 14)
+            | (0x1c << 21)
+            | (0x43L << 28)
+            | (0x49L << 35)
+            | (0x24L << 42)
+            | (0x49L << 49));
     // 11964378330978735131
     assertReadVarint(
-      bytes(0x9b, 0xa8, 0xf9, 0xc2, 0xbb, 0xd6, 0x80, 0x85, 0xa6, 0x01),
-      (0x1b << 0) | (0x28 << 7) | (0x79 << 14) | (0x42 << 21) |
-      (0x3bL << 28) | (0x56L << 35) | (0x00L << 42) |
-      (0x05L << 49) | (0x26L << 56) | (0x01L << 63));
+        bytes(0x9b, 0xa8, 0xf9, 0xc2, 0xbb, 0xd6, 0x80, 0x85, 0xa6, 0x01),
+        (0x1b << 0)
+            | (0x28 << 7)
+            | (0x79 << 14)
+            | (0x42 << 21)
+            | (0x3bL << 28)
+            | (0x56L << 35)
+            | (0x00L << 42)
+            | (0x05L << 49)
+            | (0x26L << 56)
+            | (0x01L << 63));
 
     // Failures
     assertReadVarintFailure(
-      InvalidProtocolBufferException.malformedVarint(),
-      bytes(0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
-            0x00));
-    assertReadVarintFailure(
-      InvalidProtocolBufferException.truncatedMessage(),
-      bytes(0x80));
+        InvalidProtocolBufferException.malformedVarint(),
+        bytes(0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x00));
+    assertReadVarintFailure(InvalidProtocolBufferException.truncatedMessage(), bytes(0x80));
   }
 
   /**
-   * Parses the given bytes using readRawLittleEndian32() and checks
-   * that the result matches the given value.
+   * Parses the given bytes using readRawLittleEndian32() and checks that the result matches the
+   * given value.
    */
-  private void assertReadLittleEndian32(byte[] data, int value)
-                                        throws Exception {
-    CodedInputStream input = CodedInputStream.newInstance(data);
-    assertEquals(value, input.readRawLittleEndian32());
-    assertTrue(input.isAtEnd());
-
-    // Try different block sizes.
-    for (int blockSize = 1; blockSize <= 16; blockSize *= 2) {
-      input = CodedInputStream.newInstance(
-        new SmallBlockInputStream(data, blockSize));
-      assertEquals(value, input.readRawLittleEndian32());
-      assertTrue(input.isAtEnd());
+  private void assertReadLittleEndian32(byte[] data, int value) throws Exception {
+    for (InputType inputType : InputType.values()) {
+      // Try different block sizes.
+      for (int blockSize = 1; blockSize <= 16; blockSize *= 2) {
+        CodedInputStream input = inputType.newDecoder(data, blockSize);
+        assertEquals(inputType.name(), value, input.readRawLittleEndian32());
+        assertTrue(inputType.name(), input.isAtEnd());
+      }
     }
   }
 
   /**
-   * Parses the given bytes using readRawLittleEndian64() and checks
-   * that the result matches the given value.
+   * Parses the given bytes using readRawLittleEndian64() and checks that the result matches the
+   * given value.
    */
-  private void assertReadLittleEndian64(byte[] data, long value)
-                                        throws Exception {
-    CodedInputStream input = CodedInputStream.newInstance(data);
-    assertEquals(value, input.readRawLittleEndian64());
-    assertTrue(input.isAtEnd());
-
-    // Try different block sizes.
-    for (int blockSize = 1; blockSize <= 16; blockSize *= 2) {
-      input = CodedInputStream.newInstance(
-        new SmallBlockInputStream(data, blockSize));
-      assertEquals(value, input.readRawLittleEndian64());
-      assertTrue(input.isAtEnd());
+  private void assertReadLittleEndian64(byte[] data, long value) throws Exception {
+    for (InputType inputType : InputType.values()) {
+      // Try different block sizes.
+      for (int blockSize = 1; blockSize <= 16; blockSize *= 2) {
+        CodedInputStream input = inputType.newDecoder(data, blockSize);
+        assertEquals(inputType.name(), value, input.readRawLittleEndian64());
+        assertTrue(inputType.name(), input.isAtEnd());
+      }
     }
   }
 
@@ -276,40 +281,32 @@ public class CodedInputStreamTest extends TestCase {
     assertReadLittleEndian32(bytes(0xf0, 0xde, 0xbc, 0x9a), 0x9abcdef0);
 
     assertReadLittleEndian64(
-      bytes(0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12),
-      0x123456789abcdef0L);
+        bytes(0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12), 0x123456789abcdef0L);
     assertReadLittleEndian64(
-      bytes(0x78, 0x56, 0x34, 0x12, 0xf0, 0xde, 0xbc, 0x9a),
-      0x9abcdef012345678L);
+        bytes(0x78, 0x56, 0x34, 0x12, 0xf0, 0xde, 0xbc, 0x9a), 0x9abcdef012345678L);
   }
 
   /** Test decodeZigZag32() and decodeZigZag64(). */
   public void testDecodeZigZag() throws Exception {
-    assertEquals( 0, CodedInputStream.decodeZigZag32(0));
+    assertEquals(0, CodedInputStream.decodeZigZag32(0));
     assertEquals(-1, CodedInputStream.decodeZigZag32(1));
-    assertEquals( 1, CodedInputStream.decodeZigZag32(2));
+    assertEquals(1, CodedInputStream.decodeZigZag32(2));
     assertEquals(-2, CodedInputStream.decodeZigZag32(3));
     assertEquals(0x3FFFFFFF, CodedInputStream.decodeZigZag32(0x7FFFFFFE));
     assertEquals(0xC0000000, CodedInputStream.decodeZigZag32(0x7FFFFFFF));
     assertEquals(0x7FFFFFFF, CodedInputStream.decodeZigZag32(0xFFFFFFFE));
     assertEquals(0x80000000, CodedInputStream.decodeZigZag32(0xFFFFFFFF));
 
-    assertEquals( 0, CodedInputStream.decodeZigZag64(0));
+    assertEquals(0, CodedInputStream.decodeZigZag64(0));
     assertEquals(-1, CodedInputStream.decodeZigZag64(1));
-    assertEquals( 1, CodedInputStream.decodeZigZag64(2));
+    assertEquals(1, CodedInputStream.decodeZigZag64(2));
     assertEquals(-2, CodedInputStream.decodeZigZag64(3));
-    assertEquals(0x000000003FFFFFFFL,
-                 CodedInputStream.decodeZigZag64(0x000000007FFFFFFEL));
-    assertEquals(0xFFFFFFFFC0000000L,
-                 CodedInputStream.decodeZigZag64(0x000000007FFFFFFFL));
-    assertEquals(0x000000007FFFFFFFL,
-                 CodedInputStream.decodeZigZag64(0x00000000FFFFFFFEL));
-    assertEquals(0xFFFFFFFF80000000L,
-                 CodedInputStream.decodeZigZag64(0x00000000FFFFFFFFL));
-    assertEquals(0x7FFFFFFFFFFFFFFFL,
-                 CodedInputStream.decodeZigZag64(0xFFFFFFFFFFFFFFFEL));
-    assertEquals(0x8000000000000000L,
-                 CodedInputStream.decodeZigZag64(0xFFFFFFFFFFFFFFFFL));
+    assertEquals(0x000000003FFFFFFFL, CodedInputStream.decodeZigZag64(0x000000007FFFFFFEL));
+    assertEquals(0xFFFFFFFFC0000000L, CodedInputStream.decodeZigZag64(0x000000007FFFFFFFL));
+    assertEquals(0x000000007FFFFFFFL, CodedInputStream.decodeZigZag64(0x00000000FFFFFFFEL));
+    assertEquals(0xFFFFFFFF80000000L, CodedInputStream.decodeZigZag64(0x00000000FFFFFFFFL));
+    assertEquals(0x7FFFFFFFFFFFFFFFL, CodedInputStream.decodeZigZag64(0xFFFFFFFFFFFFFFFEL));
+    assertEquals(0x8000000000000000L, CodedInputStream.decodeZigZag64(0xFFFFFFFFFFFFFFFFL));
   }
 
   /** Tests reading and parsing a whole message with every field type. */
@@ -319,14 +316,12 @@ public class CodedInputStreamTest extends TestCase {
     byte[] rawBytes = message.toByteArray();
     assertEquals(rawBytes.length, message.getSerializedSize());
 
-    TestAllTypes message2 = TestAllTypes.parseFrom(rawBytes);
-    TestUtil.assertAllFieldsSet(message2);
-
-    // Try different block sizes.
-    for (int blockSize = 1; blockSize < 256; blockSize *= 2) {
-      message2 = TestAllTypes.parseFrom(
-        new SmallBlockInputStream(rawBytes, blockSize));
-      TestUtil.assertAllFieldsSet(message2);
+    for (InputType inputType : InputType.values()) {
+      // Try different block sizes.
+      for (int blockSize = 1; blockSize < 256; blockSize *= 2) {
+        TestAllTypes message2 = TestAllTypes.parseFrom(inputType.newDecoder(rawBytes, blockSize));
+        TestUtil.assertAllFieldsSet(message2);
+      }
     }
   }
 
@@ -335,57 +330,65 @@ public class CodedInputStreamTest extends TestCase {
     TestAllTypes message = TestUtil.getAllSet();
     byte[] rawBytes = message.toByteArray();
 
-    // Create two parallel inputs.  Parse one as unknown fields while using
-    // skipField() to skip each field on the other.  Expect the same tags.
-    CodedInputStream input1 = CodedInputStream.newInstance(rawBytes);
-    CodedInputStream input2 = CodedInputStream.newInstance(rawBytes);
+    InputType[] inputTypes = InputType.values();
+    CodedInputStream[] inputs = new CodedInputStream[inputTypes.length];
+    for (int i = 0; i < inputs.length; ++i) {
+      inputs[i] = inputTypes[i].newDecoder(rawBytes);
+    }
     UnknownFieldSet.Builder unknownFields = UnknownFieldSet.newBuilder();
 
     while (true) {
+      CodedInputStream input1 = inputs[0];
       int tag = input1.readTag();
-      assertEquals(tag, input2.readTag());
+      // Ensure that the rest match.
+      for (int i = 1; i < inputs.length; ++i) {
+        assertEquals(inputTypes[i].name(), tag, inputs[i].readTag());
+      }
       if (tag == 0) {
         break;
       }
       unknownFields.mergeFieldFrom(tag, input1);
-      input2.skipField(tag);
+      // Skip the field for the rest of the inputs.
+      for (int i = 1; i < inputs.length; ++i) {
+        inputs[i].skipField(tag);
+      }
     }
   }
 
 
   /**
-   * Test that a bug in skipRawBytes() has been fixed:  if the skip skips
-   * exactly up to a limit, this should not break things.
+   * Test that a bug in skipRawBytes() has been fixed: if the skip skips exactly up to a limit, this
+   * should not break things.
    */
   public void testSkipRawBytesBug() throws Exception {
-    byte[] rawBytes = new byte[] { 1, 2 };
-    CodedInputStream input = CodedInputStream.newInstance(rawBytes);
-
-    int limit = input.pushLimit(1);
-    input.skipRawBytes(1);
-    input.popLimit(limit);
-    assertEquals(2, input.readRawByte());
+    byte[] rawBytes = new byte[] {1, 2};
+    for (InputType inputType : InputType.values()) {
+      CodedInputStream input = inputType.newDecoder(rawBytes);
+      int limit = input.pushLimit(1);
+      input.skipRawBytes(1);
+      input.popLimit(limit);
+      assertEquals(inputType.name(), 2, input.readRawByte());
+    }
   }
 
   /**
-   * Test that a bug in skipRawBytes() has been fixed:  if the skip skips
-   * past the end of a buffer with a limit that has been set past the end of
-   * that buffer, this should not break things.
+   * Test that a bug in skipRawBytes() has been fixed: if the skip skips past the end of a buffer
+   * with a limit that has been set past the end of that buffer, this should not break things.
    */
   public void testSkipRawBytesPastEndOfBufferWithLimit() throws Exception {
-    byte[] rawBytes = new byte[] { 1, 2, 3, 4, 5 };
-    CodedInputStream input = CodedInputStream.newInstance(
-        new SmallBlockInputStream(rawBytes, 3));
-
-    int limit = input.pushLimit(4);
-    // In order to expose the bug we need to read at least one byte to prime the
-    // buffer inside the CodedInputStream.
-    assertEquals(1, input.readRawByte());
-    // Skip to the end of the limit.
-    input.skipRawBytes(3);
-    assertTrue(input.isAtEnd());
-    input.popLimit(limit);
-    assertEquals(5, input.readRawByte());
+    byte[] rawBytes = new byte[] {1, 2, 3, 4, 5};
+    for (InputType inputType : InputType.values()) {
+      CodedInputStream input = inputType.newDecoder(rawBytes);
+      int limit = input.pushLimit(4);
+      // In order to expose the bug we need to read at least one byte to prime the
+      // buffer inside the CodedInputStream.
+      assertEquals(inputType.name(), 1, input.readRawByte());
+      // Skip to the end of the limit.
+      input.skipRawBytes(3);
+      assertTrue(inputType.name(), input.isAtEnd());
+      input.popLimit(limit);
+      assertEquals(inputType.name(), 5, input.readRawByte());
+    }
   }
 
   public void testReadHugeBlob() throws Exception {
@@ -401,19 +404,22 @@ public class CodedInputStreamTest extends TestCase {
     builder.setOptionalBytes(ByteString.copyFrom(blob));
     TestAllTypes message = builder.build();
 
-    // Serialize and parse it.  Make sure to parse from an InputStream, not
-    // directly from a ByteString, so that CodedInputStream uses buffered
-    // reading.
-    TestAllTypes message2 =
-      TestAllTypes.parseFrom(message.toByteString().newInput());
-
-    assertEquals(message.getOptionalBytes(), message2.getOptionalBytes());
-
-    // Make sure all the other fields were parsed correctly.
-    TestAllTypes message3 = TestAllTypes.newBuilder(message2)
-      .setOptionalBytes(TestUtil.getAllSet().getOptionalBytes())
-      .build();
-    TestUtil.assertAllFieldsSet(message3);
+    byte[] data = message.toByteArray();
+    for (InputType inputType : InputType.values()) {
+      // Serialize and parse it.  Make sure to parse from an InputStream, not
+      // directly from a ByteString, so that CodedInputStream uses buffered
+      // reading.
+      TestAllTypes message2 = TestAllTypes.parseFrom(inputType.newDecoder(data));
+
+      assertEquals(inputType.name(), message.getOptionalBytes(), message2.getOptionalBytes());
+
+      // Make sure all the other fields were parsed correctly.
+      TestAllTypes message3 =
+          TestAllTypes.newBuilder(message2)
+              .setOptionalBytes(TestUtil.getAllSet().getOptionalBytes())
+              .build();
+      TestUtil.assertAllFieldsSet(message3);
+    }
   }
 
   public void testReadMaliciouslyLargeBlob() throws Exception {
@@ -423,17 +429,19 @@ public class CodedInputStreamTest extends TestCase {
     int tag = WireFormat.makeTag(1, WireFormat.WIRETYPE_LENGTH_DELIMITED);
     output.writeRawVarint32(tag);
     output.writeRawVarint32(0x7FFFFFFF);
-    output.writeRawBytes(new byte[32]);  // Pad with a few random bytes.
+    output.writeRawBytes(new byte[32]); // Pad with a few random bytes.
     output.flush();
 
-    CodedInputStream input = rawOutput.toByteString().newCodedInput();
-    assertEquals(tag, input.readTag());
-
-    try {
-      input.readBytes();
-      fail("Should have thrown an exception!");
-    } catch (InvalidProtocolBufferException e) {
-      // success.
+    byte[] data = rawOutput.toByteString().toByteArray();
+    for (InputType inputType : InputType.values()) {
+      CodedInputStream input = inputType.newDecoder(data);
+      assertEquals(tag, input.readTag());
+      try {
+        input.readBytes();
+        fail(inputType.name() + ": Should have thrown an exception!");
+      } catch (InvalidProtocolBufferException e) {
+        // success.
+      }
     }
   }
 
@@ -441,54 +449,55 @@ public class CodedInputStreamTest extends TestCase {
     if (depth == 0) {
       return TestRecursiveMessage.newBuilder().setI(5).build();
     } else {
-      return TestRecursiveMessage.newBuilder()
-        .setA(makeRecursiveMessage(depth - 1)).build();
+      return TestRecursiveMessage.newBuilder().setA(makeRecursiveMessage(depth - 1)).build();
     }
   }
 
-  private void assertMessageDepth(TestRecursiveMessage message, int depth) {
+  private void assertMessageDepth(String msg, TestRecursiveMessage message, int depth) {
     if (depth == 0) {
-      assertFalse(message.hasA());
-      assertEquals(5, message.getI());
+      assertFalse(msg, message.hasA());
+      assertEquals(msg, 5, message.getI());
     } else {
-      assertTrue(message.hasA());
-      assertMessageDepth(message.getA(), depth - 1);
+      assertTrue(msg, message.hasA());
+      assertMessageDepth(msg, message.getA(), depth - 1);
     }
   }
 
   public void testMaliciousRecursion() throws Exception {
-    ByteString data100 = makeRecursiveMessage(100).toByteString();
-    ByteString data101 = makeRecursiveMessage(101).toByteString();
+    byte[] data100 = makeRecursiveMessage(100).toByteArray();
+    byte[] data101 = makeRecursiveMessage(101).toByteArray();
 
-    assertMessageDepth(TestRecursiveMessage.parseFrom(data100), 100);
+    for (InputType inputType : InputType.values()) {
+      assertMessageDepth(
+          inputType.name(), TestRecursiveMessage.parseFrom(inputType.newDecoder(data100)), 100);
 
-    try {
-      TestRecursiveMessage.parseFrom(data101);
-      fail("Should have thrown an exception!");
-    } catch (InvalidProtocolBufferException e) {
-      // success.
-    }
+      try {
+        TestRecursiveMessage.parseFrom(inputType.newDecoder(data101));
+        fail("Should have thrown an exception!");
+      } catch (InvalidProtocolBufferException e) {
+        // success.
+      }
 
-    CodedInputStream input = data100.newCodedInput();
-    input.setRecursionLimit(8);
-    try {
-      TestRecursiveMessage.parseFrom(input);
-      fail("Should have thrown an exception!");
-    } catch (InvalidProtocolBufferException e) {
-      // success.
+      CodedInputStream input = inputType.newDecoder(data100);
+      input.setRecursionLimit(8);
+      try {
+        TestRecursiveMessage.parseFrom(input);
+        fail(inputType.name() + ": Should have thrown an exception!");
+      } catch (InvalidProtocolBufferException e) {
+        // success.
+      }
     }
   }
 
   private void checkSizeLimitExceeded(InvalidProtocolBufferException e) {
-      assertEquals(
-          InvalidProtocolBufferException.sizeLimitExceeded().getMessage(),
-          e.getMessage());
+    assertEquals(InvalidProtocolBufferException.sizeLimitExceeded().getMessage(), e.getMessage());
   }
 
   public void testSizeLimit() throws Exception {
-    CodedInputStream input = CodedInputStream.newInstance(
-        new SmallBlockInputStream(
-            TestUtil.getAllSet().toByteString().newInput(), 16));
+    // NOTE: Size limit only applies to the stream-backed CIS.
+    CodedInputStream input =
+        CodedInputStream.newInstance(
+            new SmallBlockInputStream(TestUtil.getAllSet().toByteArray(), 16));
     input.setSizeLimit(16);
 
     try {
@@ -500,8 +509,9 @@ public class CodedInputStreamTest extends TestCase {
   }
 
   public void testResetSizeCounter() throws Exception {
-    CodedInputStream input = CodedInputStream.newInstance(
-        new SmallBlockInputStream(new byte[256], 8));
+    // NOTE: Size limit only applies to the stream-backed CIS.
+    CodedInputStream input =
+        CodedInputStream.newInstance(new SmallBlockInputStream(new byte[256], 8));
     input.setSizeLimit(16);
     input.readRawBytes(16);
     assertEquals(16, input.getTotalBytesRead());
@@ -515,7 +525,7 @@ public class CodedInputStreamTest extends TestCase {
 
     input.resetSizeCounter();
     assertEquals(0, input.getTotalBytesRead());
-    input.readRawByte();  // No exception thrown.
+    input.readRawByte(); // No exception thrown.
     input.resetSizeCounter();
     assertEquals(0, input.getTotalBytesRead());
     input.readRawBytes(16);
@@ -523,7 +533,7 @@ public class CodedInputStreamTest extends TestCase {
     input.resetSizeCounter();
 
     try {
-      input.readRawBytes(17);  // Hits limit again.
+      input.readRawBytes(17); // Hits limit again.
       fail("Should have thrown an exception!");
     } catch (InvalidProtocolBufferException expected) {
       checkSizeLimitExceeded(expected);
@@ -531,12 +541,12 @@ public class CodedInputStreamTest extends TestCase {
   }
 
   public void testSizeLimitMultipleMessages() throws Exception {
+    // NOTE: Size limit only applies to the stream-backed CIS.
     byte[] bytes = new byte[256];
     for (int i = 0; i < bytes.length; i++) {
       bytes[i] = (byte) i;
     }
-    CodedInputStream input = CodedInputStream.newInstance(
-        new SmallBlockInputStream(bytes, 7));
+    CodedInputStream input = CodedInputStream.newInstance(new SmallBlockInputStream(bytes, 7));
     input.setSizeLimit(16);
     for (int i = 0; i < 256 / 16; i++) {
       byte[] message = input.readRawBytes(16);
@@ -566,12 +576,13 @@ public class CodedInputStreamTest extends TestCase {
     output.writeRawBytes(bytes);
     output.flush();
 
-    CodedInputStream input =
-        CodedInputStream.newInstance(
-            new ByteArrayInputStream(rawOutput.toByteString().toByteArray()));
-    assertEquals(tag, input.readTag());
-    String text = input.readString();
-    assertEquals(lorem, text);
+    byte[] rawInput = rawOutput.toByteString().toByteArray();
+    for (InputType inputType : InputType.values()) {
+      CodedInputStream input = inputType.newDecoder(rawInput);
+      assertEquals(inputType.name(), tag, input.readTag());
+      String text = input.readString();
+      assertEquals(inputType.name(), lorem, text);
+    }
   }
 
   public void testReadStringRequireUtf8() throws Exception {
@@ -591,18 +602,18 @@ public class CodedInputStreamTest extends TestCase {
     output.writeRawBytes(bytes);
     output.flush();
 
-    CodedInputStream input =
-        CodedInputStream.newInstance(
-            new ByteArrayInputStream(rawOutput.toByteString().toByteArray()));
-    assertEquals(tag, input.readTag());
-    String text = input.readStringRequireUtf8();
-    assertEquals(lorem, text);
+    byte[] rawInput = rawOutput.toByteString().toByteArray();
+    for (InputType inputType : InputType.values()) {
+      CodedInputStream input = inputType.newDecoder(rawInput);
+      assertEquals(inputType.name(), tag, input.readTag());
+      String text = input.readStringRequireUtf8();
+      assertEquals(inputType.name(), lorem, text);
+    }
   }
 
   /**
-   * Tests that if we readString invalid UTF-8 bytes, no exception
-   * is thrown.  Instead, the invalid bytes are replaced with the Unicode
-   * "replacement character" U+FFFD.
+   * Tests that if we readString invalid UTF-8 bytes, no exception is thrown. Instead, the invalid
+   * bytes are replaced with the Unicode "replacement character" U+FFFD.
    */
   public void testReadStringInvalidUtf8() throws Exception {
     ByteString.Output rawOutput = ByteString.newOutput();
@@ -611,18 +622,21 @@ public class CodedInputStreamTest extends TestCase {
     int tag = WireFormat.makeTag(1, WireFormat.WIRETYPE_LENGTH_DELIMITED);
     output.writeRawVarint32(tag);
     output.writeRawVarint32(1);
-    output.writeRawBytes(new byte[] { (byte) 0x80 });
+    output.writeRawBytes(new byte[] {(byte) 0x80});
     output.flush();
 
-    CodedInputStream input = rawOutput.toByteString().newCodedInput();
-    assertEquals(tag, input.readTag());
-    String text = input.readString();
-    assertEquals(0xfffd, text.charAt(0));
+    byte[] rawInput = rawOutput.toByteString().toByteArray();
+    for (InputType inputType : InputType.values()) {
+      CodedInputStream input = inputType.newDecoder(rawInput);
+      assertEquals(inputType.name(), tag, input.readTag());
+      String text = input.readString();
+      assertEquals(inputType.name(), 0xfffd, text.charAt(0));
+    }
   }
 
   /**
-   * Tests that if we readStringRequireUtf8 invalid UTF-8 bytes, an
-   * InvalidProtocolBufferException is thrown.
+   * Tests that if we readStringRequireUtf8 invalid UTF-8 bytes, an InvalidProtocolBufferException
+   * is thrown.
    */
   public void testReadStringRequireUtf8InvalidUtf8() throws Exception {
     ByteString.Output rawOutput = ByteString.newOutput();
@@ -631,16 +645,20 @@ public class CodedInputStreamTest extends TestCase {
     int tag = WireFormat.makeTag(1, WireFormat.WIRETYPE_LENGTH_DELIMITED);
     output.writeRawVarint32(tag);
     output.writeRawVarint32(1);
-    output.writeRawBytes(new byte[] { (byte) 0x80 });
+    output.writeRawBytes(new byte[] {(byte) 0x80});
     output.flush();
 
-    CodedInputStream input = rawOutput.toByteString().newCodedInput();
-    assertEquals(tag, input.readTag());
-    try {
-      input.readStringRequireUtf8();
-      fail("Expected invalid UTF-8 exception.");
-    } catch (InvalidProtocolBufferException exception) {
-      assertEquals("Protocol message had invalid UTF-8.", exception.getMessage());
+    byte[] rawInput = rawOutput.toByteString().toByteArray();
+    for (InputType inputType : InputType.values()) {
+      CodedInputStream input = inputType.newDecoder(rawInput);
+      assertEquals(tag, input.readTag());
+      try {
+        input.readStringRequireUtf8();
+        fail(inputType.name() + ": Expected invalid UTF-8 exception.");
+      } catch (InvalidProtocolBufferException exception) {
+        assertEquals(
+            inputType.name(), "Protocol message had invalid UTF-8.", exception.getMessage());
+      }
     }
   }
 
@@ -660,13 +678,17 @@ public class CodedInputStreamTest extends TestCase {
   public void testInvalidTag() throws Exception {
     // Any tag number which corresponds to field number zero is invalid and
     // should throw InvalidProtocolBufferException.
-    for (int i = 0; i < 8; i++) {
-      try {
-        CodedInputStream.newInstance(bytes(i)).readTag();
-        fail("Should have thrown an exception.");
-      } catch (InvalidProtocolBufferException e) {
-        assertEquals(InvalidProtocolBufferException.invalidTag().getMessage(),
-                     e.getMessage());
+    for (InputType inputType : InputType.values()) {
+      for (int i = 0; i < 8; i++) {
+        try {
+          inputType.newDecoder(bytes(i)).readTag();
+          fail(inputType.name() + ": Should have thrown an exception.");
+        } catch (InvalidProtocolBufferException e) {
+          assertEquals(
+              inputType.name(),
+              InvalidProtocolBufferException.invalidTag().getMessage(),
+              e.getMessage());
+        }
       }
     }
   }
@@ -678,10 +700,10 @@ public class CodedInputStreamTest extends TestCase {
     output.writeRawVarint32(0);
     // One one-byte bytes field
     output.writeRawVarint32(1);
-    output.writeRawBytes(new byte[] { (byte) 23 });
+    output.writeRawBytes(new byte[] {(byte) 23});
     // Another one-byte bytes field
     output.writeRawVarint32(1);
-    output.writeRawBytes(new byte[] { (byte) 45 });
+    output.writeRawBytes(new byte[] {(byte) 45});
     // A bytes field large enough that won't fit into the 4K buffer.
     final int bytesLength = 16 * 1024;
     byte[] bytes = new byte[bytesLength];
@@ -691,20 +713,24 @@ public class CodedInputStreamTest extends TestCase {
     output.writeRawBytes(bytes);
 
     output.flush();
-    CodedInputStream inputStream = rawOutput.toByteString().newCodedInput();
-
-    byte[] result = inputStream.readByteArray();
-    assertEquals(0, result.length);
-    result = inputStream.readByteArray();
-    assertEquals(1, result.length);
-    assertEquals((byte) 23, result[0]);
-    result = inputStream.readByteArray();
-    assertEquals(1, result.length);
-    assertEquals((byte) 45, result[0]);
-    result = inputStream.readByteArray();
-    assertEquals(bytesLength, result.length);
-    assertEquals((byte) 67, result[0]);
-    assertEquals((byte) 89, result[bytesLength - 1]);
+
+    byte[] rawInput = rawOutput.toByteString().toByteArray();
+    for (InputType inputType : InputType.values()) {
+      CodedInputStream inputStream = inputType.newDecoder(rawInput);
+
+      byte[] result = inputStream.readByteArray();
+      assertEquals(inputType.name(), 0, result.length);
+      result = inputStream.readByteArray();
+      assertEquals(inputType.name(), 1, result.length);
+      assertEquals(inputType.name(), (byte) 23, result[0]);
+      result = inputStream.readByteArray();
+      assertEquals(inputType.name(), 1, result.length);
+      assertEquals(inputType.name(), (byte) 45, result[0]);
+      result = inputStream.readByteArray();
+      assertEquals(inputType.name(), bytesLength, result.length);
+      assertEquals(inputType.name(), (byte) 67, result[0]);
+      assertEquals(inputType.name(), (byte) 89, result[bytesLength - 1]);
+    }
   }
 
   public void testReadByteBuffer() throws Exception {
@@ -714,10 +740,10 @@ public class CodedInputStreamTest extends TestCase {
     output.writeRawVarint32(0);
     // One one-byte bytes field
     output.writeRawVarint32(1);
-    output.writeRawBytes(new byte[]{(byte) 23});
+    output.writeRawBytes(new byte[] {(byte) 23});
     // Another one-byte bytes field
     output.writeRawVarint32(1);
-    output.writeRawBytes(new byte[]{(byte) 45});
+    output.writeRawBytes(new byte[] {(byte) 45});
     // A bytes field large enough that won't fit into the 4K buffer.
     final int bytesLength = 16 * 1024;
     byte[] bytes = new byte[bytesLength];
@@ -727,21 +753,25 @@ public class CodedInputStreamTest extends TestCase {
     output.writeRawBytes(bytes);
 
     output.flush();
-    CodedInputStream inputStream = rawOutput.toByteString().newCodedInput();
-
-    ByteBuffer result = inputStream.readByteBuffer();
-    assertEquals(0, result.capacity());
-    result = inputStream.readByteBuffer();
-    assertEquals(1, result.capacity());
-    assertEquals((byte) 23, result.get());
-    result = inputStream.readByteBuffer();
-    assertEquals(1, result.capacity());
-    assertEquals((byte) 45, result.get());
-    result = inputStream.readByteBuffer();
-    assertEquals(bytesLength, result.capacity());
-    assertEquals((byte) 67, result.get());
-    result.position(bytesLength - 1);
-    assertEquals((byte) 89, result.get());
+
+    byte[] rawInput = rawOutput.toByteString().toByteArray();
+    for (InputType inputType : InputType.values()) {
+      CodedInputStream inputStream = inputType.newDecoder(rawInput);
+
+      ByteBuffer result = inputStream.readByteBuffer();
+      assertEquals(inputType.name(), 0, result.capacity());
+      result = inputStream.readByteBuffer();
+      assertEquals(inputType.name(), 1, result.capacity());
+      assertEquals(inputType.name(), (byte) 23, result.get());
+      result = inputStream.readByteBuffer();
+      assertEquals(inputType.name(), 1, result.capacity());
+      assertEquals(inputType.name(), (byte) 45, result.get());
+      result = inputStream.readByteBuffer();
+      assertEquals(inputType.name(), bytesLength, result.capacity());
+      assertEquals(inputType.name(), (byte) 67, result.get());
+      result.position(bytesLength - 1);
+      assertEquals(inputType.name(), (byte) 89, result.get());
+    }
   }
 
   public void testReadByteBufferAliasing() throws Exception {
@@ -751,10 +781,10 @@ public class CodedInputStreamTest extends TestCase {
     output.writeRawVarint32(0);
     // One one-byte bytes field
     output.writeRawVarint32(1);
-    output.writeRawBytes(new byte[]{(byte) 23});
+    output.writeRawBytes(new byte[] {(byte) 23});
     // Another one-byte bytes field
     output.writeRawVarint32(1);
-    output.writeRawBytes(new byte[]{(byte) 45});
+    output.writeRawBytes(new byte[] {(byte) 45});
     // A bytes field large enough that won't fit into the 4K buffer.
     final int bytesLength = 16 * 1024;
     byte[] bytes = new byte[bytesLength];
@@ -763,59 +793,105 @@ public class CodedInputStreamTest extends TestCase {
     output.writeRawVarint32(bytesLength);
     output.writeRawBytes(bytes);
     output.flush();
+
     byte[] data = byteArrayStream.toByteArray();
 
-    // Without aliasing
-    CodedInputStream inputStream = CodedInputStream.newInstance(data);
-    ByteBuffer result = inputStream.readByteBuffer();
-    assertEquals(0, result.capacity());
-    result = inputStream.readByteBuffer();
-    assertTrue(result.array() != data);
-    assertEquals(1, result.capacity());
-    assertEquals((byte) 23, result.get());
-    result = inputStream.readByteBuffer();
-    assertTrue(result.array() != data);
-    assertEquals(1, result.capacity());
-    assertEquals((byte) 45, result.get());
-    result = inputStream.readByteBuffer();
-    assertTrue(result.array() != data);
-    assertEquals(bytesLength, result.capacity());
-    assertEquals((byte) 67, result.get());
-    result.position(bytesLength - 1);
-    assertEquals((byte) 89, result.get());
-
-    // Enable aliasing
-    inputStream = CodedInputStream.newInstance(data);
-    inputStream.enableAliasing(true);
-    result = inputStream.readByteBuffer();
-    assertEquals(0, result.capacity());
-    result = inputStream.readByteBuffer();
-    assertTrue(result.array() == data);
-    assertEquals(1, result.capacity());
-    assertEquals((byte) 23, result.get());
-    result = inputStream.readByteBuffer();
-    assertTrue(result.array() == data);
-    assertEquals(1, result.capacity());
-    assertEquals((byte) 45, result.get());
-    result = inputStream.readByteBuffer();
-    assertTrue(result.array() == data);
-    assertEquals(bytesLength, result.capacity());
-    assertEquals((byte) 67, result.get());
-    result.position(bytesLength - 1);
-    assertEquals((byte) 89, result.get());
+    for (InputType inputType : InputType.values()) {
+      if (inputType == InputType.STREAM) {
+        // Aliasing doesn't apply to stream-backed CIS.
+        continue;
+      }
+
+      // Without aliasing
+      CodedInputStream inputStream = inputType.newDecoder(data);
+      ByteBuffer result = inputStream.readByteBuffer();
+      assertEquals(inputType.name(), 0, result.capacity());
+      result = inputStream.readByteBuffer();
+      assertTrue(inputType.name(), result.array() != data);
+      assertEquals(inputType.name(), 1, result.capacity());
+      assertEquals(inputType.name(), (byte) 23, result.get());
+      result = inputStream.readByteBuffer();
+      assertTrue(inputType.name(), result.array() != data);
+      assertEquals(inputType.name(), 1, result.capacity());
+      assertEquals(inputType.name(), (byte) 45, result.get());
+      result = inputStream.readByteBuffer();
+      assertTrue(inputType.name(), result.array() != data);
+      assertEquals(inputType.name(), bytesLength, result.capacity());
+      assertEquals(inputType.name(), (byte) 67, result.get());
+      result.position(bytesLength - 1);
+      assertEquals(inputType.name(), (byte) 89, result.get());
+
+      // Enable aliasing
+      inputStream = inputType.newDecoder(data);
+      inputStream.enableAliasing(true);
+      result = inputStream.readByteBuffer();
+      assertEquals(inputType.name(), 0, result.capacity());
+      result = inputStream.readByteBuffer();
+      if (result.hasArray()) {
+        assertTrue(inputType.name(), result.array() == data);
+      }
+      assertEquals(inputType.name(), 1, result.capacity());
+      assertEquals(inputType.name(), (byte) 23, result.get());
+      result = inputStream.readByteBuffer();
+      if (result.hasArray()) {
+        assertTrue(inputType.name(), result.array() == data);
+      }
+      assertEquals(inputType.name(), 1, result.capacity());
+      assertEquals(inputType.name(), (byte) 45, result.get());
+      result = inputStream.readByteBuffer();
+      if (result.hasArray()) {
+        assertTrue(inputType.name(), result.array() == data);
+      }
+      assertEquals(inputType.name(), bytesLength, result.capacity());
+      assertEquals(inputType.name(), (byte) 67, result.get());
+      result.position(bytesLength - 1);
+      assertEquals(inputType.name(), (byte) 89, result.get());
+    }
   }
 
   public void testCompatibleTypes() throws Exception {
     long data = 0x100000000L;
     Int64Message message = Int64Message.newBuilder().setData(data).build();
-    ByteString serialized = message.toByteString();
+    byte[] serialized = message.toByteArray();
+    for (InputType inputType : InputType.values()) {
+      CodedInputStream inputStream = inputType.newDecoder(serialized);
+
+      // Test int64(long) is compatible with bool(boolean)
+      BoolMessage msg2 = BoolMessage.parseFrom(inputStream);
+      assertTrue(msg2.getData());
+
+      // Test int64(long) is compatible with int32(int)
+      inputStream = inputType.newDecoder(serialized);
+      Int32Message msg3 = Int32Message.parseFrom(inputStream);
+      assertEquals((int) data, msg3.getData());
+    }
+  }
 
-    // Test int64(long) is compatible with bool(boolean)
-    BoolMessage msg2 = BoolMessage.parseFrom(serialized);
-    assertTrue(msg2.getData());
+  public void testSkipInvalidVarint_FastPath() throws Exception {
+    // Fast path: We have >= 10 bytes available. Ensure we properly recognize a non-ending varint.
+    byte[] data = new byte[] {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0};
+    for (InputType inputType : InputType.values()) {
+      try {
+        CodedInputStream input = inputType.newDecoder(data);
+        input.skipField(WireFormat.makeTag(1, WireFormat.WIRETYPE_VARINT));
+        fail(inputType.name() + ": Should have thrown an exception.");
+      } catch (InvalidProtocolBufferException e) {
+        // Expected
+      }
+    }
+  }
 
-    // Test int64(long) is compatible with int32(int)
-    Int32Message msg3 = Int32Message.parseFrom(serialized);
-    assertEquals((int) data, msg3.getData());
+  public void testSkipInvalidVarint_SlowPath() throws Exception {
+    // Slow path: < 10 bytes available. Ensure we properly recognize a non-ending varint.
+    byte[] data = new byte[] {-1, -1, -1, -1, -1, -1, -1, -1, -1};
+    for (InputType inputType : InputType.values()) {
+      try {
+        CodedInputStream input = inputType.newDecoder(data);
+        input.skipField(WireFormat.makeTag(1, WireFormat.WIRETYPE_VARINT));
+        fail(inputType.name() + ": Should have thrown an exception.");
+      } catch (InvalidProtocolBufferException e) {
+        // Expected
+      }
+    }
   }
 }

java/core/src/test/java/com/google/protobuf/CodedOutputStreamTest.java

@@ -35,15 +35,13 @@ import protobuf_unittest.UnittestProto.SparseEnumMessage;
 import protobuf_unittest.UnittestProto.TestAllTypes;
 import protobuf_unittest.UnittestProto.TestPackedTypes;
 import protobuf_unittest.UnittestProto.TestSparseEnum;
-
-import junit.framework.TestCase;
-
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import junit.framework.TestCase;
 
 /**
  * Unit test for {@link CodedOutputStream}.
@@ -151,16 +149,21 @@ public class CodedOutputStreamTest extends TestCase {
     private final int initialPosition;
     private final CodedOutputStream stream;
     private final ByteBuffer buffer;
+    private final boolean unsafe;
 
-    NioDirectCoder(int size) {
-      this(size, 0);
+    NioDirectCoder(int size, boolean unsafe) {
+      this(size, 0, unsafe);
     }
 
-    NioDirectCoder(int size, int initialPosition) {
+    NioDirectCoder(int size, int initialPosition, boolean unsafe) {
+      this.unsafe = unsafe;
       this.initialPosition = initialPosition;
       buffer = ByteBuffer.allocateDirect(size);
       buffer.position(initialPosition);
-      stream = CodedOutputStream.newInstance(buffer);
+      stream =
+          unsafe
+              ? CodedOutputStream.newUnsafeInstance(buffer)
+              : CodedOutputStream.newSafeInstance(buffer);
     }
 
     @Override
@@ -181,7 +184,7 @@ public class CodedOutputStreamTest extends TestCase {
 
     @Override
     public OutputType getOutputType() {
-      return OutputType.NIO_DIRECT;
+      return unsafe ? OutputType.NIO_DIRECT_SAFE : OutputType.NIO_DIRECT_UNSAFE;
     }
   }
 
@@ -198,10 +201,16 @@ public class CodedOutputStreamTest extends TestCase {
         return new NioHeapCoder(size);
       }
     },
-    NIO_DIRECT() {
+    NIO_DIRECT_SAFE() {
       @Override
       Coder newCoder(int size) {
-        return new NioDirectCoder(size);
+        return new NioDirectCoder(size, false);
+      }
+    },
+    NIO_DIRECT_UNSAFE() {
+      @Override
+      Coder newCoder(int size) {
+        return new NioDirectCoder(size, true);
       }
     },
     STREAM() {
@@ -389,6 +398,7 @@ public class CodedOutputStreamTest extends TestCase {
         != CodedOutputStream.computeUInt32SizeNoTag(string.length() * Utf8.MAX_BYTES_PER_CHAR));
 
     coder.stream().writeStringNoTag(string);
+    coder.stream().flush();
     int stringSize = CodedOutputStream.computeStringSizeNoTag(string);
 
     // Verify that the total bytes written is correct
@@ -478,11 +488,12 @@ public class CodedOutputStreamTest extends TestCase {
 
   public void testWriteByteArrayWithOffsets() throws Exception {
     byte[] fullArray = bytes(0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88);
-    byte[] destination = new byte[4];
-    CodedOutputStream codedStream = CodedOutputStream.newInstance(destination);
-    codedStream.writeByteArrayNoTag(fullArray, 2, 2);
-    assertEqualBytes(OutputType.ARRAY, bytes(0x02, 0x33, 0x44, 0x00), destination);
-    assertEquals(3, codedStream.getTotalBytesWritten());
+    for (OutputType type : new OutputType[] {OutputType.ARRAY}) {
+      Coder coder = type.newCoder(4);
+      coder.stream().writeByteArrayNoTag(fullArray, 2, 2);
+      assertEqualBytes(type, bytes(0x02, 0x33, 0x44), coder.toByteArray());
+      assertEquals(3, coder.stream().getTotalBytesWritten());
+    }
   }
 
   public void testSerializeUtf8_MultipleSmallWrites() throws Exception {
@@ -561,7 +572,12 @@ public class CodedOutputStreamTest extends TestCase {
     // Tag is one byte, varint describing string length is 1 byte, string length is 9 bytes.
     // An array of size 1 will cause a failure when trying to write the varint.
     for (OutputType outputType :
-        new OutputType[] {OutputType.ARRAY, OutputType.NIO_HEAP, OutputType.NIO_DIRECT}) {
+        new OutputType[] {
+          OutputType.ARRAY,
+          OutputType.NIO_HEAP,
+          OutputType.NIO_DIRECT_SAFE,
+          OutputType.NIO_DIRECT_UNSAFE
+        }) {
       for (int i = 0; i < 11; i++) {
         Coder coder = outputType.newCoder(i);
         try {
@@ -599,10 +615,13 @@ public class CodedOutputStreamTest extends TestCase {
 
   public void testNioEncodersWithInitialOffsets() throws Exception {
     String value = "abc";
-    for (Coder coder : new Coder[] {new NioHeapCoder(10, 2), new NioDirectCoder(10, 2)}) {
+    for (Coder coder :
+        new Coder[] {
+          new NioHeapCoder(10, 2), new NioDirectCoder(10, 2, false), new NioDirectCoder(10, 2, true)
+        }) {
       coder.stream().writeStringNoTag(value);
       coder.stream().flush();
-      assertEqualBytes(coder.getOutputType(), new byte[]{3, 'a', 'b', 'c'}, coder.toByteArray());
+      assertEqualBytes(coder.getOutputType(), new byte[] {3, 'a', 'b', 'c'}, coder.toByteArray());
     }
   }
 

java/core/src/test/java/com/google/protobuf/DeprecatedFieldTest.java

@@ -31,11 +31,9 @@
 package com.google.protobuf;
 
 import protobuf_unittest.UnittestProto.TestDeprecatedFields;
-
-import junit.framework.TestCase;
-
 import java.lang.reflect.AnnotatedElement;
 import java.lang.reflect.Method;
+import junit.framework.TestCase;
 
 /**
  * Test field deprecation

java/core/src/test/java/com/google/protobuf/DescriptorsTest.java

@@ -55,15 +55,15 @@ import protobuf_unittest.UnittestProto.ForeignMessage;
 import protobuf_unittest.UnittestProto.TestAllExtensions;
 import protobuf_unittest.UnittestProto.TestAllTypes;
 import protobuf_unittest.UnittestProto.TestExtremeDefaultValues;
+import protobuf_unittest.UnittestProto.TestJsonName;
 import protobuf_unittest.UnittestProto.TestMultipleExtensionRanges;
 import protobuf_unittest.UnittestProto.TestRequired;
 import protobuf_unittest.UnittestProto.TestReservedFields;
 import protobuf_unittest.UnittestProto.TestService;
-import junit.framework.TestCase;
-
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import junit.framework.TestCase;
 
 /**
  * Unit test for {@link Descriptors}.
@@ -805,4 +805,15 @@ public class DescriptorsTest extends TestCase {
     Descriptors.FileDescriptor.buildFrom(
         fileDescriptorProto, new FileDescriptor[0]);
   }
+
+  public void testFieldJsonName() throws Exception {
+    Descriptor d = TestJsonName.getDescriptor();
+    assertEquals(6, d.getFields().size());
+    assertEquals("fieldName1", d.getFields().get(0).getJsonName());
+    assertEquals("fieldName2", d.getFields().get(1).getJsonName());
+    assertEquals("FieldName3", d.getFields().get(2).getJsonName());
+    assertEquals("FieldName4", d.getFields().get(3).getJsonName());
+    assertEquals("FIELDNAME5", d.getFields().get(4).getJsonName());
+    assertEquals("@type", d.getFields().get(5).getJsonName());
+  }
 }

java/core/src/test/java/com/google/protobuf/DoubleArrayListTest.java

@@ -32,11 +32,10 @@ package com.google.protobuf;
 
 import static java.util.Arrays.asList;
 
-import junit.framework.TestCase;
-
 import java.util.Collections;
 import java.util.ConcurrentModificationException;
 import java.util.Iterator;
+import junit.framework.TestCase;
 
 /**
  * Tests for {@link DoubleArrayList}.

java/core/src/test/java/com/google/protobuf/DynamicMessageTest.java

@@ -37,10 +37,8 @@ import protobuf_unittest.UnittestProto.TestAllExtensions;
 import protobuf_unittest.UnittestProto.TestAllTypes;
 import protobuf_unittest.UnittestProto.TestEmptyMessage;
 import protobuf_unittest.UnittestProto.TestPackedTypes;
-
-import junit.framework.TestCase;
-
 import java.util.Arrays;
+import junit.framework.TestCase;
 
 /**
  * Unit test for {@link DynamicMessage}.  See also {@link MessageTest}, which

java/core/src/test/java/com/google/protobuf/ExtensionRegistryFactoryTest.java

@@ -32,26 +32,24 @@ package com.google.protobuf;
 
 import protobuf_unittest.NonNestedExtension;
 import protobuf_unittest.NonNestedExtensionLite;
-
-import junit.framework.Test;
-import junit.framework.TestCase;
-import junit.framework.TestSuite;
-
 import java.lang.reflect.Method;
 import java.net.URLClassLoader;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
+import junit.framework.Test;
+import junit.framework.TestCase;
+import junit.framework.TestSuite;
 
 /**
  * Tests for {@link ExtensionRegistryFactory} and the {@link ExtensionRegistry} instances it
  * creates.
- * 
+ *
  * <p>This test simulates the runtime behaviour of the ExtensionRegistryFactory by delegating test
  * definitions to two inner classes {@link InnerTest} and {@link InnerLiteTest}, the latter of
  * which is executed using a custom ClassLoader, simulating the ProtoLite environment.
- * 
+ *
  * <p>The test mechanism employed here is based on the pattern in
  * {@code com.google.common.util.concurrent.AbstractFutureFallbackAtomicHelperTest}
  */
@@ -68,6 +66,7 @@ public class ExtensionRegistryFactoryTest extends TestCase {
     void testEmpty();
     void testIsFullRegistry();
     void testAdd();
+    void testAdd_immutable();
   }
 
   /**
@@ -125,6 +124,29 @@ public class ExtensionRegistryFactoryTest extends TestCase {
       assertNotNull("Extension is registered in masqueraded full registry",
           fullRegistry2.findImmutableExtensionByName("protobuf_unittest.nonNestedExtension"));
     }
+
+    @Override
+    public void testAdd_immutable() {
+      ExtensionRegistryLite registry1 = ExtensionRegistryLite.newInstance().getUnmodifiable();
+      try {
+        NonNestedExtensionLite.registerAllExtensions(registry1);
+        fail();
+      } catch (UnsupportedOperationException expected) {}
+      try {
+        registry1.add(NonNestedExtensionLite.nonNestedExtensionLite);
+        fail();
+      } catch (UnsupportedOperationException expected) {}
+
+      ExtensionRegistryLite registry2 = ExtensionRegistryLite.newInstance().getUnmodifiable();
+      try {
+        NonNestedExtension.registerAllExtensions((ExtensionRegistry) registry2);
+        fail();
+      } catch (IllegalArgumentException expected) {}
+      try {
+        registry2.add(NonNestedExtension.nonNestedExtension);
+        fail();
+      } catch (IllegalArgumentException expected) {}
+    }
   }
 
   /**
@@ -162,12 +184,21 @@ public class ExtensionRegistryFactoryTest extends TestCase {
               NonNestedExtensionLite.MessageLiteToBeExtended.getDefaultInstance(), 1);
       assertNotNull("Extension is registered in Lite registry", extension);
     }
+
+    @Override
+    public void testAdd_immutable() {
+      ExtensionRegistryLite registry = ExtensionRegistryLite.newInstance().getUnmodifiable();
+      try {
+        NonNestedExtensionLite.registerAllExtensions(registry);
+        fail();
+      } catch (UnsupportedOperationException expected) {}
+    }
   }
 
   /**
    * Defines a suite of tests which the JUnit3 runner retrieves by reflection.
    */
-  public static Test suite() { 
+  public static Test suite() {
     TestSuite suite = new TestSuite();
     for (Method method : RegistryTests.class.getMethods()) {
       suite.addTest(TestSuite.createTest(ExtensionRegistryFactoryTest.class, method.getName()));

java/core/src/test/java/com/google/protobuf/FloatArrayListTest.java

@@ -32,11 +32,10 @@ package com.google.protobuf;
 
 import static java.util.Arrays.asList;
 
-import junit.framework.TestCase;
-
 import java.util.Collections;
 import java.util.ConcurrentModificationException;
 import java.util.Iterator;
+import junit.framework.TestCase;
 
 /**
  * Tests for {@link FloatArrayList}.

java/core/src/test/java/com/google/protobuf/GeneratedMessageTest.java

@@ -65,9 +65,6 @@ import protobuf_unittest.UnittestProto.TestExtremeDefaultValues;
 import protobuf_unittest.UnittestProto.TestOneof2;
 import protobuf_unittest.UnittestProto.TestPackedTypes;
 import protobuf_unittest.UnittestProto.TestUnpackedTypes;
-
-import junit.framework.TestCase;
-
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.ObjectInputStream;
@@ -76,6 +73,7 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
+import junit.framework.TestCase;
 
 /**
  * Unit test for generated messages and generated code.  See also

java/core/src/test/java/com/google/protobuf/IntArrayListTest.java

@@ -32,11 +32,10 @@ package com.google.protobuf;
 
 import static java.util.Arrays.asList;
 
-import junit.framework.TestCase;
-
 import java.util.Collections;
 import java.util.ConcurrentModificationException;
 import java.util.Iterator;
+import junit.framework.TestCase;
 
 /**
  * Tests for {@link IntArrayList}.

java/core/src/test/java/com/google/protobuf/LazyFieldLiteTest.java

@@ -35,10 +35,8 @@ import static protobuf_unittest.UnittestProto.optionalInt64Extension;
 
 import protobuf_unittest.UnittestProto.TestAllExtensions;
 import protobuf_unittest.UnittestProto.TestAllTypes;
-
-import junit.framework.TestCase;
-
 import java.io.IOException;
+import junit.framework.TestCase;
 
 /**
  * Unit test for {@link LazyFieldLite}.

java/core/src/test/java/com/google/protobuf/LazyFieldTest.java

@@ -32,10 +32,8 @@ package com.google.protobuf;
 
 import protobuf_unittest.UnittestProto.TestAllExtensions;
 import protobuf_unittest.UnittestProto.TestAllTypes;
-
-import junit.framework.TestCase;
-
 import java.io.IOException;
+import junit.framework.TestCase;
 
 /**
  * Unit test for {@link LazyField}.

java/core/src/test/java/com/google/protobuf/LazyMessageLiteTest.java

@@ -34,10 +34,8 @@ import protobuf_unittest.LazyFieldsLite.LazyExtension;
 import protobuf_unittest.LazyFieldsLite.LazyInnerMessageLite;
 import protobuf_unittest.LazyFieldsLite.LazyMessageLite;
 import protobuf_unittest.LazyFieldsLite.LazyNestedInnerMessageLite;
-
-import junit.framework.TestCase;
-
 import java.util.ArrayList;
+import junit.framework.TestCase;
 
 /**
  * Unit test for messages with lazy fields.

java/core/src/test/java/com/google/protobuf/LazyStringArrayListTest.java

@@ -32,13 +32,12 @@ package com.google.protobuf;
 
 import static java.util.Arrays.asList;
 
-import junit.framework.TestCase;
-
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.ConcurrentModificationException;
 import java.util.Iterator;
 import java.util.List;
+import junit.framework.TestCase;
 
 /**
  * Tests for {@link LazyStringArrayList}.

java/core/src/test/java/com/google/protobuf/LazyStringEndToEndTest.java

@@ -32,10 +32,8 @@ package com.google.protobuf;
 
 
 import protobuf_unittest.UnittestProto;
-
-import junit.framework.TestCase;
-
 import java.io.IOException;
+import junit.framework.TestCase;
 
 /**
  * Tests to make sure the lazy conversion of UTF8-encoded byte arrays to

java/core/src/test/java/com/google/protobuf/LiteTest.java

@@ -35,7 +35,6 @@ import static java.util.Collections.singletonList;
 
 import com.google.protobuf.UnittestImportLite.ImportEnumLite;
 import com.google.protobuf.UnittestImportPublicLite.PublicImportMessageLite;
-import com.google.protobuf.UnittestLite;
 import com.google.protobuf.UnittestLite.ForeignEnumLite;
 import com.google.protobuf.UnittestLite.ForeignMessageLite;
 import com.google.protobuf.UnittestLite.TestAllExtensionsLite;
@@ -52,15 +51,8 @@ import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.BarPrime;
 import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.Foo;
 import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.TestOneofEquals;
 import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.TestRecursiveOneof;
-
 import junit.framework.TestCase;
 
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.NotSerializableException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-
 /**
  * Test lite runtime.
  *
@@ -139,7 +131,6 @@ public class LiteTest extends TestCase {
         UnittestLite.optionalNestedMessageExtensionLite).getBb());
   }
   
-  
   public void testClone() {
     TestAllTypesLite.Builder expected = TestAllTypesLite.newBuilder()
         .setOptionalInt32(123);

java/core/src/test/java/com/google/protobuf/LiteralByteStringTest.java

@@ -30,8 +30,6 @@
 
 package com.google.protobuf;
 
-import junit.framework.TestCase;
-
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.EOFException;
@@ -45,6 +43,7 @@ import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.List;
 import java.util.NoSuchElementException;
+import junit.framework.TestCase;
 
 /**
  * Test {@code LiteralByteString} by setting up a reference string in {@link #setUp()}.

java/core/src/test/java/com/google/protobuf/LongArrayListTest.java

@@ -32,11 +32,10 @@ package com.google.protobuf;
 
 import static java.util.Arrays.asList;
 
-import junit.framework.TestCase;
-
 import java.util.Collections;
 import java.util.ConcurrentModificationException;
 import java.util.Iterator;
+import junit.framework.TestCase;
 
 /**
  * Tests for {@link LongArrayList}.

java/core/src/test/java/com/google/protobuf/MapForProto2LiteTest.java

@@ -35,15 +35,13 @@ import map_lite_test.MapForProto2TestProto.TestMap;
 import map_lite_test.MapForProto2TestProto.TestMap.MessageValue;
 import map_lite_test.MapForProto2TestProto.TestMapOrBuilder;
 import map_lite_test.MapForProto2TestProto.TestUnknownEnumValue;
-
-import junit.framework.TestCase;
-
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
+import junit.framework.TestCase;
 
 /**
  * Unit tests for map fields.

java/core/src/test/java/com/google/protobuf/MapForProto2Test.java

@@ -32,14 +32,14 @@ package com.google.protobuf;
 
 import com.google.protobuf.Descriptors.FieldDescriptor;
 import map_test.MapForProto2TestProto.BizarroTestMap;
+import map_test.MapForProto2TestProto.ReservedAsMapField;
+import map_test.MapForProto2TestProto.ReservedAsMapFieldWithEnumValue;
 import map_test.MapForProto2TestProto.TestMap;
 import map_test.MapForProto2TestProto.TestMap.MessageValue;
 import map_test.MapForProto2TestProto.TestMap.MessageWithRequiredFields;
 import map_test.MapForProto2TestProto.TestMapOrBuilder;
 import map_test.MapForProto2TestProto.TestRecursiveMap;
 import map_test.MapForProto2TestProto.TestUnknownEnumValue;
-import junit.framework.TestCase;
-
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -47,6 +47,7 @@ import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import junit.framework.TestCase;
 
 /**
  * Unit tests for map fields in proto2 protos.
@@ -1143,6 +1144,11 @@ public class MapForProto2Test extends TestCase {
     assertEquals(2, mapEntry.getAllFields().size());
   }
 
+  public void testReservedWordsFieldNames() {
+    ReservedAsMapField.newBuilder().build();
+    ReservedAsMapFieldWithEnumValue.newBuilder().build();
+  }
+
   private static <K, V> Map<K, V> newMap(K key1, V value1) {
     Map<K, V> map = new HashMap<K, V>();
     map.put(key1, value1);

java/core/src/test/java/com/google/protobuf/MapTest.java

@@ -36,12 +36,12 @@ import com.google.protobuf.Descriptors.EnumDescriptor;
 import com.google.protobuf.Descriptors.EnumValueDescriptor;
 import com.google.protobuf.Descriptors.FieldDescriptor;
 import map_test.MapTestProto.BizarroTestMap;
+import map_test.MapTestProto.ReservedAsMapField;
+import map_test.MapTestProto.ReservedAsMapFieldWithEnumValue;
 import map_test.MapTestProto.TestMap;
 import map_test.MapTestProto.TestMap.MessageValue;
 import map_test.MapTestProto.TestMapOrBuilder;
 import map_test.MapTestProto.TestOnChangeEventPropagation;
-import junit.framework.TestCase;
-
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -49,6 +49,7 @@ import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import junit.framework.TestCase;
 
 /**
  * Unit tests for map fields.
@@ -1307,6 +1308,171 @@ public class MapTest extends TestCase {
     }
   }
 
+  public void testReservedWordsFieldNames() {
+    ReservedAsMapField.newBuilder().build();
+    ReservedAsMapFieldWithEnumValue.newBuilder().build();
+  }
+
+  public void testDeterministicSerialziation() throws Exception {
+    TestMap.Builder builder = TestMap.newBuilder();
+    // int32->int32
+    builder.putInt32ToInt32Field(5, 1);
+    builder.putInt32ToInt32Field(1, 1);
+    builder.putInt32ToInt32Field(4, 1);
+    builder.putInt32ToInt32Field(-2, 1);
+    builder.putInt32ToInt32Field(0, 1);
+
+    // uint32->int32
+    builder.putUint32ToInt32Field(5, 1);
+    builder.putUint32ToInt32Field(1, 1);
+    builder.putUint32ToInt32Field(4, 1);
+    builder.putUint32ToInt32Field(-2, 1);
+    builder.putUint32ToInt32Field(0, 1);
+
+    // int64->int32
+    builder.putInt64ToInt32Field(5L, 1);
+    builder.putInt64ToInt32Field(1L, 1);
+    builder.putInt64ToInt32Field(4L, 1);
+    builder.putInt64ToInt32Field(-2L, 1);
+    builder.putInt64ToInt32Field(0L, 1);
+
+    // string->int32
+    builder.putStringToInt32Field("baz", 1);
+    builder.putStringToInt32Field("foo", 1);
+    builder.putStringToInt32Field("bar", 1);
+    builder.putStringToInt32Field("", 1);
+    builder.putStringToInt32Field("hello", 1);
+    builder.putStringToInt32Field("world", 1);
+
+    TestMap message = builder.build();
+    byte[] serialized = new byte[message.getSerializedSize()];
+    CodedOutputStream output = CodedOutputStream.newInstance(serialized);
+    output.useDeterministicSerialization();
+    message.writeTo(output);
+    output.flush();
+
+    CodedInputStream input = CodedInputStream.newInstance(serialized);
+    List<Integer> int32Keys = new ArrayList<Integer>();
+    List<Integer> uint32Keys = new ArrayList<Integer>();
+    List<Long> int64Keys = new ArrayList<Long>();
+    List<String> stringKeys = new ArrayList<String>();
+    int tag;
+    while (true) {
+      tag = input.readTag();
+      if (tag == 0) {
+        break;
+      }
+      int length = input.readRawVarint32();
+      int oldLimit = input.pushLimit(length);
+      switch (WireFormat.getTagFieldNumber(tag)) {
+        case TestMap.STRING_TO_INT32_FIELD_FIELD_NUMBER:
+          stringKeys.add(readMapStringKey(input));
+          break;
+        case TestMap.INT32_TO_INT32_FIELD_FIELD_NUMBER:
+          int32Keys.add(readMapIntegerKey(input));
+          break;
+        case TestMap.UINT32_TO_INT32_FIELD_FIELD_NUMBER:
+          uint32Keys.add(readMapIntegerKey(input));
+          break;
+        case TestMap.INT64_TO_INT32_FIELD_FIELD_NUMBER:
+          int64Keys.add(readMapLongKey(input));
+          break;
+        default:
+          fail("Unexpected fields.");
+      }
+      input.popLimit(oldLimit);
+    }
+    assertEquals(
+        Arrays.asList(-2, 0, 1, 4, 5),
+        int32Keys);
+    assertEquals(
+        Arrays.asList(-2, 0, 1, 4, 5),
+        uint32Keys);
+    assertEquals(
+        Arrays.asList(-2L, 0L, 1L, 4L, 5L),
+        int64Keys);
+    assertEquals(
+        Arrays.asList("", "bar", "baz", "foo", "hello", "world"),
+        stringKeys);
+  }
+
+  public void testInitFromPartialDynamicMessage() {
+    FieldDescriptor fieldDescriptor =
+        TestMap.getDescriptor().findFieldByNumber(TestMap.INT32_TO_MESSAGE_FIELD_FIELD_NUMBER);
+    Descriptor mapEntryType = fieldDescriptor.getMessageType();
+    FieldDescriptor keyField = mapEntryType.findFieldByNumber(1);
+    FieldDescriptor valueField = mapEntryType.findFieldByNumber(2);
+    DynamicMessage dynamicMessage =
+        DynamicMessage.newBuilder(TestMap.getDescriptor())
+            .addRepeatedField(
+                fieldDescriptor,
+                DynamicMessage.newBuilder(mapEntryType)
+                    .setField(keyField, 10)
+                    .setField(valueField, TestMap.MessageValue.newBuilder().setValue(10).build())
+                    .build())
+            .build();
+    TestMap message = TestMap.newBuilder().mergeFrom(dynamicMessage).build();
+    assertEquals(
+        TestMap.MessageValue.newBuilder().setValue(10).build(),
+        message.getInt32ToMessageFieldMap().get(10));
+  }
+
+  public void testInitFromFullyDynamicMessage() {
+    FieldDescriptor fieldDescriptor =
+        TestMap.getDescriptor().findFieldByNumber(TestMap.INT32_TO_MESSAGE_FIELD_FIELD_NUMBER);
+    Descriptor mapEntryType = fieldDescriptor.getMessageType();
+    FieldDescriptor keyField = mapEntryType.findFieldByNumber(1);
+    FieldDescriptor valueField = mapEntryType.findFieldByNumber(2);
+    DynamicMessage dynamicMessage =
+        DynamicMessage.newBuilder(TestMap.getDescriptor())
+            .addRepeatedField(
+                fieldDescriptor,
+                DynamicMessage.newBuilder(mapEntryType)
+                    .setField(keyField, 10)
+                    .setField(
+                        valueField,
+                        DynamicMessage.newBuilder(TestMap.MessageValue.getDescriptor())
+                            .setField(
+                                TestMap.MessageValue.getDescriptor().findFieldByName("value"), 10)
+                            .build())
+                    .build())
+            .build();
+    TestMap message = TestMap.newBuilder().mergeFrom(dynamicMessage).build();
+    assertEquals(
+        TestMap.MessageValue.newBuilder().setValue(10).build(),
+        message.getInt32ToMessageFieldMap().get(10));
+  }
+
+  private int readMapIntegerKey(CodedInputStream input) throws IOException {
+    int tag = input.readTag();
+    assertEquals(WireFormat.makeTag(1, WireFormat.WIRETYPE_VARINT), tag);
+    int ret = input.readInt32();
+    // skip the value field.
+    input.skipField(input.readTag());
+    assertTrue(input.isAtEnd());
+    return ret;
+  }
+
+  private long readMapLongKey(CodedInputStream input) throws IOException {
+    int tag = input.readTag();
+    assertEquals(WireFormat.makeTag(1, WireFormat.WIRETYPE_VARINT), tag);
+    long ret = input.readInt64();
+    // skip the value field.
+    input.skipField(input.readTag());
+    assertTrue(input.isAtEnd());
+    return ret;
+  }
+
+  private String readMapStringKey(CodedInputStream input) throws IOException {
+    int tag = input.readTag();
+    assertEquals(WireFormat.makeTag(1, WireFormat.WIRETYPE_LENGTH_DELIMITED), tag);
+    String ret = input.readString();
+    // skip the value field.
+    input.skipField(input.readTag());
+    assertTrue(input.isAtEnd());
+    return ret;
+  }
+
   private static <K, V> Map<K, V> newMap(K key1, V value1) {
     Map<K, V> map = new HashMap<K, V>();
     map.put(key1, value1);

java/core/src/test/java/com/google/protobuf/MessageTest.java

@@ -35,10 +35,8 @@ import protobuf_unittest.UnittestProto.TestAllExtensions;
 import protobuf_unittest.UnittestProto.TestAllTypes;
 import protobuf_unittest.UnittestProto.TestRequired;
 import protobuf_unittest.UnittestProto.TestRequiredForeign;
-
-import junit.framework.TestCase;
-
 import java.util.List;
+import junit.framework.TestCase;
 
 /**
  * Misc. unit tests for message operations that apply to both generated

java/core/src/test/java/com/google/protobuf/NestedBuildersTest.java

@@ -32,11 +32,9 @@ package com.google.protobuf;
 
 import protobuf_unittest.Vehicle;
 import protobuf_unittest.Wheel;
-
-import junit.framework.TestCase;
-
 import java.util.ArrayList;
 import java.util.List;
+import junit.framework.TestCase;
 
 /**
  * Test cases that exercise end-to-end use cases involving

java/core/src/test/java/com/google/protobuf/NioByteStringTest.java

@@ -32,8 +32,6 @@ package com.google.protobuf;
 
 import static com.google.protobuf.Internal.UTF_8;
 
-import junit.framework.TestCase;
-
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.EOFException;
@@ -48,6 +46,7 @@ import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.List;
 import java.util.NoSuchElementException;
+import junit.framework.TestCase;
 
 /**
  * Tests for {@link NioByteString}.

java/core/src/test/java/com/google/protobuf/ParseExceptionsTest.java

@@ -36,16 +36,14 @@ import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 import com.google.protobuf.DescriptorProtos.DescriptorProto;
-
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.FilterInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
 
 /**
  * Tests the exceptions thrown when parsing from a stream. The methods on the {@link Parser}

java/core/src/test/java/com/google/protobuf/ParserTest.java

@@ -42,13 +42,11 @@ import protobuf_unittest.UnittestProto.TestAllTypes;
 import protobuf_unittest.UnittestProto.TestEmptyMessage;
 import protobuf_unittest.UnittestProto.TestParsingMerge;
 import protobuf_unittest.UnittestProto.TestRequired;
-
-import junit.framework.TestCase;
-
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import junit.framework.TestCase;
 
 /**
  * Unit test for {@link Parser}.

java/core/src/test/java/com/google/protobuf/ProtobufArrayListTest.java

@@ -32,12 +32,11 @@ package com.google.protobuf;
 
 import static java.util.Arrays.asList;
 
-import junit.framework.TestCase;
-
 import java.util.Collections;
 import java.util.ConcurrentModificationException;
 import java.util.Iterator;
 import java.util.List;
+import junit.framework.TestCase;
 
 /**
  * Tests for {@link ProtobufArrayList}.

java/core/src/test/java/com/google/protobuf/RepeatedFieldBuilderV3Test.java

@@ -32,11 +32,9 @@ package com.google.protobuf;
 
 import protobuf_unittest.UnittestProto.TestAllTypes;
 import protobuf_unittest.UnittestProto.TestAllTypesOrBuilder;
-
-import junit.framework.TestCase;
-
 import java.util.Collections;
 import java.util.List;
+import junit.framework.TestCase;
 
 /**
  * Tests for {@link RepeatedFieldBuilderV3}. This tests basic functionality.

java/core/src/test/java/com/google/protobuf/ServiceTest.java

@@ -42,15 +42,13 @@ import protobuf_unittest.UnittestProto.FooResponse;
 import protobuf_unittest.UnittestProto.TestAllTypes;
 import protobuf_unittest.UnittestProto.TestService;
 
+import java.util.HashSet;
+import java.util.Set;
 import junit.framework.TestCase;
-
 import org.easymock.classextension.EasyMock;
 import org.easymock.IArgumentMatcher;
 import org.easymock.classextension.IMocksControl;
 
-import java.util.HashSet;
-import java.util.Set;
-
 /**
  * Tests services and stubs.
  *
@@ -271,6 +269,8 @@ public class ServiceTest extends TestCase {
         file.getServices().get(0).getMethods().get(0).getName());
   }
 
+
+
   // =================================================================
 
   /**

java/core/src/test/java/com/google/protobuf/SmallSortedMapTest.java

@@ -30,8 +30,6 @@
 
 package com.google.protobuf;
 
-import junit.framework.TestCase;
-
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -40,6 +38,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
+import junit.framework.TestCase;
 
 /**
  * @author darick@google.com Darick Tong

java/core/src/test/java/com/google/protobuf/TestUtil.java

@@ -232,12 +232,10 @@ import protobuf_unittest.UnittestProto.TestOneof2;
 import protobuf_unittest.UnittestProto.TestPackedExtensions;
 import protobuf_unittest.UnittestProto.TestPackedTypes;
 import protobuf_unittest.UnittestProto.TestUnpackedTypes;
-
-import junit.framework.Assert;
-
 import java.io.File;
 import java.io.IOException;
 import java.io.RandomAccessFile;
+import junit.framework.Assert;
 
 /**
  * Contains methods for setting all fields of {@code TestAllTypes} to
@@ -259,6 +257,13 @@ public final class TestUtil {
     return ByteString.copyFrom(str.getBytes(Internal.UTF_8));
   }
 
+  /**
+   * Dirties the message by resetting the momoized serialized size.
+   */
+  public static void resetMemoizedSize(AbstractMessage message) {
+    message.memoizedSize = -1;
+  }
+
   /**
    * Get a {@code TestAllTypes} with all fields set as they would be by
    * {@link #setAllFields(TestAllTypes.Builder)}.

java/core/src/test/java/com/google/protobuf/TextFormatTest.java

@@ -42,11 +42,9 @@ import protobuf_unittest.UnittestProto.TestAllTypes.NestedMessage;
 import protobuf_unittest.UnittestProto.TestEmptyMessage;
 import protobuf_unittest.UnittestProto.TestOneof2;
 import proto2_wireformat_unittest.UnittestMsetWireFormat.TestMessageSet;
-
-import junit.framework.TestCase;
-
 import java.io.StringReader;
 import java.util.List;
+import junit.framework.TestCase;
 
 /**
  * Test case for {@link TextFormat}.
@@ -992,10 +990,35 @@ public class TextFormatTest extends TestCase {
     assertParseSuccessWithOverwriteForbidden("repeated_nested_message [{ bb: 1 }, { bb: 2 }]\n");
   }
 
+  public void testParseShortRepeatedFormOfEmptyRepeatedFields() throws Exception {
+    assertParseSuccessWithOverwriteForbidden("repeated_foreign_enum: []");
+    assertParseSuccessWithOverwriteForbidden("repeated_int32: []\n");
+    assertParseSuccessWithOverwriteForbidden("RepeatedGroup []\n");
+    assertParseSuccessWithOverwriteForbidden("repeated_nested_message []\n");
+  }
+
+  public void testParseShortRepeatedFormWithTrailingComma() throws Exception {
+    assertParseErrorWithOverwriteForbidden(
+        "1:38: Expected identifier. Found \']\'",
+        "repeated_foreign_enum: [FOREIGN_FOO, ]\n");
+    assertParseErrorWithOverwriteForbidden(
+        "1:22: Couldn't parse integer: For input string: \"]\"",
+        "repeated_int32: [ 1, ]\n");
+    assertParseErrorWithOverwriteForbidden(
+        "1:25: Expected \"{\".",
+        "RepeatedGroup [{ a: 1 },]\n");
+    assertParseErrorWithOverwriteForbidden(
+        "1:37: Expected \"{\".",
+        "repeated_nested_message [{ bb: 1 }, ]\n");
+  }
+
   public void testParseShortRepeatedFormOfNonRepeatedFields() throws Exception {
     assertParseErrorWithOverwriteForbidden(
         "1:17: Couldn't parse integer: For input string: \"[\"",
         "optional_int32: [1]\n");
+    assertParseErrorWithOverwriteForbidden(
+        "1:17: Couldn't parse integer: For input string: \"[\"",
+        "optional_int32: []\n");
   }
 
   // =======================================================================

java/core/src/test/java/com/google/protobuf/UnknownFieldSetLiteTest.java

@@ -35,11 +35,9 @@ import com.google.protobuf.UnittestLite.TestAllTypesLite;
 import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash;
 import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.Bar;
 import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.Foo;
-
-import junit.framework.TestCase;
-
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import junit.framework.TestCase;
 
 /**
  * Tests for {@link UnknownFieldSetLite}.

java/core/src/test/java/com/google/protobuf/UnknownFieldSetTest.java

@@ -38,11 +38,9 @@ import protobuf_unittest.UnittestProto.TestEmptyMessage;
 import protobuf_unittest.UnittestProto.TestEmptyMessageWithExtensions;
 import protobuf_unittest.UnittestProto.TestPackedExtensions;
 import protobuf_unittest.UnittestProto.TestPackedTypes;
-
-import junit.framework.TestCase;
-
 import java.util.Arrays;
 import java.util.Map;
+import junit.framework.TestCase;
 
 /**
  * Tests related to unknown field handling.

java/core/src/test/java/com/google/protobuf/UnmodifiableLazyStringListTest.java

@@ -30,11 +30,10 @@
 
 package com.google.protobuf;
 
-import junit.framework.TestCase;
-
 import java.util.Iterator;
 import java.util.List;
 import java.util.ListIterator;
+import junit.framework.TestCase;
 
 /**
  * Tests for {@link UnmodifiableLazyStringList}.

java/core/src/test/java/com/google/protobuf/WireFormatTest.java

@@ -44,12 +44,10 @@ import protobuf_unittest.UnittestProto.TestOneofBackwardsCompatible;
 import protobuf_unittest.UnittestProto.TestPackedExtensions;
 import protobuf_unittest.UnittestProto.TestPackedTypes;
 import proto2_wireformat_unittest.UnittestMsetWireFormat.TestMessageSet;
-
-import junit.framework.TestCase;
-
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.util.List;
+import junit.framework.TestCase;
 
 /**
  * Tests related to parsing and serialization.

java/core/src/test/proto/com/google/protobuf/field_presence_test.proto

@@ -36,7 +36,6 @@ import "google/protobuf/unittest.proto";
 
 option java_package = "com.google.protobuf";
 option java_outer_classname = "FieldPresenceTestProto";
-option java_generate_equals_and_hash = true;
 
 message TestAllTypes {
   enum NestedEnum {

java/core/src/test/proto/com/google/protobuf/map_for_proto2_lite_test.proto

@@ -32,7 +32,6 @@ syntax = "proto2";
 
 
 option java_outer_classname = "MapForProto2TestProto";
-option java_generate_equals_and_hash = true;
 
 message TestMap {
   message MessageValue {
@@ -81,6 +80,42 @@ message BizarroTestMap {
   map<string, bytes> int32_to_message_field = 5; // different key and value types
   map<string, bytes> string_to_int32_field = 6;  // same key type, different value
 }
+
+// Used to test that java reserved words can be used as protobuf field names
+// Not all reserved words are tested (to avoid bloat) but instead an arbitrary
+// subset of them chosen to cover various keyword categories like
+// type, modifier, declaration, etc.
+message ReservedAsMapField {
+  map<string, uint32> if = 1;
+  map<string, uint32> const = 2;
+  map<string, uint32> private = 3;
+  map<string, uint32> class = 4;
+  map<string, uint32> int = 5;
+  map<string, uint32> void = 6;
+  map<string, uint32> string = 7; // These are also proto keywords
+  map<string, uint32> package = 8;
+  map<string, uint32> enum = 9; // Most recent Java reserved word
+  map<string, uint32> null = 10;
+  // null is not a 'reserved word' per se but as a literal needs similar care
+}
+
+message ReservedAsMapFieldWithEnumValue {
+  enum SampleEnum {
+    A = 0;
+    B = 1;
+  }
+  map<string, SampleEnum> if = 1;
+  map<string, SampleEnum> const = 2;
+  map<string, SampleEnum> private = 3;
+  map<string, SampleEnum> class = 4;
+  map<string, SampleEnum> int = 5;
+  map<string, SampleEnum> void = 6;
+  map<string, SampleEnum> string = 7; // These are also proto keywords
+  map<string, SampleEnum> package = 8;
+  map<string, SampleEnum> enum = 9; // Most recent Java reserved word
+  map<string, SampleEnum> null = 10;
+  // null is not a 'reserved word' per se but as a literal needs similar care
+}
 package map_for_proto2_lite_test;
 option java_package = "map_lite_test";
 option optimize_for = LITE_RUNTIME;

java/core/src/test/proto/com/google/protobuf/map_for_proto2_test.proto

@@ -34,7 +34,6 @@ package map_for_proto2_test;
 
 option java_package = "map_test";
 option java_outer_classname = "MapForProto2TestProto";
-option java_generate_equals_and_hash = true;
 
 message TestMap {
   message MessageValue {
@@ -83,3 +82,39 @@ message BizarroTestMap {
   map<string, bytes> int32_to_message_field = 5; // different key and value types
   map<string, bytes> string_to_int32_field = 6;  // same key type, different value
 }
+
+// Used to test that java reserved words can be used as protobuf field names
+// Not all reserved words are tested (to avoid bloat) but instead an arbitrary
+// subset of them chosen to cover various keyword categories like
+// type, modifier, declaration, etc.
+message ReservedAsMapField {
+  map<string, uint32> if = 1;
+  map<string, uint32> const = 2;
+  map<string, uint32> private = 3;
+  map<string, uint32> class = 4;
+  map<string, uint32> int = 5;
+  map<string, uint32> void = 6;
+  map<string, uint32> string = 7; // These are also proto keywords
+  map<string, uint32> package = 8;
+  map<string, uint32> enum = 9; // Most recent Java reserved word
+  map<string, uint32> null = 10;
+  // null is not a 'reserved word' per se but as a literal needs similar care
+}
+
+message ReservedAsMapFieldWithEnumValue {
+  enum SampleEnum {
+    A = 0;
+    B = 1;
+  }
+  map<string, SampleEnum> if = 1;
+  map<string, SampleEnum> const = 2;
+  map<string, SampleEnum> private = 3;
+  map<string, SampleEnum> class = 4;
+  map<string, SampleEnum> int = 5;
+  map<string, SampleEnum> void = 6;
+  map<string, SampleEnum> string = 7; // These are also proto keywords
+  map<string, SampleEnum> package = 8;
+  map<string, SampleEnum> enum = 9; // Most recent Java reserved word
+  map<string, SampleEnum> null = 10;
+  // null is not a 'reserved word' per se but as a literal needs similar care
+}

java/core/src/test/proto/com/google/protobuf/map_test.proto

@@ -34,7 +34,6 @@ package map_test;
 
 option java_package = "map_test";
 option java_outer_classname = "MapTestProto";
-option java_generate_equals_and_hash = true;
 
 message TestMap {
   message MessageValue {
@@ -53,6 +52,8 @@ message TestMap {
   map<int32, EnumValue>    int32_to_enum_field = 4;
   map<int32, MessageValue> int32_to_message_field = 5;
   map<string, int32>       string_to_int32_field = 6;
+  map<uint32, int32>       uint32_to_int32_field = 7;
+  map<int64, int32>        int64_to_int32_field = 8;
 }
 
 // Used to test that a nested builder containing map fields will properly
@@ -71,3 +72,39 @@ message BizarroTestMap {
   map<string, bytes> int32_to_message_field = 5; // different key and value types
   map<string, bytes> string_to_int32_field = 6;  // same key type, different value
 }
+
+// Used to test that java reserved words can be used as protobuf field names
+// Not all reserved words are tested (to avoid bloat) but instead an arbitrary
+// subset of them chosen to cover various keyword categories like
+// type, modifier, declaration, etc.
+message ReservedAsMapField {
+  map<string, uint32> if = 1;
+  map<string, uint32> const = 2;
+  map<string, uint32> private = 3;
+  map<string, uint32> class = 4;
+  map<string, uint32> int = 5;
+  map<string, uint32> void = 6;
+  map<string, uint32> string = 7; // These are also proto keywords
+  map<string, uint32> package = 8;
+  map<string, uint32> enum = 9; // Most recent Java reserved word
+  map<string, uint32> null = 10;
+  // null is not a 'reserved word' per se but as a literal needs similar care
+}
+
+message ReservedAsMapFieldWithEnumValue {
+  enum SampleEnum {
+    A = 0;
+    B = 1;
+  }
+  map<string, SampleEnum> if = 1;
+  map<string, SampleEnum> const = 2;
+  map<string, SampleEnum> private = 3;
+  map<string, SampleEnum> class = 4;
+  map<string, SampleEnum> int = 5;
+  map<string, SampleEnum> void = 6;
+  map<string, SampleEnum> string = 7; // These are also proto keywords
+  map<string, SampleEnum> package = 8;
+  map<string, SampleEnum> enum = 9; // Most recent Java reserved word
+  map<string, SampleEnum> null = 10;
+  // null is not a 'reserved word' per se but as a literal needs similar care
+}

java/core/src/test/proto/com/google/protobuf/test_bad_identifiers.proto

@@ -43,7 +43,6 @@ package io_protocol_tests;
 
 option java_package = "com.google.protobuf";
 option java_outer_classname = "TestBadIdentifiersProto";
-option java_generate_equals_and_hash = true;
 
 message TestMessage {
   optional string cached_size = 1;

java/src/main/java/com/google/protobuf/AbstractMessage.java

@@ -0,0 +1,646 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package com.google.protobuf;
+
+import com.google.protobuf.Descriptors.EnumValueDescriptor;
+import com.google.protobuf.Descriptors.FieldDescriptor;
+import com.google.protobuf.Descriptors.OneofDescriptor;
+import com.google.protobuf.Internal.EnumLite;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A partial implementation of the {@link Message} interface which implements
+ * as many methods of that interface as possible in terms of other methods.
+ *
+ * @author kenton@google.com Kenton Varda
+ */
+public abstract class AbstractMessage
+    // TODO(dweis): Update GeneratedMessage to parameterize with MessageType and BuilderType.
+    extends AbstractMessageLite
+    implements Message {
+
+  @Override
+  public boolean isInitialized() {
+    return MessageReflection.isInitialized(this);
+  }
+
+  /**
+   * Interface for the parent of a Builder that allows the builder to
+   * communicate invalidations back to the parent for use when using nested
+   * builders.
+   */
+  protected interface BuilderParent {
+
+    /**
+     * A builder becomes dirty whenever a field is modified -- including fields
+     * in nested builders -- and becomes clean when build() is called.  Thus,
+     * when a builder becomes dirty, all its parents become dirty as well, and
+     * when it becomes clean, all its children become clean.  The dirtiness
+     * state is used to invalidate certain cached values.
+     * <br>
+     * To this end, a builder calls markDirty() on its parent whenever it
+     * transitions from clean to dirty.  The parent must propagate this call to
+     * its own parent, unless it was already dirty, in which case the
+     * grandparent must necessarily already be dirty as well.  The parent can
+     * only transition back to "clean" after calling build() on all children.
+     */
+    void markDirty();
+  }
+
+  /** Create a nested builder. */
+  protected Message.Builder newBuilderForType(BuilderParent parent) {
+    throw new UnsupportedOperationException("Nested builder is not supported for this type.");
+  }
+
+
+  @Override
+  public List<String> findInitializationErrors() {
+    return MessageReflection.findMissingFields(this);
+  }
+
+  @Override
+  public String getInitializationErrorString() {
+    return MessageReflection.delimitWithCommas(findInitializationErrors());
+  }
+
+  /** TODO(jieluo): Clear it when all subclasses have implemented this method. */
+  @Override
+  public boolean hasOneof(OneofDescriptor oneof) {
+    throw new UnsupportedOperationException("hasOneof() is not implemented.");
+  }
+
+  /** TODO(jieluo): Clear it when all subclasses have implemented this method. */
+  @Override
+  public FieldDescriptor getOneofFieldDescriptor(OneofDescriptor oneof) {
+    throw new UnsupportedOperationException(
+        "getOneofFieldDescriptor() is not implemented.");
+  }
+
+  @Override
+  public final String toString() {
+    return TextFormat.printToString(this);
+  }
+
+  @Override
+  public void writeTo(final CodedOutputStream output) throws IOException {
+    MessageReflection.writeMessageTo(this, getAllFields(), output, false);
+  }
+
+  protected int memoizedSize = -1;
+
+  @Override
+  public int getSerializedSize() {
+    int size = memoizedSize;
+    if (size != -1) {
+      return size;
+    }
+
+    memoizedSize = MessageReflection.getSerializedSize(this, getAllFields());
+    return memoizedSize;
+  }
+
+  @Override
+  public boolean equals(final Object other) {
+    if (other == this) {
+      return true;
+    }
+    if (!(other instanceof Message)) {
+      return false;
+    }
+    final Message otherMessage = (Message) other;
+    if (getDescriptorForType() != otherMessage.getDescriptorForType()) {
+      return false;
+    }
+    return compareFields(getAllFields(), otherMessage.getAllFields()) &&
+        getUnknownFields().equals(otherMessage.getUnknownFields());
+  }
+
+  @Override
+  public int hashCode() {
+    int hash = memoizedHashCode;
+    if (hash == 0) {
+      hash = 41;
+      hash = (19 * hash) + getDescriptorForType().hashCode();
+      hash = hashFields(hash, getAllFields());
+      hash = (29 * hash) + getUnknownFields().hashCode();
+      memoizedHashCode = hash;
+    }
+    return hash;
+  }
+  
+  private static ByteString toByteString(Object value) {
+    if (value instanceof byte[]) {
+      return ByteString.copyFrom((byte[]) value);
+    } else {
+      return (ByteString) value;
+    }
+  }
+ 
+  /**
+   * Compares two bytes fields. The parameters must be either a byte array or a
+   * ByteString object. They can be of different type though.
+   */
+  private static boolean compareBytes(Object a, Object b) {
+    if (a instanceof byte[] && b instanceof byte[]) {
+      return Arrays.equals((byte[])a, (byte[])b);
+    }
+    return toByteString(a).equals(toByteString(b));
+  }
+  
+  /**
+   * Converts a list of MapEntry messages into a Map used for equals() and
+   * hashCode().
+   */
+  @SuppressWarnings({"rawtypes", "unchecked"})
+  private static Map convertMapEntryListToMap(List list) {
+    if (list.isEmpty()) {
+      return Collections.emptyMap();
+    }
+    Map result = new HashMap();
+    Iterator iterator = list.iterator();
+    Message entry = (Message) iterator.next();
+    Descriptors.Descriptor descriptor = entry.getDescriptorForType();
+    Descriptors.FieldDescriptor key = descriptor.findFieldByName("key");
+    Descriptors.FieldDescriptor value = descriptor.findFieldByName("value");
+    Object fieldValue = entry.getField(value);
+    if (fieldValue instanceof EnumValueDescriptor) {
+      fieldValue = ((EnumValueDescriptor) fieldValue).getNumber();
+    }
+    result.put(entry.getField(key), fieldValue);
+    while (iterator.hasNext()) {
+      entry = (Message) iterator.next();
+      fieldValue = entry.getField(value);
+      if (fieldValue instanceof EnumValueDescriptor) {
+        fieldValue = ((EnumValueDescriptor) fieldValue).getNumber();
+      }
+      result.put(entry.getField(key), fieldValue);
+    }
+    return result;
+  }
+  
+  /**
+   * Compares two map fields. The parameters must be a list of MapEntry
+   * messages.
+   */
+  @SuppressWarnings({"rawtypes", "unchecked"})
+  private static boolean compareMapField(Object a, Object b) {
+    Map ma = convertMapEntryListToMap((List) a);
+    Map mb = convertMapEntryListToMap((List) b);
+    return MapFieldLite.equals(ma, mb);
+  }
+  
+  /**
+   * Compares two set of fields.
+   * This method is used to implement {@link AbstractMessage#equals(Object)}
+   * and {@link AbstractMutableMessage#equals(Object)}. It takes special care
+   * of bytes fields because immutable messages and mutable messages use
+   * different Java type to reprensent a bytes field and this method should be
+   * able to compare immutable messages, mutable messages and also an immutable
+   * message to a mutable message.
+   */
+  static boolean compareFields(Map<FieldDescriptor, Object> a,
+      Map<FieldDescriptor, Object> b) {
+    if (a.size() != b.size()) {
+      return false;
+    }
+    for (FieldDescriptor descriptor : a.keySet()) {
+      if (!b.containsKey(descriptor)) {
+        return false;
+      }
+      Object value1 = a.get(descriptor);
+      Object value2 = b.get(descriptor);
+      if (descriptor.getType() == FieldDescriptor.Type.BYTES) {
+        if (descriptor.isRepeated()) {
+          List list1 = (List) value1;
+          List list2 = (List) value2;
+          if (list1.size() != list2.size()) {
+            return false;
+          }
+          for (int i = 0; i < list1.size(); i++) {
+            if (!compareBytes(list1.get(i), list2.get(i))) {
+              return false;
+            }
+          }
+        } else {
+          // Compares a singular bytes field.
+          if (!compareBytes(value1, value2)) {
+            return false;
+          }
+        }
+      } else if (descriptor.isMapField()) {
+        if (!compareMapField(value1, value2)) {
+          return false;
+        }
+      } else {
+        // Compare non-bytes fields.
+        if (!value1.equals(value2)) {
+          return false;
+        }
+      }
+    }
+    return true;
+  }
+  
+  /**
+   * Calculates the hash code of a map field. {@code value} must be a list of
+   * MapEntry messages.
+   */
+  @SuppressWarnings("unchecked")
+  private static int hashMapField(Object value) {
+    return MapFieldLite.calculateHashCodeForMap(convertMapEntryListToMap((List) value));
+  }
+
+  /** Get a hash code for given fields and values, using the given seed. */
+  @SuppressWarnings("unchecked")
+  protected static int hashFields(int hash, Map<FieldDescriptor, Object> map) {
+    for (Map.Entry<FieldDescriptor, Object> entry : map.entrySet()) {
+      FieldDescriptor field = entry.getKey();
+      Object value = entry.getValue();
+      hash = (37 * hash) + field.getNumber();
+      if (field.isMapField()) {
+        hash = (53 * hash) + hashMapField(value);
+      } else if (field.getType() != FieldDescriptor.Type.ENUM){
+        hash = (53 * hash) + value.hashCode();
+      } else if (field.isRepeated()) {
+        List<? extends EnumLite> list = (List<? extends EnumLite>) value;
+        hash = (53 * hash) + Internal.hashEnumList(list);
+      } else {
+        hash = (53 * hash) + Internal.hashEnum((EnumLite) value);
+      }
+    }
+    return hash;
+  }
+
+  /**
+   * Package private helper method for AbstractParser to create
+   * UninitializedMessageException with missing field information.
+   */
+  @Override
+  UninitializedMessageException newUninitializedMessageException() {
+    return Builder.newUninitializedMessageException(this);
+  }
+
+  // =================================================================
+
+  /**
+   * A partial implementation of the {@link Message.Builder} interface which
+   * implements as many methods of that interface as possible in terms of
+   * other methods.
+   */
+  @SuppressWarnings("unchecked")
+  public static abstract class Builder<BuilderType extends Builder<BuilderType>>
+      extends AbstractMessageLite.Builder
+      implements Message.Builder {
+    // The compiler produces an error if this is not declared explicitly.
+    @Override
+    public abstract BuilderType clone();
+
+    /** TODO(jieluo): Clear it when all subclasses have implemented this method. */
+    @Override
+    public boolean hasOneof(OneofDescriptor oneof) {
+      throw new UnsupportedOperationException("hasOneof() is not implemented.");
+    }
+
+    /** TODO(jieluo): Clear it when all subclasses have implemented this method. */
+    @Override
+    public FieldDescriptor getOneofFieldDescriptor(OneofDescriptor oneof) {
+      throw new UnsupportedOperationException(
+          "getOneofFieldDescriptor() is not implemented.");
+    }
+
+    /** TODO(jieluo): Clear it when all subclasses have implemented this method. */
+    @Override
+    public BuilderType clearOneof(OneofDescriptor oneof) {
+      throw new UnsupportedOperationException("clearOneof() is not implemented.");
+    }
+
+    @Override
+    public BuilderType clear() {
+      for (final Map.Entry<FieldDescriptor, Object> entry :
+           getAllFields().entrySet()) {
+        clearField(entry.getKey());
+      }
+      return (BuilderType) this;
+    }
+
+    @Override
+    public List<String> findInitializationErrors() {
+      return MessageReflection.findMissingFields(this);
+    }
+
+    @Override
+    public String getInitializationErrorString() {
+      return MessageReflection.delimitWithCommas(findInitializationErrors());
+    }
+    
+    @Override
+    protected BuilderType internalMergeFrom(AbstractMessageLite other) {
+      return mergeFrom((Message) other);
+    }
+
+    @Override
+    public BuilderType mergeFrom(final Message other) {
+      if (other.getDescriptorForType() != getDescriptorForType()) {
+        throw new IllegalArgumentException(
+          "mergeFrom(Message) can only merge messages of the same type.");
+      }
+
+      // Note:  We don't attempt to verify that other's fields have valid
+      //   types.  Doing so would be a losing battle.  We'd have to verify
+      //   all sub-messages as well, and we'd have to make copies of all of
+      //   them to insure that they don't change after verification (since
+      //   the Message interface itself cannot enforce immutability of
+      //   implementations).
+      // TODO(kenton):  Provide a function somewhere called makeDeepCopy()
+      //   which allows people to make secure deep copies of messages.
+
+      for (final Map.Entry<FieldDescriptor, Object> entry :
+           other.getAllFields().entrySet()) {
+        final FieldDescriptor field = entry.getKey();
+        if (field.isRepeated()) {
+          for (final Object element : (List)entry.getValue()) {
+            addRepeatedField(field, element);
+          }
+        } else if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) {
+          final Message existingValue = (Message)getField(field);
+          if (existingValue == existingValue.getDefaultInstanceForType()) {
+            setField(field, entry.getValue());
+          } else {
+            setField(field,
+              existingValue.newBuilderForType()
+                .mergeFrom(existingValue)
+                .mergeFrom((Message)entry.getValue())
+                .build());
+          }
+        } else {
+          setField(field, entry.getValue());
+        }
+      }
+
+      mergeUnknownFields(other.getUnknownFields());
+
+      return (BuilderType) this;
+    }
+
+    @Override
+    public BuilderType mergeFrom(final CodedInputStream input)
+                                 throws IOException {
+      return mergeFrom(input, ExtensionRegistry.getEmptyRegistry());
+    }
+
+    @Override
+    public BuilderType mergeFrom(
+        final CodedInputStream input,
+        final ExtensionRegistryLite extensionRegistry)
+        throws IOException {
+      final UnknownFieldSet.Builder unknownFields =
+        UnknownFieldSet.newBuilder(getUnknownFields());
+      while (true) {
+        final int tag = input.readTag();
+        if (tag == 0) {
+          break;
+        }
+
+        MessageReflection.BuilderAdapter builderAdapter =
+            new MessageReflection.BuilderAdapter(this);
+        if (!MessageReflection.mergeFieldFrom(input, unknownFields,
+                                              extensionRegistry,
+                                              getDescriptorForType(),
+                                              builderAdapter,
+                                              tag)) {
+          // end group tag
+          break;
+        }
+      }
+      setUnknownFields(unknownFields.build());
+      return (BuilderType) this;
+    }
+
+    @Override
+    public BuilderType mergeUnknownFields(final UnknownFieldSet unknownFields) {
+      setUnknownFields(
+        UnknownFieldSet.newBuilder(getUnknownFields())
+                       .mergeFrom(unknownFields)
+                       .build());
+      return (BuilderType) this;
+    }
+
+    @Override
+    public Message.Builder getFieldBuilder(final FieldDescriptor field) {
+      throw new UnsupportedOperationException(
+          "getFieldBuilder() called on an unsupported message type.");
+    }
+
+    @Override
+    public Message.Builder getRepeatedFieldBuilder(final FieldDescriptor field, int index) {
+      throw new UnsupportedOperationException(
+          "getRepeatedFieldBuilder() called on an unsupported message type.");
+    }
+
+    @Override
+    public String toString() {
+      return TextFormat.printToString(this);
+    }
+
+    /**
+     * Construct an UninitializedMessageException reporting missing fields in
+     * the given message.
+     */
+    protected static UninitializedMessageException
+        newUninitializedMessageException(Message message) {
+      return new UninitializedMessageException(
+          MessageReflection.findMissingFields(message));
+    }
+
+    /**
+     * Used to support nested builders and called to mark this builder as clean.
+     * Clean builders will propagate the {@link BuilderParent#markDirty()} event
+     * to their parent builders, while dirty builders will not, as their parents
+     * should be dirty already.
+     *
+     * NOTE: Implementations that don't support nested builders don't need to
+     * override this method.
+     */
+    void markClean() {
+      throw new IllegalStateException("Should be overridden by subclasses.");
+    }
+
+    /**
+     * Used to support nested builders and called when this nested builder is
+     * no longer used by its parent builder and should release the reference
+     * to its parent builder.
+     *
+     * NOTE: Implementations that don't support nested builders don't need to
+     * override this method.
+     */
+    void dispose() {
+      throw new IllegalStateException("Should be overridden by subclasses.");
+    }
+
+    // ===============================================================
+    // The following definitions seem to be required in order to make javac
+    // not produce weird errors like:
+    //
+    // java/com/google/protobuf/DynamicMessage.java:203: types
+    //   com.google.protobuf.AbstractMessage.Builder<
+    //     com.google.protobuf.DynamicMessage.Builder> and
+    //   com.google.protobuf.AbstractMessage.Builder<
+    //     com.google.protobuf.DynamicMessage.Builder> are incompatible; both
+    //   define mergeFrom(com.google.protobuf.ByteString), but with unrelated
+    //   return types.
+    //
+    // Strangely, these lines are only needed if javac is invoked separately
+    // on AbstractMessage.java and AbstractMessageLite.java.  If javac is
+    // invoked on both simultaneously, it works.  (Or maybe the important
+    // point is whether or not DynamicMessage.java is compiled together with
+    // AbstractMessageLite.java -- not sure.)  I suspect this is a compiler
+    // bug.
+
+    @Override
+    public BuilderType mergeFrom(final ByteString data)
+        throws InvalidProtocolBufferException {
+      return (BuilderType) super.mergeFrom(data);
+    }
+
+    @Override
+    public BuilderType mergeFrom(
+        final ByteString data,
+        final ExtensionRegistryLite extensionRegistry)
+        throws InvalidProtocolBufferException {
+      return (BuilderType) super.mergeFrom(data, extensionRegistry);
+    }
+
+    @Override
+    public BuilderType mergeFrom(final byte[] data)
+        throws InvalidProtocolBufferException {
+      return (BuilderType) super.mergeFrom(data);
+    }
+
+    @Override
+    public BuilderType mergeFrom(
+        final byte[] data, final int off, final int len)
+        throws InvalidProtocolBufferException {
+      return (BuilderType) super.mergeFrom(data, off, len);
+    }
+
+    @Override
+    public BuilderType mergeFrom(
+        final byte[] data,
+        final ExtensionRegistryLite extensionRegistry)
+        throws InvalidProtocolBufferException {
+      return (BuilderType) super.mergeFrom(data, extensionRegistry);
+    }
+
+    @Override
+    public BuilderType mergeFrom(
+        final byte[] data, final int off, final int len,
+        final ExtensionRegistryLite extensionRegistry)
+        throws InvalidProtocolBufferException {
+      return (BuilderType) super.mergeFrom(data, off, len, extensionRegistry);
+    }
+
+    @Override
+    public BuilderType mergeFrom(final InputStream input)
+        throws IOException {
+      return (BuilderType) super.mergeFrom(input);
+    }
+
+    @Override
+    public BuilderType mergeFrom(
+        final InputStream input,
+        final ExtensionRegistryLite extensionRegistry)
+        throws IOException {
+      return (BuilderType) super.mergeFrom(input, extensionRegistry);
+    }
+
+    @Override
+    public boolean mergeDelimitedFrom(final InputStream input)
+        throws IOException {
+      return super.mergeDelimitedFrom(input);
+    }
+
+    @Override
+    public boolean mergeDelimitedFrom(
+        final InputStream input,
+        final ExtensionRegistryLite extensionRegistry)
+        throws IOException {
+      return super.mergeDelimitedFrom(input, extensionRegistry);
+    }
+  }
+
+  /**
+   * @deprecated from v3.0.0-beta-3+, for compatibility with v2.5.0 and v2.6.1
+   * generated code.
+   */
+  @Deprecated
+  protected static int hashLong(long n) {
+    return (int) (n ^ (n >>> 32));
+  }
+  //
+  /**
+   * @deprecated from v3.0.0-beta-3+, for compatibility with v2.5.0 and v2.6.1
+   * generated code.
+   */
+  @Deprecated
+  protected static int hashBoolean(boolean b) {
+    return b ? 1231 : 1237;
+  }
+  //
+  /**
+   * @deprecated from v3.0.0-beta-3+, for compatibility with v2.5.0 and v2.6.1
+   * generated code.
+   */
+  @Deprecated
+  protected static int hashEnum(EnumLite e) {
+    return e.getNumber();
+  }
+  //
+  /**
+   * @deprecated from v3.0.0-beta-3+, for compatibility with v2.5.0 and v2.6.1
+   * generated code.
+   */
+  @Deprecated
+  protected static int hashEnumList(List<? extends EnumLite> list) {
+    int hash = 1;
+    for (EnumLite e : list) {
+      hash = 31 * hash + hashEnum(e);
+    }
+    return hash;
+  }
+}

java/src/main/java/com/google/protobuf/AbstractMessageLite.java

@@ -0,0 +1,384 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package com.google.protobuf;
+
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Collection;
+
+/**
+ * A partial implementation of the {@link MessageLite} interface which
+ * implements as many methods of that interface as possible in terms of other
+ * methods.
+ *
+ * @author kenton@google.com Kenton Varda
+ */
+public abstract class AbstractMessageLite<
+    MessageType extends AbstractMessageLite<MessageType, BuilderType>,
+    BuilderType extends AbstractMessageLite.Builder<MessageType, BuilderType>>
+        implements MessageLite {
+  protected int memoizedHashCode = 0;
+
+  @Override
+  public ByteString toByteString() {
+    try {
+      final ByteString.CodedBuilder out =
+        ByteString.newCodedBuilder(getSerializedSize());
+      writeTo(out.getCodedOutput());
+      return out.build();
+    } catch (IOException e) {
+      throw new RuntimeException(getSerializingExceptionMessage("ByteString"), e);
+    }
+  }
+
+  @Override
+  public byte[] toByteArray() {
+    try {
+      final byte[] result = new byte[getSerializedSize()];
+      final CodedOutputStream output = CodedOutputStream.newInstance(result);
+      writeTo(output);
+      output.checkNoSpaceLeft();
+      return result;
+    } catch (IOException e) {
+      throw new RuntimeException(getSerializingExceptionMessage("byte array"), e);
+    }
+  }
+
+  @Override
+  public void writeTo(final OutputStream output) throws IOException {
+    final int bufferSize =
+        CodedOutputStream.computePreferredBufferSize(getSerializedSize());
+    final CodedOutputStream codedOutput =
+        CodedOutputStream.newInstance(output, bufferSize);
+    writeTo(codedOutput);
+    codedOutput.flush();
+  }
+
+  @Override
+  public void writeDelimitedTo(final OutputStream output) throws IOException {
+    final int serialized = getSerializedSize();
+    final int bufferSize = CodedOutputStream.computePreferredBufferSize(
+        CodedOutputStream.computeRawVarint32Size(serialized) + serialized);
+    final CodedOutputStream codedOutput =
+        CodedOutputStream.newInstance(output, bufferSize);
+    codedOutput.writeRawVarint32(serialized);
+    writeTo(codedOutput);
+    codedOutput.flush();
+  }
+
+
+  /**
+   * Package private helper method for AbstractParser to create
+   * UninitializedMessageException.
+   */
+  UninitializedMessageException newUninitializedMessageException() {
+    return new UninitializedMessageException(this);
+  }
+
+  private String getSerializingExceptionMessage(String target) {
+    return "Serializing " + getClass().getName() + " to a " + target
+        + " threw an IOException (should never happen).";
+  }
+
+  protected static void checkByteStringIsUtf8(ByteString byteString)
+      throws IllegalArgumentException {
+    if (!byteString.isValidUtf8()) {
+      throw new IllegalArgumentException("Byte string is not UTF-8.");
+    }
+  }
+
+  protected static <T> void addAll(final Iterable<T> values,
+      final Collection<? super T> list) {
+    Builder.addAll(values, list);
+  }
+
+  /**
+   * A partial implementation of the {@link Message.Builder} interface which
+   * implements as many methods of that interface as possible in terms of
+   * other methods.
+   */
+  @SuppressWarnings("unchecked")
+  public abstract static class Builder<
+      MessageType extends AbstractMessageLite<MessageType, BuilderType>,
+      BuilderType extends Builder<MessageType, BuilderType>>
+      implements MessageLite.Builder {
+    // The compiler produces an error if this is not declared explicitly.
+    @Override
+    public abstract BuilderType clone();
+
+    @Override
+    public BuilderType mergeFrom(final CodedInputStream input) throws IOException {
+      return mergeFrom(input, ExtensionRegistryLite.getEmptyRegistry());
+    }
+
+    // Re-defined here for return type covariance.
+    @Override
+    public abstract BuilderType mergeFrom(
+        final CodedInputStream input, final ExtensionRegistryLite extensionRegistry)
+        throws IOException;
+
+    @Override
+    public BuilderType mergeFrom(final ByteString data) throws InvalidProtocolBufferException {
+      try {
+        final CodedInputStream input = data.newCodedInput();
+        mergeFrom(input);
+        input.checkLastTagWas(0);
+        return (BuilderType) this;
+      } catch (InvalidProtocolBufferException e) {
+        throw e;
+      } catch (IOException e) {
+        throw new RuntimeException(getReadingExceptionMessage("ByteString"), e);
+      }
+    }
+
+    @Override
+    public BuilderType mergeFrom(
+        final ByteString data, final ExtensionRegistryLite extensionRegistry)
+        throws InvalidProtocolBufferException {
+      try {
+        final CodedInputStream input = data.newCodedInput();
+        mergeFrom(input, extensionRegistry);
+        input.checkLastTagWas(0);
+        return (BuilderType) this;
+      } catch (InvalidProtocolBufferException e) {
+        throw e;
+      } catch (IOException e) {
+        throw new RuntimeException(getReadingExceptionMessage("ByteString"), e);
+      }
+    }
+
+    @Override
+    public BuilderType mergeFrom(final byte[] data) throws InvalidProtocolBufferException {
+      return mergeFrom(data, 0, data.length);
+    }
+
+    @Override
+    public BuilderType mergeFrom(final byte[] data, final int off, final int len)
+        throws InvalidProtocolBufferException {
+      try {
+        final CodedInputStream input =
+            CodedInputStream.newInstance(data, off, len);
+        mergeFrom(input);
+        input.checkLastTagWas(0);
+        return (BuilderType) this;
+      } catch (InvalidProtocolBufferException e) {
+        throw e;
+      } catch (IOException e) {
+        throw new RuntimeException(getReadingExceptionMessage("byte array"), e);
+      }
+    }
+
+    @Override
+    public BuilderType mergeFrom(final byte[] data, final ExtensionRegistryLite extensionRegistry)
+        throws InvalidProtocolBufferException {
+      return mergeFrom(data, 0, data.length, extensionRegistry);
+    }
+
+    @Override
+    public BuilderType mergeFrom(
+        final byte[] data,
+        final int off,
+        final int len,
+        final ExtensionRegistryLite extensionRegistry)
+        throws InvalidProtocolBufferException {
+      try {
+        final CodedInputStream input =
+            CodedInputStream.newInstance(data, off, len);
+        mergeFrom(input, extensionRegistry);
+        input.checkLastTagWas(0);
+        return (BuilderType) this;
+      } catch (InvalidProtocolBufferException e) {
+        throw e;
+      } catch (IOException e) {
+        throw new RuntimeException(getReadingExceptionMessage("byte array"), e);
+      }
+    }
+
+    @Override
+    public BuilderType mergeFrom(final InputStream input) throws IOException {
+      final CodedInputStream codedInput = CodedInputStream.newInstance(input);
+      mergeFrom(codedInput);
+      codedInput.checkLastTagWas(0);
+      return (BuilderType) this;
+    }
+
+    @Override
+    public BuilderType mergeFrom(
+        final InputStream input, final ExtensionRegistryLite extensionRegistry) throws IOException {
+      final CodedInputStream codedInput = CodedInputStream.newInstance(input);
+      mergeFrom(codedInput, extensionRegistry);
+      codedInput.checkLastTagWas(0);
+      return (BuilderType) this;
+    }
+
+    /**
+     * An InputStream implementations which reads from some other InputStream
+     * but is limited to a particular number of bytes.  Used by
+     * mergeDelimitedFrom().  This is intentionally package-private so that
+     * UnknownFieldSet can share it.
+     */
+    static final class LimitedInputStream extends FilterInputStream {
+      private int limit;
+
+      LimitedInputStream(InputStream in, int limit) {
+        super(in);
+        this.limit = limit;
+      }
+
+      @Override
+      public int available() throws IOException {
+        return Math.min(super.available(), limit);
+      }
+
+      @Override
+      public int read() throws IOException {
+        if (limit <= 0) {
+          return -1;
+        }
+        final int result = super.read();
+        if (result >= 0) {
+          --limit;
+        }
+        return result;
+      }
+
+      @Override
+      public int read(final byte[] b, final int off, int len)
+                      throws IOException {
+        if (limit <= 0) {
+          return -1;
+        }
+        len = Math.min(len, limit);
+        final int result = super.read(b, off, len);
+        if (result >= 0) {
+          limit -= result;
+        }
+        return result;
+      }
+
+      @Override
+      public long skip(final long n) throws IOException {
+        final long result = super.skip(Math.min(n, limit));
+        if (result >= 0) {
+          limit -= result;
+        }
+        return result;
+      }
+    }
+
+    @Override
+    public boolean mergeDelimitedFrom(
+        final InputStream input, final ExtensionRegistryLite extensionRegistry) throws IOException {
+      final int firstByte = input.read();
+      if (firstByte == -1) {
+        return false;
+      }
+      final int size = CodedInputStream.readRawVarint32(firstByte, input);
+      final InputStream limitedInput = new LimitedInputStream(input, size);
+      mergeFrom(limitedInput, extensionRegistry);
+      return true;
+    }
+
+    @Override
+    public boolean mergeDelimitedFrom(final InputStream input) throws IOException {
+      return mergeDelimitedFrom(input,
+          ExtensionRegistryLite.getEmptyRegistry());
+    }
+
+    @Override
+    @SuppressWarnings("unchecked") // isInstance takes care of this
+    public BuilderType mergeFrom(final MessageLite other) {
+      if (!getDefaultInstanceForType().getClass().isInstance(other)) {
+        throw new IllegalArgumentException(
+            "mergeFrom(MessageLite) can only merge messages of the same type.");
+      }
+
+      return internalMergeFrom((MessageType) other);
+    }
+
+    protected abstract BuilderType internalMergeFrom(MessageType message);
+
+    private String getReadingExceptionMessage(String target) {
+      return "Reading " + getClass().getName() + " from a " + target
+          + " threw an IOException (should never happen).";
+    }
+
+    /**
+     * Construct an UninitializedMessageException reporting missing fields in
+     * the given message.
+     */
+    protected static UninitializedMessageException
+        newUninitializedMessageException(MessageLite message) {
+      return new UninitializedMessageException(message);
+    }
+
+    /**
+     * Adds the {@code values} to the {@code list}.  This is a helper method
+     * used by generated code.  Users should ignore it.
+     *
+     * @throws NullPointerException if {@code values} or any of the elements of
+     * {@code values} is null. When that happens, some elements of
+     * {@code values} may have already been added to the result {@code list}.
+     */
+    protected static <T> void addAll(final Iterable<T> values,
+                                     final Collection<? super T> list) {
+      if (values == null) {
+        throw new NullPointerException();
+      }
+      if (values instanceof LazyStringList) {
+        // For StringOrByteStringLists, check the underlying elements to avoid
+        // forcing conversions of ByteStrings to Strings.
+        checkForNullValues(((LazyStringList) values).getUnderlyingElements());
+        list.addAll((Collection<T>) values);
+      } else if (values instanceof Collection) {
+        checkForNullValues(values);
+        list.addAll((Collection<T>) values);
+      } else {
+        for (final T value : values) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          list.add(value);
+        }
+      }
+    }
+
+    private static void checkForNullValues(final Iterable<?> values) {
+      for (final Object value : values) {
+        if (value == null) {
+          throw new NullPointerException();
+        }
+      }
+    }
+  }
+}

java/src/main/java/com/google/protobuf/AbstractParser.java

@@ -0,0 +1,258 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package com.google.protobuf;
+
+import com.google.protobuf.AbstractMessageLite.Builder.LimitedInputStream;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * A partial implementation of the {@link Parser} interface which implements
+ * as many methods of that interface as possible in terms of other methods.
+ *
+ * Note: This class implements all the convenience methods in the
+ * {@link Parser} interface. See {@link Parser} for related javadocs.
+ * Subclasses need to implement
+ * {@link Parser#parsePartialFrom(CodedInputStream, ExtensionRegistryLite)}
+ *
+ * @author liujisi@google.com (Pherl Liu)
+ */
+public abstract class AbstractParser<MessageType extends MessageLite>
+    implements Parser<MessageType> {
+  /**
+   * Creates an UninitializedMessageException for MessageType.
+   */
+  private UninitializedMessageException
+      newUninitializedMessageException(MessageType message) {
+    if (message instanceof AbstractMessageLite) {
+      return ((AbstractMessageLite) message).newUninitializedMessageException();
+    }
+    return new UninitializedMessageException(message);
+  }
+
+  /**
+   * Helper method to check if message is initialized.
+   *
+   * @throws InvalidProtocolBufferException if it is not initialized.
+   * @return The message to check.
+   */
+  private MessageType checkMessageInitialized(MessageType message)
+      throws InvalidProtocolBufferException {
+    if (message != null && !message.isInitialized()) {
+      throw newUninitializedMessageException(message)
+          .asInvalidProtocolBufferException()
+          .setUnfinishedMessage(message);
+    }
+    return message;
+  }
+
+  private static final ExtensionRegistryLite EMPTY_REGISTRY
+      = ExtensionRegistryLite.getEmptyRegistry();
+
+  @Override
+  public MessageType parsePartialFrom(CodedInputStream input)
+      throws InvalidProtocolBufferException {
+    return parsePartialFrom(input, EMPTY_REGISTRY);
+  }
+
+  @Override
+  public MessageType parseFrom(CodedInputStream input, ExtensionRegistryLite extensionRegistry)
+      throws InvalidProtocolBufferException {
+    return checkMessageInitialized(
+        parsePartialFrom(input, extensionRegistry));
+  }
+
+  @Override
+  public MessageType parseFrom(CodedInputStream input) throws InvalidProtocolBufferException {
+    return parseFrom(input, EMPTY_REGISTRY);
+  }
+
+  @Override
+  public MessageType parsePartialFrom(ByteString data, ExtensionRegistryLite extensionRegistry)
+      throws InvalidProtocolBufferException {
+    MessageType message;
+    try {
+      CodedInputStream input = data.newCodedInput();
+      message = parsePartialFrom(input, extensionRegistry);
+      try {
+        input.checkLastTagWas(0);
+      } catch (InvalidProtocolBufferException e) {
+        throw e.setUnfinishedMessage(message);
+      }
+      return message;
+    } catch (InvalidProtocolBufferException e) {
+      throw e;
+    }
+  }
+
+  @Override
+  public MessageType parsePartialFrom(ByteString data) throws InvalidProtocolBufferException {
+    return parsePartialFrom(data, EMPTY_REGISTRY);
+  }
+
+  @Override
+  public MessageType parseFrom(ByteString data, ExtensionRegistryLite extensionRegistry)
+      throws InvalidProtocolBufferException {
+    return checkMessageInitialized(parsePartialFrom(data, extensionRegistry));
+  }
+
+  @Override
+  public MessageType parseFrom(ByteString data) throws InvalidProtocolBufferException {
+    return parseFrom(data, EMPTY_REGISTRY);
+  }
+
+  @Override
+  public MessageType parsePartialFrom(
+      byte[] data, int off, int len, ExtensionRegistryLite extensionRegistry)
+      throws InvalidProtocolBufferException {
+    try {
+      CodedInputStream input = CodedInputStream.newInstance(data, off, len);
+      MessageType message = parsePartialFrom(input, extensionRegistry);
+      try {
+        input.checkLastTagWas(0);
+      } catch (InvalidProtocolBufferException e) {
+        throw e.setUnfinishedMessage(message);
+      }
+      return message;
+    } catch (InvalidProtocolBufferException e) {
+      throw e;
+    }
+  }
+
+  @Override
+  public MessageType parsePartialFrom(byte[] data, int off, int len)
+      throws InvalidProtocolBufferException {
+    return parsePartialFrom(data, off, len, EMPTY_REGISTRY);
+  }
+
+  @Override
+  public MessageType parsePartialFrom(byte[] data, ExtensionRegistryLite extensionRegistry)
+      throws InvalidProtocolBufferException {
+    return parsePartialFrom(data, 0, data.length, extensionRegistry);
+  }
+
+  @Override
+  public MessageType parsePartialFrom(byte[] data) throws InvalidProtocolBufferException {
+    return parsePartialFrom(data, 0, data.length, EMPTY_REGISTRY);
+  }
+
+  @Override
+  public MessageType parseFrom(
+      byte[] data, int off, int len, ExtensionRegistryLite extensionRegistry)
+      throws InvalidProtocolBufferException {
+    return checkMessageInitialized(
+        parsePartialFrom(data, off, len, extensionRegistry));
+  }
+
+  @Override
+  public MessageType parseFrom(byte[] data, int off, int len)
+      throws InvalidProtocolBufferException {
+    return parseFrom(data, off, len, EMPTY_REGISTRY);
+  }
+
+  @Override
+  public MessageType parseFrom(byte[] data, ExtensionRegistryLite extensionRegistry)
+      throws InvalidProtocolBufferException {
+    return parseFrom(data, 0, data.length, extensionRegistry);
+  }
+
+  @Override
+  public MessageType parseFrom(byte[] data) throws InvalidProtocolBufferException {
+    return parseFrom(data, EMPTY_REGISTRY);
+  }
+
+  @Override
+  public MessageType parsePartialFrom(InputStream input, ExtensionRegistryLite extensionRegistry)
+      throws InvalidProtocolBufferException {
+    CodedInputStream codedInput = CodedInputStream.newInstance(input);
+    MessageType message = parsePartialFrom(codedInput, extensionRegistry);
+    try {
+      codedInput.checkLastTagWas(0);
+    } catch (InvalidProtocolBufferException e) {
+      throw e.setUnfinishedMessage(message);
+    }
+    return message;
+  }
+
+  @Override
+  public MessageType parsePartialFrom(InputStream input) throws InvalidProtocolBufferException {
+    return parsePartialFrom(input, EMPTY_REGISTRY);
+  }
+
+  @Override
+  public MessageType parseFrom(InputStream input, ExtensionRegistryLite extensionRegistry)
+      throws InvalidProtocolBufferException {
+    return checkMessageInitialized(
+        parsePartialFrom(input, extensionRegistry));
+  }
+
+  @Override
+  public MessageType parseFrom(InputStream input) throws InvalidProtocolBufferException {
+    return parseFrom(input, EMPTY_REGISTRY);
+  }
+
+  @Override
+  public MessageType parsePartialDelimitedFrom(
+      InputStream input, ExtensionRegistryLite extensionRegistry)
+      throws InvalidProtocolBufferException {
+    int size;
+    try {
+      int firstByte = input.read();
+      if (firstByte == -1) {
+        return null;
+      }
+      size = CodedInputStream.readRawVarint32(firstByte, input);
+    } catch (IOException e) {
+      throw new InvalidProtocolBufferException(e.getMessage());
+    }
+    InputStream limitedInput = new LimitedInputStream(input, size);
+    return parsePartialFrom(limitedInput, extensionRegistry);
+  }
+
+  @Override
+  public MessageType parsePartialDelimitedFrom(InputStream input)
+      throws InvalidProtocolBufferException {
+    return parsePartialDelimitedFrom(input, EMPTY_REGISTRY);
+  }
+
+  @Override
+  public MessageType parseDelimitedFrom(InputStream input, ExtensionRegistryLite extensionRegistry)
+      throws InvalidProtocolBufferException {
+    return checkMessageInitialized(
+        parsePartialDelimitedFrom(input, extensionRegistry));
+  }
+
+  @Override
+  public MessageType parseDelimitedFrom(InputStream input) throws InvalidProtocolBufferException {
+    return parseDelimitedFrom(input, EMPTY_REGISTRY);
+  }
+}

java/src/main/java/com/google/protobuf/AbstractProtobufList.java

@@ -0,0 +1,180 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package com.google.protobuf;
+
+import com.google.protobuf.Internal.ProtobufList;
+
+import java.util.AbstractList;
+import java.util.Collection;
+import java.util.List;
+import java.util.RandomAccess;
+
+/**
+ * An abstract implementation of {@link ProtobufList} which manages mutability semantics. All mutate
+ * methods must check if the list is mutable before proceeding. Subclasses must invoke
+ * {@link #ensureIsMutable()} manually when overriding those methods.
+ * <p>
+ * This implementation assumes all subclasses are array based, supporting random access.
+ */
+abstract class AbstractProtobufList<E> extends AbstractList<E> implements ProtobufList<E> {
+
+  protected static final int DEFAULT_CAPACITY = 10;
+
+  /**
+   * Whether or not this list is modifiable.
+   */
+  private boolean isMutable;
+
+  /**
+   * Constructs a mutable list by default.
+   */
+  AbstractProtobufList() {
+    isMutable = true;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (o == this) {
+      return true;
+    }
+    if (!(o instanceof List)) {
+      return false;
+    }
+    // Handle lists that do not support RandomAccess as efficiently as possible by using an iterator
+    // based approach in our super class. Otherwise our index based approach will avoid those
+    // allocations.
+    if (!(o instanceof RandomAccess)) {
+      return super.equals(o);
+    }
+
+    List<?> other = (List<?>) o;
+    final int size = size();
+    if (size != other.size()) {
+      return false;
+    }
+    for (int i = 0; i < size; i++) {
+      if (!get(i).equals(other.get(i))) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  @Override
+  public int hashCode() {
+    final int size = size();
+    int hashCode = 1;
+    for (int i = 0; i < size; i++) {
+      hashCode = (31 * hashCode) + get(i).hashCode();
+    }
+    return hashCode;
+  }
+
+  @Override
+  public boolean add(E e) {
+    ensureIsMutable();
+    return super.add(e);
+  }
+
+  @Override
+  public void add(int index, E element) {
+    ensureIsMutable();
+    super.add(index, element);
+  }
+
+  @Override
+  public boolean addAll(Collection<? extends E> c) {
+    ensureIsMutable();
+    return super.addAll(c);
+  }
+  
+  @Override
+  public boolean addAll(int index, Collection<? extends E> c) {
+    ensureIsMutable();
+    return super.addAll(index, c);
+  }
+
+  @Override
+  public void clear() {
+    ensureIsMutable();
+    super.clear();
+  }
+  
+  @Override
+  public boolean isModifiable() {
+    return isMutable;
+  }
+  
+  @Override
+  public final void makeImmutable() {
+    isMutable = false;
+  }
+  
+  @Override
+  public E remove(int index) {
+    ensureIsMutable();
+    return super.remove(index);
+  }
+  
+  @Override
+  public boolean remove(Object o) {
+    ensureIsMutable();
+    return super.remove(o);
+  }
+  
+  @Override
+  public boolean removeAll(Collection<?> c) {
+    ensureIsMutable();
+    return super.removeAll(c);
+  }
+  
+  @Override
+  public boolean retainAll(Collection<?> c) {
+    ensureIsMutable();
+    return super.retainAll(c);
+  }
+  
+  @Override
+  public E set(int index, E element) {
+    ensureIsMutable();
+    return super.set(index, element);
+  }
+  
+  /**
+   * Throws an {@link UnsupportedOperationException} if the list is immutable. Subclasses are
+   * responsible for invoking this method on mutate operations.
+   */
+  protected void ensureIsMutable() {
+    if (!isMutable) {
+      throw new UnsupportedOperationException();
+    }
+  }
+}

java/src/main/java/com/google/protobuf/BlockingRpcChannel.java

@@ -0,0 +1,51 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package com.google.protobuf;
+
+/**
+ * <p>Abstract interface for a blocking RPC channel.  {@code BlockingRpcChannel}
+ * is the blocking equivalent to {@link RpcChannel}.
+ *
+ * @author kenton@google.com Kenton Varda
+ * @author cpovirk@google.com Chris Povirk
+ */
+public interface BlockingRpcChannel {
+  /**
+   * Call the given method of the remote service and blocks until it returns.
+   * {@code callBlockingMethod()} is the blocking equivalent to
+   * {@link RpcChannel#callMethod}.
+   */
+  Message callBlockingMethod(
+      Descriptors.MethodDescriptor method,
+      RpcController controller,
+      Message request,
+      Message responsePrototype) throws ServiceException;
+}

java/src/main/java/com/google/protobuf/BlockingService.java

@@ -0,0 +1,64 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package com.google.protobuf;
+
+/**
+ * Blocking equivalent to {@link Service}.
+ *
+ * @author kenton@google.com Kenton Varda
+ * @author cpovirk@google.com Chris Povirk
+ */
+public interface BlockingService {
+  /**
+   * Equivalent to {@link Service#getDescriptorForType}.
+   */
+  Descriptors.ServiceDescriptor getDescriptorForType();
+
+  /**
+   * Equivalent to {@link Service#callMethod}, except that
+   * {@code callBlockingMethod()} returns the result of the RPC or throws a
+   * {@link ServiceException} if there is a failure, rather than passing the
+   * information to a callback.
+   */
+  Message callBlockingMethod(Descriptors.MethodDescriptor method,
+                             RpcController controller,
+                             Message request) throws ServiceException;
+
+  /**
+   * Equivalent to {@link Service#getRequestPrototype}.
+   */
+  Message getRequestPrototype(Descriptors.MethodDescriptor method);
+
+  /**
+   * Equivalent to {@link Service#getResponsePrototype}.
+   */
+  Message getResponsePrototype(Descriptors.MethodDescriptor method);
+}

java/util/src/main/java/com/google/protobuf/util/Durations.java

@@ -30,50 +30,86 @@
 
 package com.google.protobuf.util;
 
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.math.IntMath.checkedAdd;
+import static com.google.common.math.IntMath.checkedSubtract;
+import static com.google.common.math.LongMath.checkedAdd;
+import static com.google.common.math.LongMath.checkedMultiply;
+import static com.google.common.math.LongMath.checkedSubtract;
 import static com.google.protobuf.util.Timestamps.MICROS_PER_SECOND;
 import static com.google.protobuf.util.Timestamps.MILLIS_PER_SECOND;
 import static com.google.protobuf.util.Timestamps.NANOS_PER_MICROSECOND;
 import static com.google.protobuf.util.Timestamps.NANOS_PER_MILLISECOND;
 import static com.google.protobuf.util.Timestamps.NANOS_PER_SECOND;
 
+import com.google.common.collect.ComparisonChain;
 import com.google.protobuf.Duration;
-
 import java.text.ParseException;
+import java.util.Comparator;
 
 /**
- * Utilities to help create/manipulate {@code protobuf/duration.proto}.
+ * Utilities to help create/manipulate {@code protobuf/duration.proto}. All operations throw an
+ * {@link IllegalArgumentException} if the input(s) are not {@linkplain #isValid(Duration) valid}.
  */
 public final class Durations {
   static final long DURATION_SECONDS_MIN = -315576000000L;
   static final long DURATION_SECONDS_MAX = 315576000000L;
 
-  // TODO(kak): Do we want to expose Duration constants for MAX/MIN?
+  /** A constant holding the minimum valid {@link Duration}, approximately {@code -10,000} years. */
+  public static final Duration MIN_VALUE =
+      Duration.newBuilder().setSeconds(DURATION_SECONDS_MIN).setNanos(-999999999).build();
+
+  /** A constant holding the maximum valid {@link Duration}, approximately {@code +10,000} years. */
+  public static final Duration MAX_VALUE =
+      Duration.newBuilder().setSeconds(DURATION_SECONDS_MAX).setNanos(999999999).build();
 
   private Durations() {}
 
+  private static final Comparator<Duration> COMPARATOR =
+      new Comparator<Duration>() {
+        @Override
+        public int compare(Duration d1, Duration d2) {
+          checkValid(d1);
+          checkValid(d2);
+
+          return ComparisonChain.start()
+              .compare(d1.getSeconds(), d2.getSeconds())
+              .compare(d1.getNanos(), d2.getNanos())
+              .result();
+        }
+      };
+
+  /**
+   * Returns a {@link Comparator} for {@link Duration}s which sorts in increasing chronological
+   * order. Nulls and invalid {@link Duration}s are not allowed (see {@link #isValid}).
+   */
+  public static Comparator<Duration> comparator() {
+    return COMPARATOR;
+  }
+
   /**
    * Returns true if the given {@link Duration} is valid. The {@code seconds} value must be in the
    * range [-315,576,000,000, +315,576,000,000]. The {@code nanos} value must be in the range
    * [-999,999,999, +999,999,999].
    *
-   * <p>Note: Durations less than one second are represented with a 0 {@code seconds} field and a
-   * positive or negative {@code nanos} field. For durations of one second or more, a non-zero value
-   * for the {@code nanos} field must be of the same sign as the {@code seconds} field.
+   * <p><b>Note:</b> Durations less than one second are represented with a 0 {@code seconds} field
+   * and a positive or negative {@code nanos} field. For durations of one second or more, a non-zero
+   * value for the {@code nanos} field must be of the same sign as the {@code seconds} field.
    */
   public static boolean isValid(Duration duration) {
     return isValid(duration.getSeconds(), duration.getNanos());
   }
 
   /**
-   * Returns true if the given number of seconds and nanos is a valid {@link Duration}. The
-   * {@code seconds} value must be in the range [-315,576,000,000, +315,576,000,000]. The
-   * {@code nanos} value must be in the range [-999,999,999, +999,999,999].
+   * Returns true if the given number of seconds and nanos is a valid {@link Duration}. The {@code
+   * seconds} value must be in the range [-315,576,000,000, +315,576,000,000]. The {@code nanos}
+   * value must be in the range [-999,999,999, +999,999,999].
    *
-   * <p>Note: Durations less than one second are represented with a 0 {@code seconds} field and a
-   * positive or negative {@code nanos} field. For durations of one second or more, a non-zero value
-   * for the {@code nanos} field must be of the same sign as the {@code seconds} field.
+   * <p><b>Note:</b> Durations less than one second are represented with a 0 {@code seconds} field
+   * and a positive or negative {@code nanos} field. For durations of one second or more, a non-zero
+   * value for the {@code nanos} field must be of the same sign as the {@code seconds} field.
    */
-  public static boolean isValid(long seconds, long nanos) {
+  public static boolean isValid(long seconds, int nanos) {
     if (seconds < DURATION_SECONDS_MIN || seconds > DURATION_SECONDS_MAX) {
       return false;
     }
@@ -88,35 +124,35 @@ public final class Durations {
     return true;
   }
 
-  /**
-   * Throws an {@link IllegalArgumentException} if the given seconds/nanos are not
-   * a valid {@link Duration}.
-   */
-  private static void checkValid(long seconds, int nanos) {
-    if (!isValid(seconds, nanos)) {
-      throw new IllegalArgumentException(String.format(
-          "Duration is not valid. See proto definition for valid values. "
-          + "Seconds (%s) must be in range [-315,576,000,000, +315,576,000,000]."
-          + "Nanos (%s) must be in range [-999,999,999, +999,999,999]. "
-          + "Nanos must have the same sign as seconds", seconds, nanos));
-    }
+  /** Throws an {@link IllegalArgumentException} if the given {@link Duration} is not valid. */
+  public static Duration checkValid(Duration duration) {
+    long seconds = duration.getSeconds();
+    int nanos = duration.getNanos();
+    checkArgument(
+        isValid(seconds, nanos),
+        "Duration is not valid. See proto definition for valid values. "
+            + "Seconds (%s) must be in range [-315,576,000,000, +315,576,000,000]. "
+            + "Nanos (%s) must be in range [-999,999,999, +999,999,999]. "
+            + "Nanos must have the same sign as seconds",
+        seconds,
+        nanos);
+    return duration;
   }
 
   /**
-   * Convert Duration to string format. The string format will contains 3, 6,
-   * or 9 fractional digits depending on the precision required to represent
-   * the exact Duration value. For example: "1s", "1.010s", "1.000000100s",
-   * "-3.100s" The range that can be represented by Duration is from
+   * Convert Duration to string format. The string format will contains 3, 6, or 9 fractional digits
+   * depending on the precision required to represent the exact Duration value. For example: "1s",
+   * "1.010s", "1.000000100s", "-3.100s" The range that can be represented by Duration is from
    * -315,576,000,000 to +315,576,000,000 inclusive (in seconds).
    *
    * @return The string representation of the given duration.
-   * @throws IllegalArgumentException if the given duration is not in the valid
-   *         range.
+   * @throws IllegalArgumentException if the given duration is not in the valid range.
    */
   public static String toString(Duration duration) {
+    checkValid(duration);
+
     long seconds = duration.getSeconds();
     int nanos = duration.getNanos();
-    checkValid(seconds, nanos);
 
     StringBuilder result = new StringBuilder();
     if (seconds < 0 || nanos < 0) {
@@ -172,9 +208,20 @@ public final class Durations {
     }
   }
 
+  /** Create a Duration from the number of seconds. */
+  public static Duration fromSeconds(long seconds) {
+    return normalizedDuration(seconds, 0);
+  }
+
   /**
-   * Create a Duration from the number of milliseconds.
+   * Convert a Duration to the number of seconds. The result will be rounded towards 0 to the
+   * nearest second. E.g., if the duration represents -1 nanosecond, it will be rounded to 0.
    */
+  public static long toSeconds(Duration duration) {
+    return checkValid(duration).getSeconds();
+  }
+
+  /** Create a Duration from the number of milliseconds. */
   public static Duration fromMillis(long milliseconds) {
     return normalizedDuration(
         milliseconds / MILLIS_PER_SECOND,
@@ -182,17 +229,17 @@ public final class Durations {
   }
 
   /**
-   * Convert a Duration to the number of milliseconds.The result will be
-   * rounded towards 0 to the nearest millisecond. E.g., if the duration
-   * represents -1 nanosecond, it will be rounded to 0.
+   * Convert a Duration to the number of milliseconds. The result will be rounded towards 0 to the
+   * nearest millisecond. E.g., if the duration represents -1 nanosecond, it will be rounded to 0.
    */
   public static long toMillis(Duration duration) {
-    return duration.getSeconds() * MILLIS_PER_SECOND + duration.getNanos() / NANOS_PER_MILLISECOND;
+    checkValid(duration);
+    return checkedAdd(
+        checkedMultiply(duration.getSeconds(), MILLIS_PER_SECOND),
+        duration.getNanos() / NANOS_PER_MILLISECOND);
   }
 
-  /**
-   * Create a Duration from the number of microseconds.
-   */
+  /** Create a Duration from the number of microseconds. */
   public static Duration fromMicros(long microseconds) {
     return normalizedDuration(
         microseconds / MICROS_PER_SECOND,
@@ -200,57 +247,60 @@ public final class Durations {
   }
 
   /**
-   * Convert a Duration to the number of microseconds.The result will be
-   * rounded towards 0 to the nearest microseconds. E.g., if the duration
-   * represents -1 nanosecond, it will be rounded to 0.
+   * Convert a Duration to the number of microseconds. The result will be rounded towards 0 to the
+   * nearest microseconds. E.g., if the duration represents -1 nanosecond, it will be rounded to 0.
    */
   public static long toMicros(Duration duration) {
-    return duration.getSeconds() * MICROS_PER_SECOND + duration.getNanos() / NANOS_PER_MICROSECOND;
+    checkValid(duration);
+    return checkedAdd(
+        checkedMultiply(duration.getSeconds(), MICROS_PER_SECOND),
+        duration.getNanos() / NANOS_PER_MICROSECOND);
   }
 
-  /**
-   * Create a Duration from the number of nanoseconds.
-   */
+  /** Create a Duration from the number of nanoseconds. */
   public static Duration fromNanos(long nanoseconds) {
     return normalizedDuration(
         nanoseconds / NANOS_PER_SECOND, (int) (nanoseconds % NANOS_PER_SECOND));
   }
 
-  /**
-   * Convert a Duration to the number of nanoseconds.
-   */
+  /** Convert a Duration to the number of nanoseconds. */
   public static long toNanos(Duration duration) {
-    return duration.getSeconds() * NANOS_PER_SECOND + duration.getNanos();
+    checkValid(duration);
+    return checkedAdd(
+        checkedMultiply(duration.getSeconds(), NANOS_PER_SECOND), duration.getNanos());
   }
 
-  /**
-   * Add two durations.
-   */
+  /** Add two durations. */
   public static Duration add(Duration d1, Duration d2) {
-    return normalizedDuration(d1.getSeconds() + d2.getSeconds(), d1.getNanos() + d2.getNanos());
+    checkValid(d1);
+    checkValid(d2);
+    return normalizedDuration(
+        checkedAdd(d1.getSeconds(), d2.getSeconds()), checkedAdd(d1.getNanos(), d2.getNanos()));
   }
 
-  /**
-   * Subtract a duration from another.
-   */
+  /** Subtract a duration from another. */
   public static Duration subtract(Duration d1, Duration d2) {
-    return normalizedDuration(d1.getSeconds() - d2.getSeconds(), d1.getNanos() - d2.getNanos());
+    checkValid(d1);
+    checkValid(d2);
+    return normalizedDuration(
+        checkedSubtract(d1.getSeconds(), d2.getSeconds()),
+        checkedSubtract(d1.getNanos(), d2.getNanos()));
   }
 
   static Duration normalizedDuration(long seconds, int nanos) {
     if (nanos <= -NANOS_PER_SECOND || nanos >= NANOS_PER_SECOND) {
-      seconds += nanos / NANOS_PER_SECOND;
+      seconds = checkedAdd(seconds, nanos / NANOS_PER_SECOND);
       nanos %= NANOS_PER_SECOND;
     }
     if (seconds > 0 && nanos < 0) {
-      nanos += NANOS_PER_SECOND;
-      seconds -= 1;
+      nanos += NANOS_PER_SECOND; // no overflow since nanos is negative (and we're adding)
+      seconds--; // no overflow since seconds is positive (and we're decrementing)
     }
     if (seconds < 0 && nanos > 0) {
-      nanos -= NANOS_PER_SECOND;
-      seconds += 1;
+      nanos -= NANOS_PER_SECOND; // no overflow since nanos is positive (and we're subtracting)
+      seconds++; // no overflow since seconds is negative (and we're incrementing)
     }
-    checkValid(seconds, nanos);
-    return Duration.newBuilder().setSeconds(seconds).setNanos(nanos).build();
+    Duration duration = Duration.newBuilder().setSeconds(seconds).setNanos(nanos).build();
+    return checkValid(duration);
   }
 }

java/util/src/main/java/com/google/protobuf/util/JsonFormat.java

@@ -640,6 +640,10 @@ public class JsonFormat {
 
     /** Prints google.protobuf.Any */
     private void printAny(MessageOrBuilder message) throws IOException {
+      if (Any.getDefaultInstance().equals(message)) {
+        generator.print("{}");
+        return;
+      }
       Descriptor descriptor = message.getDescriptorForType();
       FieldDescriptor typeUrlField = descriptor.findFieldByName("type_url");
       FieldDescriptor valueField = descriptor.findFieldByName("value");
@@ -1235,6 +1239,9 @@ public class JsonFormat {
         throw new InvalidProtocolBufferException("Expect message object but got: " + json);
       }
       JsonObject object = (JsonObject) json;
+      if (object.entrySet().isEmpty()) {
+        return; // builder never modified, so it will end up building the default instance of Any
+      }
       JsonElement typeUrlElement = object.get("@type");
       if (typeUrlElement == null) {
         throw new InvalidProtocolBufferException("Missing type url when parsing: " + json);
@@ -1327,6 +1334,9 @@ public class JsonFormat {
         Message.Builder listBuilder = builder.newBuilderForField(field);
         merge(json, listBuilder);
         builder.setField(field, listBuilder.build());
+      } else if (json instanceof JsonNull) {
+        builder.setField(
+            type.findFieldByName("null_value"), NullValue.NULL_VALUE.getValueDescriptor());
       } else {
         throw new IllegalStateException("Unexpected json data: " + json);
       }
@@ -1620,11 +1630,6 @@ public class JsonFormat {
     }
 
     private ByteString parseBytes(JsonElement json) throws InvalidProtocolBufferException {
-      String encoded = json.getAsString();
-      if (encoded.length() % 4 != 0) {
-        throw new InvalidProtocolBufferException(
-            "Bytes field is not encoded in standard BASE64 with paddings: " + encoded);
-      }
       return ByteString.copyFrom(BaseEncoding.base64().decode(json.getAsString()));
     }
 

java/util/src/main/java/com/google/protobuf/util/Timestamps.java

@@ -30,19 +30,29 @@
 
 package com.google.protobuf.util;
 
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.math.IntMath.checkedAdd;
+import static com.google.common.math.IntMath.checkedSubtract;
+import static com.google.common.math.LongMath.checkedAdd;
+import static com.google.common.math.LongMath.checkedMultiply;
+import static com.google.common.math.LongMath.checkedSubtract;
+
+import com.google.common.collect.ComparisonChain;
 import com.google.protobuf.Duration;
 import com.google.protobuf.Timestamp;
-
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
+import java.util.Comparator;
 import java.util.Date;
 import java.util.GregorianCalendar;
 import java.util.TimeZone;
 
 /**
- * Utilities to help create/manipulate {@code protobuf/timestamp.proto}.
+ * Utilities to help create/manipulate {@code protobuf/timestamp.proto}. All operations throw an
+ * {@link IllegalArgumentException} if the input(s) are not {@linkplain #isValid(Timestamp) valid}.
  */
 public final class Timestamps {
+
   // Timestamp for "0001-01-01T00:00:00Z"
   static final long TIMESTAMP_SECONDS_MIN = -62135596800L;
 
@@ -55,10 +65,19 @@ public final class Timestamps {
   static final long MILLIS_PER_SECOND = 1000;
   static final long MICROS_PER_SECOND = 1000000;
 
-  // TODO(kak): Do we want to expose Timestamp constants for MAX/MIN?
+  /** A constant holding the minimum valid {@link Timestamp}, {@code 0001-01-01T00:00:00Z}. */
+  public static final Timestamp MIN_VALUE =
+      Timestamp.newBuilder().setSeconds(TIMESTAMP_SECONDS_MIN).setNanos(0).build();
+
+  /**
+   * A constant holding the maximum valid {@link Timestamp}, {@code 9999-12-31T23:59:59.999999999Z}.
+   */
+  public static final Timestamp MAX_VALUE =
+      Timestamp.newBuilder().setSeconds(TIMESTAMP_SECONDS_MAX).setNanos(999999999).build();
 
   private static final ThreadLocal<SimpleDateFormat> timestampFormat =
       new ThreadLocal<SimpleDateFormat>() {
+        @Override
         protected SimpleDateFormat initialValue() {
           return createTimestampFormat();
         }
@@ -76,28 +95,50 @@ public final class Timestamps {
 
   private Timestamps() {}
 
+  private static final Comparator<Timestamp> COMPARATOR =
+      new Comparator<Timestamp>() {
+        @Override
+        public int compare(Timestamp t1, Timestamp t2) {
+          checkValid(t1);
+          checkValid(t2);
+
+          return ComparisonChain.start()
+              .compare(t1.getSeconds(), t2.getSeconds())
+              .compare(t1.getNanos(), t2.getNanos())
+              .result();
+        }
+      };
+
+  /**
+   * Returns a {@link Comparator} for {@link Timestamp}s which sorts in increasing chronological
+   * order. Nulls and invalid {@link Timestamp}s are not allowed (see {@link #isValid}).
+   */
+  public static Comparator<Timestamp> comparator() {
+    return COMPARATOR;
+  }
+
   /**
    * Returns true if the given {@link Timestamp} is valid. The {@code seconds} value must be in the
    * range [-62,135,596,800, +253,402,300,799] (i.e., between 0001-01-01T00:00:00Z and
    * 9999-12-31T23:59:59Z). The {@code nanos} value must be in the range [0, +999,999,999].
    *
-   * <p>Note: Negative second values with fractions must still have non-negative nanos value that
-   * counts forward in time.
+   * <p><b>Note:</b> Negative second values with fractional seconds must still have non-negative
+   * nanos values that count forward in time.
    */
   public static boolean isValid(Timestamp timestamp) {
     return isValid(timestamp.getSeconds(), timestamp.getNanos());
   }
 
   /**
-   * Returns true if the given number of seconds and nanos is a valid {@link Timestamp}. The
-   * {@code seconds} value must be in the range [-62,135,596,800, +253,402,300,799] (i.e., between
+   * Returns true if the given number of seconds and nanos is a valid {@link Timestamp}. The {@code
+   * seconds} value must be in the range [-62,135,596,800, +253,402,300,799] (i.e., between
    * 0001-01-01T00:00:00Z and 9999-12-31T23:59:59Z). The {@code nanos} value must be in the range
    * [0, +999,999,999].
    *
-   * <p>Note: Negative second values with fractions must still have non-negative nanos value that
-   * counts forward in time.
+   * <p><b>Note:</b> Negative second values with fractional seconds must still have non-negative
+   * nanos values that count forward in time.
    */
-  public static boolean isValid(long seconds, long nanos) {
+  public static boolean isValid(long seconds, int nanos) {
     if (seconds < TIMESTAMP_SECONDS_MIN || seconds > TIMESTAMP_SECONDS_MAX) {
       return false;
     }
@@ -107,37 +148,37 @@ public final class Timestamps {
     return true;
   }
 
-  /**
-   * Throws an {@link IllegalArgumentException} if the given seconds/nanos are not
-   * a valid {@link Timestamp}.
-   */
-  private static void checkValid(long seconds, int nanos) {
-    if (!isValid(seconds, nanos)) {
-      throw new IllegalArgumentException(String.format(
-          "Timestamp is not valid. See proto definition for valid values. "
-          + "Seconds (%s) must be in range [-62,135,596,800, +253,402,300,799]."
-          + "Nanos (%s) must be in range [0, +999,999,999].",
-          seconds, nanos));
-    }
+  /** Throws an {@link IllegalArgumentException} if the given {@link Timestamp} is not valid. */
+  public static Timestamp checkValid(Timestamp timestamp) {
+    long seconds = timestamp.getSeconds();
+    int nanos = timestamp.getNanos();
+    checkArgument(
+        isValid(seconds, nanos),
+        "Timestamp is not valid. See proto definition for valid values. "
+            + "Seconds (%s) must be in range [-62,135,596,800, +253,402,300,799]. "
+            + "Nanos (%s) must be in range [0, +999,999,999].",
+        seconds,
+        nanos);
+    return timestamp;
   }
 
   /**
-   * Convert Timestamp to RFC 3339 date string format. The output will always
-   * be Z-normalized and uses 3, 6 or 9 fractional digits as required to
-   * represent the exact value. Note that Timestamp can only represent time
-   * from 0001-01-01T00:00:00Z to 9999-12-31T23:59:59.999999999Z. See
+   * Convert Timestamp to RFC 3339 date string format. The output will always be Z-normalized and
+   * uses 3, 6 or 9 fractional digits as required to represent the exact value. Note that Timestamp
+   * can only represent time from 0001-01-01T00:00:00Z to 9999-12-31T23:59:59.999999999Z. See
    * https://www.ietf.org/rfc/rfc3339.txt
    *
    * <p>Example of generated format: "1972-01-01T10:00:20.021Z"
    *
    * @return The string representation of the given timestamp.
-   * @throws IllegalArgumentException if the given timestamp is not in the
-   *         valid range.
+   * @throws IllegalArgumentException if the given timestamp is not in the valid range.
    */
   public static String toString(Timestamp timestamp) {
+    checkValid(timestamp);
+
     long seconds = timestamp.getSeconds();
     int nanos = timestamp.getNanos();
-    checkValid(seconds, nanos);
+
     StringBuilder result = new StringBuilder();
     // Format the seconds part.
     Date date = new Date(seconds * MILLIS_PER_SECOND);
@@ -152,10 +193,9 @@ public final class Timestamps {
   }
 
   /**
-   * Parse from RFC 3339 date string to Timestamp. This method accepts all
-   * outputs of {@link #toString(Timestamp)} and it also accepts any fractional
-   * digits (or none) and any offset as long as they fit into nano-seconds
-   * precision.
+   * Parse from RFC 3339 date string to Timestamp. This method accepts all outputs of {@link
+   * #toString(Timestamp)} and it also accepts any fractional digits (or none) and any offset as
+   * long as they fit into nano-seconds precision.
    *
    * <p>Example of accepted format: "1972-01-01T10:00:20.021-05:00"
    *
@@ -210,13 +250,26 @@ public final class Timestamps {
     try {
       return normalizedTimestamp(seconds, nanos);
     } catch (IllegalArgumentException e) {
-      throw new ParseException("Failed to parse timestmap: timestamp is out of range.", 0);
+      throw new ParseException("Failed to parse timestamp: timestamp is out of range.", 0);
     }
   }
 
+  /** Create a Timestamp from the number of seconds elapsed from the epoch. */
+  public static Timestamp fromSeconds(long seconds) {
+    return normalizedTimestamp(seconds, 0);
+  }
+
   /**
-   * Create a Timestamp from the number of milliseconds elapsed from the epoch.
+   * Convert a Timestamp to the number of seconds elapsed from the epoch.
+   *
+   * <p>The result will be rounded down to the nearest second. E.g., if the timestamp represents
+   * "1969-12-31T23:59:59.999999999Z", it will be rounded to -1 second.
    */
+  public static long toSeconds(Timestamp timestamp) {
+    return checkValid(timestamp).getSeconds();
+  }
+
+  /** Create a Timestamp from the number of milliseconds elapsed from the epoch. */
   public static Timestamp fromMillis(long milliseconds) {
     return normalizedTimestamp(
         milliseconds / MILLIS_PER_SECOND,
@@ -226,18 +279,17 @@ public final class Timestamps {
   /**
    * Convert a Timestamp to the number of milliseconds elapsed from the epoch.
    *
-   * <p>The result will be rounded down to the nearest millisecond. E.g., if the
-   * timestamp represents "1969-12-31T23:59:59.999999999Z", it will be rounded
-   * to -1 millisecond.
+   * <p>The result will be rounded down to the nearest millisecond. E.g., if the timestamp
+   * represents "1969-12-31T23:59:59.999999999Z", it will be rounded to -1 millisecond.
    */
   public static long toMillis(Timestamp timestamp) {
-    return timestamp.getSeconds() * MILLIS_PER_SECOND
-        + timestamp.getNanos() / NANOS_PER_MILLISECOND;
+    checkValid(timestamp);
+    return checkedAdd(
+        checkedMultiply(timestamp.getSeconds(), MILLIS_PER_SECOND),
+        timestamp.getNanos() / NANOS_PER_MILLISECOND);
   }
 
-  /**
-   * Create a Timestamp from the number of microseconds elapsed from the epoch.
-   */
+  /** Create a Timestamp from the number of microseconds elapsed from the epoch. */
   public static Timestamp fromMicros(long microseconds) {
     return normalizedTimestamp(
         microseconds / MICROS_PER_SECOND,
@@ -247,65 +299,67 @@ public final class Timestamps {
   /**
    * Convert a Timestamp to the number of microseconds elapsed from the epoch.
    *
-   * <p>The result will be rounded down to the nearest microsecond. E.g., if the
-   * timestamp represents "1969-12-31T23:59:59.999999999Z", it will be rounded
-   * to -1 millisecond.
+   * <p>The result will be rounded down to the nearest microsecond. E.g., if the timestamp
+   * represents "1969-12-31T23:59:59.999999999Z", it will be rounded to -1 millisecond.
    */
   public static long toMicros(Timestamp timestamp) {
-    return timestamp.getSeconds() * MICROS_PER_SECOND
-        + timestamp.getNanos() / NANOS_PER_MICROSECOND;
+    checkValid(timestamp);
+    return checkedAdd(
+        checkedMultiply(timestamp.getSeconds(), MICROS_PER_SECOND),
+        timestamp.getNanos() / NANOS_PER_MICROSECOND);
   }
 
-  /**
-   * Create a Timestamp from the number of nanoseconds elapsed from the epoch.
-   */
+  /** Create a Timestamp from the number of nanoseconds elapsed from the epoch. */
   public static Timestamp fromNanos(long nanoseconds) {
     return normalizedTimestamp(
         nanoseconds / NANOS_PER_SECOND, (int) (nanoseconds % NANOS_PER_SECOND));
   }
 
-  /**
-   * Convert a Timestamp to the number of nanoseconds elapsed from the epoch.
-   */
+  /** Convert a Timestamp to the number of nanoseconds elapsed from the epoch. */
   public static long toNanos(Timestamp timestamp) {
-    return timestamp.getSeconds() * NANOS_PER_SECOND + timestamp.getNanos();
+    checkValid(timestamp);
+    return checkedAdd(
+        checkedMultiply(timestamp.getSeconds(), NANOS_PER_SECOND), timestamp.getNanos());
   }
 
-  /**
-   * Calculate the difference between two timestamps.
-   */
+  /** Calculate the difference between two timestamps. */
   public static Duration between(Timestamp from, Timestamp to) {
+    checkValid(from);
+    checkValid(to);
     return Durations.normalizedDuration(
-        to.getSeconds() - from.getSeconds(), to.getNanos() - from.getNanos());
+        checkedSubtract(to.getSeconds(), from.getSeconds()),
+        checkedSubtract(to.getNanos(), from.getNanos()));
   }
 
-  /**
-   * Add a duration to a timestamp.
-   */
+  /** Add a duration to a timestamp. */
   public static Timestamp add(Timestamp start, Duration length) {
+    checkValid(start);
+    Durations.checkValid(length);
     return normalizedTimestamp(
-        start.getSeconds() + length.getSeconds(), start.getNanos() + length.getNanos());
+        checkedAdd(start.getSeconds(), length.getSeconds()),
+        checkedAdd(start.getNanos(), length.getNanos()));
   }
 
-  /**
-   * Subtract a duration from a timestamp.
-   */
+  /** Subtract a duration from a timestamp. */
   public static Timestamp subtract(Timestamp start, Duration length) {
+    checkValid(start);
+    Durations.checkValid(length);
     return normalizedTimestamp(
-        start.getSeconds() - length.getSeconds(), start.getNanos() - length.getNanos());
+        checkedSubtract(start.getSeconds(), length.getSeconds()),
+        checkedSubtract(start.getNanos(), length.getNanos()));
   }
 
-  private static Timestamp normalizedTimestamp(long seconds, int nanos) {
+  static Timestamp normalizedTimestamp(long seconds, int nanos) {
     if (nanos <= -NANOS_PER_SECOND || nanos >= NANOS_PER_SECOND) {
-      seconds += nanos / NANOS_PER_SECOND;
+      seconds = checkedAdd(seconds, nanos / NANOS_PER_SECOND);
       nanos %= NANOS_PER_SECOND;
     }
     if (nanos < 0) {
-      nanos += NANOS_PER_SECOND;
-      seconds -= 1;
+      nanos += NANOS_PER_SECOND; // no overflow since nanos is negative (and we're adding)
+      seconds = checkedSubtract(seconds, 1);
     }
-    checkValid(seconds, nanos);
-    return Timestamp.newBuilder().setSeconds(seconds).setNanos(nanos).build();
+    Timestamp timestamp = Timestamp.newBuilder().setSeconds(seconds).setNanos(nanos).build();
+    return checkValid(timestamp);
   }
 
   private static long parseTimezoneOffset(String value) throws ParseException {
@@ -324,7 +378,7 @@ public final class Timestamps {
       result = result * 10;
       if (i < value.length()) {
         if (value.charAt(i) < '0' || value.charAt(i) > '9') {
-          throw new ParseException("Invalid nanosecnds.", 0);
+          throw new ParseException("Invalid nanoseconds.", 0);
         }
         result += value.charAt(i) - '0';
       }
@@ -332,11 +386,8 @@ public final class Timestamps {
     return result;
   }
 
-  /**
-   * Format the nano part of a timestamp or a duration.
-   */
+  /** Format the nano part of a timestamp or a duration. */
   static String formatNanos(int nanos) {
-    assert nanos >= 1 && nanos <= 999999999;
     // Determine whether to use 3, 6, or 9 digits for the nano part.
     if (nanos % NANOS_PER_MILLISECOND == 0) {
       return String.format("%1$03d", nanos / NANOS_PER_MILLISECOND);

java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java

@@ -60,12 +60,10 @@ import com.google.protobuf.util.JsonTestProto.TestOneof;
 import com.google.protobuf.util.JsonTestProto.TestStruct;
 import com.google.protobuf.util.JsonTestProto.TestTimestamp;
 import com.google.protobuf.util.JsonTestProto.TestWrappers;
-
-import junit.framework.TestCase;
-
 import java.io.IOException;
 import java.math.BigDecimal;
 import java.math.BigInteger;
+import junit.framework.TestCase;
 
 public class JsonFormatTest extends TestCase {
   private void setAllFields(TestAllTypes.Builder builder) {
@@ -819,6 +817,15 @@ public class JsonFormatTest extends TestCase {
         printer.print(message));
     assertRoundTripEquals(message, registry);
 
+    TestAny messageWithDefaultAnyValue =
+        TestAny.newBuilder().setAnyValue(Any.getDefaultInstance()).build();
+    assertEquals(
+        "{\n"
+            + "  \"anyValue\": {}\n"
+            + "}",
+        printer.print(messageWithDefaultAnyValue));
+    assertRoundTripEquals(messageWithDefaultAnyValue, registry);
+
     // Well-known types have a special formatting when embedded in Any.
     //
     // 1. Any in Any.
@@ -952,6 +959,8 @@ public class JsonFormatTest extends TestCase {
             + "}",
         printer.print(anyMessage));
     assertRoundTripEquals(anyMessage, registry);
+
+    // 7. Value (number type) in Any
     Value.Builder valueBuilder = Value.newBuilder();
     valueBuilder.setNumberValue(1);
     anyMessage = Any.pack(valueBuilder.build());
@@ -962,6 +971,95 @@ public class JsonFormatTest extends TestCase {
             + "}",
         printer.print(anyMessage));
     assertRoundTripEquals(anyMessage, registry);
+
+    // 8. Value (null type) in Any
+    anyMessage = Any.pack(Value.newBuilder().setNullValue(NullValue.NULL_VALUE).build());
+    assertEquals(
+        "{\n"
+            + "  \"@type\": \"type.googleapis.com/google.protobuf.Value\",\n"
+            + "  \"value\": null\n"
+            + "}",
+        printer.print(anyMessage));
+    assertRoundTripEquals(anyMessage, registry);
+  }
+
+  public void testAnyInMaps() throws Exception {
+    JsonFormat.TypeRegistry registry =
+        JsonFormat.TypeRegistry.newBuilder().add(TestAllTypes.getDescriptor()).build();
+    JsonFormat.Printer printer = JsonFormat.printer().usingTypeRegistry(registry);
+
+    TestAny.Builder testAny = TestAny.newBuilder();
+    testAny.putAnyMap("int32_wrapper", Any.pack(Int32Value.newBuilder().setValue(123).build()));
+    testAny.putAnyMap("int64_wrapper", Any.pack(Int64Value.newBuilder().setValue(456).build()));
+    testAny.putAnyMap("timestamp", Any.pack(Timestamps.parse("1969-12-31T23:59:59Z")));
+    testAny.putAnyMap("duration", Any.pack(Durations.parse("12345.1s")));
+    testAny.putAnyMap("field_mask", Any.pack(FieldMaskUtil.fromString("foo.bar,baz")));
+    Value numberValue = Value.newBuilder().setNumberValue(1.125).build();
+    Struct.Builder struct = Struct.newBuilder();
+    struct.putFields("number", numberValue);
+    testAny.putAnyMap("struct", Any.pack(struct.build()));
+    Value nullValue = Value.newBuilder().setNullValue(NullValue.NULL_VALUE).build();
+    testAny.putAnyMap(
+        "list_value",
+        Any.pack(ListValue.newBuilder().addValues(numberValue).addValues(nullValue).build()));
+    testAny.putAnyMap("number_value", Any.pack(numberValue));
+    testAny.putAnyMap("any_value_number", Any.pack(Any.pack(numberValue)));
+    testAny.putAnyMap("any_value_default", Any.pack(Any.getDefaultInstance()));
+    testAny.putAnyMap("default", Any.getDefaultInstance());
+
+    assertEquals(
+        "{\n"
+            + "  \"anyMap\": {\n"
+            + "    \"int32_wrapper\": {\n"
+            + "      \"@type\": \"type.googleapis.com/google.protobuf.Int32Value\",\n"
+            + "      \"value\": 123\n"
+            + "    },\n"
+            + "    \"int64_wrapper\": {\n"
+            + "      \"@type\": \"type.googleapis.com/google.protobuf.Int64Value\",\n"
+            + "      \"value\": \"456\"\n"
+            + "    },\n"
+            + "    \"timestamp\": {\n"
+            + "      \"@type\": \"type.googleapis.com/google.protobuf.Timestamp\",\n"
+            + "      \"value\": \"1969-12-31T23:59:59Z\"\n"
+            + "    },\n"
+            + "    \"duration\": {\n"
+            + "      \"@type\": \"type.googleapis.com/google.protobuf.Duration\",\n"
+            + "      \"value\": \"12345.100s\"\n"
+            + "    },\n"
+            + "    \"field_mask\": {\n"
+            + "      \"@type\": \"type.googleapis.com/google.protobuf.FieldMask\",\n"
+            + "      \"value\": \"foo.bar,baz\"\n"
+            + "    },\n"
+            + "    \"struct\": {\n"
+            + "      \"@type\": \"type.googleapis.com/google.protobuf.Struct\",\n"
+            + "      \"value\": {\n"
+            + "        \"number\": 1.125\n"
+            + "      }\n"
+            + "    },\n"
+            + "    \"list_value\": {\n"
+            + "      \"@type\": \"type.googleapis.com/google.protobuf.ListValue\",\n"
+            + "      \"value\": [1.125, null]\n"
+            + "    },\n"
+            + "    \"number_value\": {\n"
+            + "      \"@type\": \"type.googleapis.com/google.protobuf.Value\",\n"
+            + "      \"value\": 1.125\n"
+            + "    },\n"
+            + "    \"any_value_number\": {\n"
+            + "      \"@type\": \"type.googleapis.com/google.protobuf.Any\",\n"
+            + "      \"value\": {\n"
+            + "        \"@type\": \"type.googleapis.com/google.protobuf.Value\",\n"
+            + "        \"value\": 1.125\n"
+            + "      }\n"
+            + "    },\n"
+            + "    \"any_value_default\": {\n"
+            + "      \"@type\": \"type.googleapis.com/google.protobuf.Any\",\n"
+            + "      \"value\": {}\n"
+            + "    },\n"
+            + "    \"default\": {}\n"
+            + "  }\n"
+            + "}",
+        printer.print(testAny.build()));
+    assertRoundTripEquals(testAny.build(), registry);
   }
 
   public void testParserMissingTypeUrl() throws Exception {
@@ -1016,8 +1114,10 @@ public class JsonFormatTest extends TestCase {
 
   public void testParserRejectInvalidBase64() throws Exception {
     assertRejects("optionalBytes", "!@#$");
-    // We use standard BASE64 with paddings.
-    assertRejects("optionalBytes", "AQI");
+  }
+
+  public void testParserAcceptBase64Variants() throws Exception {
+    assertAccepts("optionalBytes", "AQI");
   }
 
   public void testParserRejectInvalidEnumValue() throws Exception {

java/util/src/test/java/com/google/protobuf/util/TimeUtilTest.java

@@ -32,14 +32,11 @@ package com.google.protobuf.util;
 
 import com.google.protobuf.Duration;
 import com.google.protobuf.Timestamp;
-
-import junit.framework.TestCase;
-
-import org.junit.Assert;
-
 import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.List;
+import junit.framework.TestCase;
+import org.junit.Assert;
 
 /** Unit tests for {@link TimeUtil}. */
 public class TimeUtilTest extends TestCase {

java/util/src/test/proto/com/google/protobuf/util/json_test.proto

@@ -195,6 +195,7 @@ message TestStruct {
 
 message TestAny {
   google.protobuf.Any any_value = 1;
+  map<string, google.protobuf.Any> any_map = 2;
 }
 
 message TestCustomJsonName {

js/binary/proto_test.js

@@ -496,7 +496,7 @@ describe('protoBinaryTest', function() {
     msg.setRepeatedBytesList([BYTES_B64, BYTES_B64]);
     assertGetters();
 
-    msg.setRepeatedBytesList(null);
+    msg.setRepeatedBytesList([]);
     assertEquals(0, msg.getRepeatedBytesList().length);
     assertEquals(0, msg.getRepeatedBytesList_asB64().length);
     assertEquals(0, msg.getRepeatedBytesList_asU8().length);

js/debug.js

@@ -95,8 +95,7 @@ jspb.debug.dump_ = function(thing) {
     if (match && name != 'getExtension' &&
         name != 'getJsPbMessageId') {
       var has = 'has' + match[1];
-      if (!thing[has] || thing[has]())
-      {
+      if (!thing[has] || thing[has]()) {
         var val = thing[name]();
         object[jspb.debug.formatFieldName_(match[1])] = jspb.debug.dump_(val);
       }

js/debug_test.js

@@ -65,7 +65,7 @@ describe('debugTest', function() {
       'aBoolean': true
     }, jspb.debug.dump(message));
 
-    message.setAString(undefined);
+    message.clearAString();
 
     assertObjectEquals({
       $name: 'proto.jspb.test.Simple1',

js/map.js

@@ -103,7 +103,10 @@ jspb.Map.prototype.toArray = function() {
       var m = this.map_;
       for (var p in m) {
         if (Object.prototype.hasOwnProperty.call(m, p)) {
-          m[p].valueWrapper.toArray();
+          var valueWrapper = /** @type {?jspb.Message} */ (m[p].valueWrapper);
+          if (valueWrapper) {
+            valueWrapper.toArray();
+          }
         }
       }
     }
@@ -343,13 +346,13 @@ jspb.Map.prototype.has = function(key) {
  * number.
  * @param {number} fieldNumber
  * @param {!jspb.BinaryWriter} writer
- * @param {function(this:jspb.BinaryWriter,number,K)=} keyWriterFn
+ * @param {!function(this:jspb.BinaryWriter,number,K)} keyWriterFn
  *     The method on BinaryWriter that writes type K to the stream.
- * @param {function(this:jspb.BinaryWriter,number,V)|
- *         function(this:jspb.BinaryReader,V,?)=} valueWriterFn
+ * @param {!function(this:jspb.BinaryWriter,number,V)|
+ *          function(this:jspb.BinaryReader,V,?)} valueWriterFn
  *     The method on BinaryWriter that writes type V to the stream.  May be
  *     writeMessage, in which case the second callback arg form is used.
- * @param {?function(V,!jspb.BinaryWriter)=} opt_valueWriterCallback
+ * @param {function(V,!jspb.BinaryWriter)=} opt_valueWriterCallback
  *    The BinaryWriter serialization callback for type V, if V is a message
  *    type.
  */
@@ -376,14 +379,15 @@ jspb.Map.prototype.serializeBinary = function(
  * Read one key/value message from the given BinaryReader. Compatible as the
  * `reader` callback parameter to jspb.BinaryReader.readMessage, to be called
  * when a key/value pair submessage is encountered.
+ * @template K, V
  * @param {!jspb.Map} map
  * @param {!jspb.BinaryReader} reader
- * @param {function(this:jspb.BinaryReader):K=} keyReaderFn
+ * @param {!function(this:jspb.BinaryReader):K} keyReaderFn
  *     The method on BinaryReader that reads type K from the stream.
  *
- * @param {function(this:jspb.BinaryReader):V|
- *         function(this:jspb.BinaryReader,V,
- *                  function(V,!jspb.BinaryReader))=} valueReaderFn
+ * @param {!function(this:jspb.BinaryReader):V|
+ *          function(this:jspb.BinaryReader,V,
+ *                  function(V,!jspb.BinaryReader))} valueReaderFn
  *    The method on BinaryReader that reads type V from the stream. May be
  *    readMessage, in which case the second callback arg form is used.
  *

js/maps_test.js

@@ -0,0 +1,359 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+goog.require('goog.testing.asserts');
+goog.require('goog.userAgent');
+goog.require('proto.jspb.test.MapValueEnum');
+goog.require('proto.jspb.test.MapValueMessage');
+goog.require('proto.jspb.test.MapValueMessageNoBinary');
+goog.require('proto.jspb.test.TestMapFields');
+goog.require('proto.jspb.test.TestMapFieldsNoBinary');
+
+/**
+ * Helper: check that the given map has exactly this set of (sorted) entries.
+ * @param {!jspb.Map} map
+ * @param {!Array<!Array<?>>} entries
+ */
+function checkMapEquals(map, entries) {
+  var arr = map.toArray();
+  assertEquals(arr.length, entries.length);
+  for (var i = 0; i < arr.length; i++) {
+    assertElementsEquals(arr[i], entries[i]);
+  }
+}
+
+/**
+ * Converts an ES6 iterator to an array.
+ * @template T
+ * @param {!Iterator<T>} iter an iterator
+ * @return {!Array<T>}
+ */
+function toArray(iter) {
+  var arr = [];
+  while (true) {
+    var val = iter.next();
+    if (val.done) {
+      break;
+    }
+    arr.push(val.value);
+  }
+  return arr;
+}
+
+
+/**
+ * Helper: generate test methods for this TestMapFields class.
+ * @param {?} msgInfo
+ * @param {?} submessageCtor
+ * @param {!string} suffix
+ */
+function makeTests(msgInfo, submessageCtor, suffix) {
+  /**
+   * Helper: fill all maps on a TestMapFields.
+   * @param {?} msg
+   */
+  var fillMapFields = function(msg) {
+    msg.getMapStringStringMap().set('asdf', 'jkl;').set('key 2', 'hello world');
+    msg.getMapStringInt32Map().set('a', 1).set('b', -2);
+    msg.getMapStringInt64Map().set('c', 0x100000000).set('d', 0x200000000);
+    msg.getMapStringBoolMap().set('e', true).set('f', false);
+    msg.getMapStringDoubleMap().set('g', 3.14159).set('h', 2.71828);
+    msg.getMapStringEnumMap()
+        .set('i', proto.jspb.test.MapValueEnum.MAP_VALUE_BAR)
+        .set('j', proto.jspb.test.MapValueEnum.MAP_VALUE_BAZ);
+    msg.getMapStringMsgMap()
+        .set('k', new submessageCtor())
+        .set('l', new submessageCtor());
+    msg.getMapStringMsgMap().get('k').setFoo(42);
+    msg.getMapStringMsgMap().get('l').setFoo(84);
+    msg.getMapInt32StringMap().set(-1, 'a').set(42, 'b');
+    msg.getMapInt64StringMap().set(0x123456789abc, 'c').set(0xcba987654321, 'd');
+    msg.getMapBoolStringMap().set(false, 'e').set(true, 'f');
+  };
+
+  /**
+   * Helper: check all maps on a TestMapFields.
+   * @param {?} msg
+   */
+  var checkMapFields = function(msg) {
+    checkMapEquals(msg.getMapStringStringMap(), [
+          ['asdf', 'jkl;'],
+          ['key 2', 'hello world']
+    ]);
+    checkMapEquals(msg.getMapStringInt32Map(), [
+          ['a', 1],
+          ['b', -2]
+    ]);
+    checkMapEquals(msg.getMapStringInt64Map(), [
+          ['c', 0x100000000],
+          ['d', 0x200000000]
+    ]);
+    checkMapEquals(msg.getMapStringBoolMap(), [
+          ['e', true],
+          ['f', false]
+    ]);
+    checkMapEquals(msg.getMapStringDoubleMap(), [
+          ['g', 3.14159],
+          ['h', 2.71828]
+    ]);
+    checkMapEquals(msg.getMapStringEnumMap(), [
+          ['i', proto.jspb.test.MapValueEnum.MAP_VALUE_BAR],
+          ['j', proto.jspb.test.MapValueEnum.MAP_VALUE_BAZ]
+    ]);
+    checkMapEquals(msg.getMapInt32StringMap(), [
+          [-1, 'a'],
+          [42, 'b']
+    ]);
+    checkMapEquals(msg.getMapInt64StringMap(), [
+          [0x123456789abc, 'c'],
+          [0xcba987654321, 'd']
+    ]);
+    checkMapEquals(msg.getMapBoolStringMap(), [
+          [false, 'e'],
+          [true, 'f']
+    ]);
+
+    assertEquals(msg.getMapStringMsgMap().getLength(), 2);
+    assertEquals(msg.getMapStringMsgMap().get('k').getFoo(), 42);
+    assertEquals(msg.getMapStringMsgMap().get('l').getFoo(), 84);
+
+    var entries = toArray(msg.getMapStringMsgMap().entries());
+    assertEquals(entries.length, 2);
+    entries.forEach(function(entry) {
+      var key = entry[0];
+      var val = entry[1];
+      assert(val === msg.getMapStringMsgMap().get(key));
+    });
+
+    msg.getMapStringMsgMap().forEach(function(val, key) {
+      assert(val === msg.getMapStringMsgMap().get(key));
+    });
+  };
+
+  it('testMapStringStringField' + suffix, function() {
+    var msg = new msgInfo.constructor();
+    assertEquals(msg.getMapStringStringMap().getLength(), 0);
+    assertEquals(msg.getMapStringInt32Map().getLength(), 0);
+    assertEquals(msg.getMapStringInt64Map().getLength(), 0);
+    assertEquals(msg.getMapStringBoolMap().getLength(), 0);
+    assertEquals(msg.getMapStringDoubleMap().getLength(), 0);
+    assertEquals(msg.getMapStringEnumMap().getLength(), 0);
+    assertEquals(msg.getMapStringMsgMap().getLength(), 0);
+
+    // Re-create to clear out any internally-cached wrappers, etc.
+    msg = new msgInfo.constructor();
+    var m = msg.getMapStringStringMap();
+    assertEquals(m.has('asdf'), false);
+    assertEquals(m.get('asdf'), undefined);
+    m.set('asdf', 'hello world');
+    assertEquals(m.has('asdf'), true);
+    assertEquals(m.get('asdf'), 'hello world');
+    m.set('jkl;', 'key 2');
+    assertEquals(m.has('jkl;'), true);
+    assertEquals(m.get('jkl;'), 'key 2');
+    assertEquals(m.getLength(), 2);
+    var it = m.entries();
+    assertElementsEquals(it.next().value, ['asdf', 'hello world']);
+    assertElementsEquals(it.next().value, ['jkl;', 'key 2']);
+    assertEquals(it.next().done, true);
+    checkMapEquals(m, [
+        ['asdf', 'hello world'],
+        ['jkl;', 'key 2']
+    ]);
+    m.del('jkl;');
+    assertEquals(m.has('jkl;'), false);
+    assertEquals(m.get('jkl;'), undefined);
+    assertEquals(m.getLength(), 1);
+    it = m.keys();
+    assertEquals(it.next().value, 'asdf');
+    assertEquals(it.next().done, true);
+    it = m.values();
+    assertEquals(it.next().value, 'hello world');
+    assertEquals(it.next().done, true);
+
+    var count = 0;
+    m.forEach(function(value, key, map) {
+      assertEquals(map, m);
+      assertEquals(key, 'asdf');
+      assertEquals(value, 'hello world');
+      count++;
+    });
+    assertEquals(count, 1);
+
+    m.clear();
+    assertEquals(m.getLength(), 0);
+  });
+
+
+  /**
+   * Tests operations on maps with all key and value types.
+   */
+  it('testAllMapTypes' + suffix, function() {
+    var msg = new msgInfo.constructor();
+    fillMapFields(msg);
+    checkMapFields(msg);
+  });
+
+
+  if (msgInfo.deserializeBinary) {
+    /**
+     * Tests serialization and deserialization in binary format.
+     */
+    it('testBinaryFormat' + suffix, function() {
+      if (goog.userAgent.IE && !goog.userAgent.isDocumentModeOrHigher(10)) {
+        // IE8/9 currently doesn't support binary format because they lack
+        // TypedArray.
+        return;
+      }
+
+      // Check that the format is correct.
+      var msg = new msgInfo.constructor();
+      msg.getMapStringStringMap().set('A', 'a');
+      var serialized = msg.serializeBinary();
+      var expectedSerialized = [
+          0x0a, 0x6, // field 1 (map_string_string), delimited, length 6
+          0x0a, 0x1, // field 1 in submessage (key), delimited, length 1
+          0x41,      // ASCII 'A'
+          0x12, 0x1, // field 2 in submessage (value), delimited, length 1
+          0x61       // ASCII 'a'
+      ];
+      assertEquals(serialized.length, expectedSerialized.length);
+      for (var i = 0; i < serialized.length; i++) {
+        assertEquals(serialized[i], expectedSerialized[i]);
+      }
+
+      // Check that all map fields successfully round-trip.
+      msg = new msgInfo.constructor();
+      fillMapFields(msg);
+      serialized = msg.serializeBinary();
+      var decoded = msgInfo.deserializeBinary(serialized);
+      checkMapFields(decoded);
+    });
+  }
+
+
+  /**
+   * Tests serialization and deserialization in JSPB format.
+   */
+  it('testJSPBFormat' + suffix, function() {
+    var msg = new msgInfo.constructor();
+    fillMapFields(msg);
+    var serialized = msg.serialize();
+    var decoded = msgInfo.deserialize(serialized);
+    checkMapFields(decoded);
+  });
+
+  /**
+   * Tests serialization and deserialization in JSPB format, when there is
+   * a submessage that also contains map entries.  This tests recursive
+   * sync.
+   */
+  it('testJSPBFormatNested' + suffix, function() {
+    var submsg = new msgInfo.constructor();
+    var mapValue = new msgInfo.constructor();
+    var msg = new msgInfo.constructor();
+
+    msg.getMapStringTestmapfieldsMap().set('test', mapValue);
+    msg.setTestMapFields(submsg);
+
+    fillMapFields(submsg);
+    fillMapFields(msg);
+    fillMapFields(mapValue);
+
+    var serialized = msg.serialize();
+
+    var decoded = msgInfo.deserialize(serialized);
+    checkMapFields(decoded);
+
+    var decodedSubmsg = decoded.getTestMapFields();
+    assertNotNull(decodedSubmsg);
+    checkMapFields(decodedSubmsg);
+
+    var decodedMapValue = decoded.getMapStringTestmapfieldsMap().get('test');
+    assertNotNull(decodedMapValue);
+    checkMapFields(decodedMapValue);
+  });
+
+  /**
+   * Tests toObject()/fromObject().
+   */
+  it('testToFromObject' + suffix, function() {
+    var msg = new msgInfo.constructor();
+    fillMapFields(msg);
+    var obj = msg.toObject();
+    var decoded = msgInfo.fromObject(obj);
+    checkMapFields(decoded);
+    obj = msgInfo.deserialize(msg.serialize()).toObject();
+    decoded = msgInfo.fromObject(obj);
+    checkMapFields(decoded);
+  });
+
+
+  /**
+   * Exercises the lazy map<->underlying array sync.
+   */
+  it('testLazyMapSync' + suffix, function() {
+    // Start with a JSPB array containing a few map entries.
+    var entries = [
+        ['a', 'entry 1'],
+        ['c', 'entry 2'],
+        ['b', 'entry 3']
+    ];
+    var msg = new msgInfo.constructor([entries]);
+    assertEquals(entries.length, 3);
+    assertEquals(entries[0][0], 'a');
+    assertEquals(entries[1][0], 'c');
+    assertEquals(entries[2][0], 'b');
+    msg.getMapStringStringMap().del('a');
+    assertEquals(entries.length, 3);  // not yet sync'd
+    msg.toArray();                // force a sync
+    assertEquals(entries.length, 2);
+    assertEquals(entries[0][0], 'b'); // now in sorted order
+    assertEquals(entries[1][0], 'c');
+
+    var a = msg.toArray();
+    assertEquals(a[0], entries);  // retains original reference
+  });
+}
+
+describe('mapsTest', function() {
+  makeTests({
+    constructor: proto.jspb.test.TestMapFields,
+    fromObject: proto.jspb.test.TestMapFields.fromObject,
+    deserialize: proto.jspb.test.TestMapFields.deserialize,
+    deserializeBinary: proto.jspb.test.TestMapFields.deserializeBinary
+  }, proto.jspb.test.MapValueMessage, "_Binary");
+  makeTests({
+    constructor: proto.jspb.test.TestMapFieldsNoBinary,
+    fromObject: proto.jspb.test.TestMapFieldsNoBinary.fromObject,
+    deserialize: proto.jspb.test.TestMapFieldsNoBinary.deserialize,
+    deserializeBinary: null
+  }, proto.jspb.test.MapValueMessageNoBinary, "_NoBinary");
+});

js/message.js

@@ -106,17 +106,17 @@ jspb.ExtensionFieldInfo = function(fieldNumber, fieldName, ctor, toObjectFn,
 /**
  * Stores binary-related information for a single extension field.
  * @param {!jspb.ExtensionFieldInfo<T>} fieldInfo
- * @param {?function(number,?)=} binaryReaderFn
- * @param {?function(number,?)|function(number,?,?,?,?,?)=} binaryWriterFn
- * @param {?function(?,?)=} opt_binaryMessageSerializeFn
- * @param {?function(?,?)=} opt_binaryMessageDeserializeFn
- * @param {?boolean=} opt_isPacked
+ * @param {!function(number,?)} binaryReaderFn
+ * @param {!function(number,?)|function(number,?,?,?,?,?)} binaryWriterFn
+ * @param {function(?,?)=} opt_binaryMessageSerializeFn
+ * @param {function(?,?)=} opt_binaryMessageDeserializeFn
+ * @param {boolean=} opt_isPacked
  * @constructor
  * @struct
  * @template T
  */
 jspb.ExtensionFieldBinaryInfo = function(fieldInfo, binaryReaderFn, binaryWriterFn,
-    binaryMessageSerializeFn, binaryMessageDeserializeFn, isPacked) {
+    opt_binaryMessageSerializeFn, opt_binaryMessageDeserializeFn, opt_isPacked) {
   /** @const */
   this.fieldInfo = fieldInfo;
   /** @const */
@@ -124,11 +124,11 @@ jspb.ExtensionFieldBinaryInfo = function(fieldInfo, binaryReaderFn, binaryWriter
   /** @const */
   this.binaryWriterFn = binaryWriterFn;
   /** @const */
-  this.binaryMessageSerializeFn = binaryMessageSerializeFn;
+  this.binaryMessageSerializeFn = opt_binaryMessageSerializeFn;
   /** @const */
-  this.binaryMessageDeserializeFn = binaryMessageDeserializeFn;
+  this.binaryMessageDeserializeFn = opt_binaryMessageDeserializeFn;
   /** @const */
-  this.isPacked = isPacked;
+  this.isPacked = opt_isPacked;
 };
 
 /**
@@ -744,7 +744,7 @@ jspb.Message.assertConsistentTypes_ = function(array) {
  * @return {T} The field's value.
  * @protected
  */
-jspb.Message.getFieldProto3 = function(msg, fieldNumber, defaultValue) {
+jspb.Message.getFieldWithDefault = function(msg, fieldNumber, defaultValue) {
   var value = jspb.Message.getField(msg, fieldNumber);
   if (value == null) {
     return defaultValue;
@@ -810,6 +810,24 @@ jspb.Message.setField = function(msg, fieldNumber, value) {
 };
 
 
+/**
+ * Adds a value to a repeated, primitive field.
+ * @param {!jspb.Message} msg A jspb proto.
+ * @param {number} fieldNumber The field number.
+ * @param {string|number|boolean|!Uint8Array} value New value
+ * @param {number=} opt_index Index where to put new value.
+ * @protected
+ */
+jspb.Message.addToRepeatedField = function(msg, fieldNumber, value, opt_index) {
+  var arr = jspb.Message.getField(msg, fieldNumber);
+  if (opt_index != undefined) {
+    arr.splice(opt_index, 0, value);
+  } else {
+    arr.push(value);
+  }
+};
+
+
 /**
  * Sets the value of a field in a oneof union and clears all other fields in
  * the union.
@@ -907,6 +925,24 @@ jspb.Message.getWrapperField = function(msg, ctor, fieldNumber, opt_required) {
  * @protected
  */
 jspb.Message.getRepeatedWrapperField = function(msg, ctor, fieldNumber) {
+  jspb.Message.wrapRepeatedField_(msg, ctor, fieldNumber);
+  var val = msg.wrappers_[fieldNumber];
+  if (val == jspb.Message.EMPTY_LIST_SENTINEL_) {
+    val = msg.wrappers_[fieldNumber] = [];
+  }
+  return /** @type {!Array<!jspb.Message>} */ (val);
+};
+
+
+/**
+ * Wraps underlying array into proto message representation if it wasn't done
+ * before.
+ * @param {!jspb.Message} msg A jspb proto.
+ * @param {function(new:jspb.Message, ?Array)} ctor Constructor for the field.
+ * @param {number} fieldNumber The field number.
+ * @private
+ */
+jspb.Message.wrapRepeatedField_ = function(msg, ctor, fieldNumber) {
   if (!msg.wrappers_) {
     msg.wrappers_ = {};
   }
@@ -917,11 +953,6 @@ jspb.Message.getRepeatedWrapperField = function(msg, ctor, fieldNumber) {
     }
     msg.wrappers_[fieldNumber] = wrappers;
   }
-  var val = msg.wrappers_[fieldNumber];
-  if (val == jspb.Message.EMPTY_LIST_SENTINEL_) {
-    val = msg.wrappers_[fieldNumber] = [];
-  }
-  return /** @type {Array<!jspb.Message>} */ (val);
 };
 
 
@@ -980,6 +1011,48 @@ jspb.Message.setRepeatedWrapperField = function(msg, fieldNumber, value) {
 };
 
 
+/**
+ * Add a message to a repeated proto field.
+ * @param {!jspb.Message} msg A jspb proto.
+ * @param {number} fieldNumber The field number.
+ * @param {T_CHILD|undefined} value Proto that will be added to the
+ *     repeated field.
+ * @param {function(new:T_CHILD, ?Array=)} ctor The constructor of the
+ *     message type.
+ * @param {number|undefined} index Index at which to insert the value.
+ * @return {T_CHILD_NOT_UNDEFINED} proto that was inserted to the repeated field
+ * @template MessageType
+ * Use go/closure-ttl to declare a non-undefined version of T_CHILD. Replace the
+ * undefined in blah|undefined with none. This is necessary because the compiler
+ * will infer T_CHILD to be |undefined.
+ * @template T_CHILD
+ * @template T_CHILD_NOT_UNDEFINED :=
+ *     cond(isUnknown(T_CHILD), unknown(),
+ *       mapunion(T_CHILD, (X) =>
+ *         cond(eq(X, 'undefined'), none(), X)))
+ * =:
+ * @protected
+ */
+jspb.Message.addToRepeatedWrapperField = function(
+    msg, fieldNumber, value, ctor, index) {
+  jspb.Message.wrapRepeatedField_(msg, ctor, fieldNumber);
+  var wrapperArray = msg.wrappers_[fieldNumber];
+  if (!wrapperArray) {
+    wrapperArray = msg.wrappers_[fieldNumber] = [];
+  }
+  var insertedValue = value ? value : new ctor();
+  var array = jspb.Message.getField(msg, fieldNumber);
+  if (index != undefined) {
+    wrapperArray.splice(index, 0, insertedValue);
+    array.splice(index, 0, insertedValue.toArray());
+  } else {
+    wrapperArray.push(insertedValue);
+    array.push(insertedValue.toArray());
+  }
+  return insertedValue;
+};
+
+
 /**
  * Converts a JsPb repeated message field into a map. The map will contain
  * protos unless an optional toObject function is given, in which case it will
@@ -1114,32 +1187,39 @@ jspb.Message.prototype.getExtension = function(fieldInfo) {
  * @param {jspb.ExtensionFieldInfo} fieldInfo Specifies the field to set.
  * @param {jspb.Message|string|Uint8Array|number|boolean|Array?} value The value
  *     to set.
+ * @return {THIS} For chaining
+ * @this {THIS}
+ * @template THIS
  */
 jspb.Message.prototype.setExtension = function(fieldInfo, value) {
-  if (!this.wrappers_) {
-    this.wrappers_ = {};
+  // Cast self, since the inferred THIS is unknown inside the function body.
+  // https://github.com/google/closure-compiler/issues/1411#issuecomment-232442220
+  var self = /** @type {!jspb.Message} */ (this);
+  if (!self.wrappers_) {
+    self.wrappers_ = {};
   }
-  jspb.Message.maybeInitEmptyExtensionObject_(this);
+  jspb.Message.maybeInitEmptyExtensionObject_(self);
   var fieldNumber = fieldInfo.fieldIndex;
   if (fieldInfo.isRepeated) {
     value = value || [];
     if (fieldInfo.isMessageType()) {
-      this.wrappers_[fieldNumber] = value;
-      this.extensionObject_[fieldNumber] = goog.array.map(
+      self.wrappers_[fieldNumber] = value;
+      self.extensionObject_[fieldNumber] = goog.array.map(
           /** @type {Array<jspb.Message>} */ (value), function(msg) {
         return msg.toArray();
       });
     } else {
-      this.extensionObject_[fieldNumber] = value;
+      self.extensionObject_[fieldNumber] = value;
     }
   } else {
     if (fieldInfo.isMessageType()) {
-      this.wrappers_[fieldNumber] = value;
-      this.extensionObject_[fieldNumber] = value ? value.toArray() : value;
+      self.wrappers_[fieldNumber] = value;
+      self.extensionObject_[fieldNumber] = value ? value.toArray() : value;
     } else {
-      this.extensionObject_[fieldNumber] = value;
+      self.extensionObject_[fieldNumber] = value;
     }
   }
+  return self;
 };
 
 
@@ -1308,7 +1388,30 @@ jspb.Message.compareFields = function(field1, field2) {
 
 
 /**
- * Static clone function. NOTE: A type-safe method called "cloneMessage" exists
+ * Templated, type-safe cloneMessage definition.
+ * @return {THIS}
+ * @this {THIS}
+ * @template THIS
+ */
+jspb.Message.prototype.cloneMessage = function() {
+  return jspb.Message.cloneMessage(/** @type {!jspb.Message} */ (this));
+};
+
+/**
+ * Alias clone to cloneMessage. goog.object.unsafeClone uses clone to
+ * efficiently copy objects. Without this alias, copying jspb messages comes
+ * with a large performance penalty.
+ * @return {THIS}
+ * @this {THIS}
+ * @template THIS
+ */
+jspb.Message.prototype.clone = function() {
+  return jspb.Message.cloneMessage(/** @type {!jspb.Message} */ (this));
+};
+
+/**
+ * Static clone function. NOTE: A type-safe method called "cloneMessage"
+ * exists
  * on each generated JsPb class. Do not call this function directly.
  * @param {!jspb.Message} msg A message to clone.
  * @return {!jspb.Message} A deep clone of the given message.
@@ -1429,3 +1532,4 @@ jspb.Message.registry_ = {};
  * @type {!Object.<number, jspb.ExtensionFieldInfo>}
  */
 jspb.Message.messageSetExtensions = {};
+jspb.Message.messageSetExtensionsBinary = {};

js/message_test.js

@@ -269,7 +269,7 @@ describe('Message test suite', function() {
     assertFalse(response.hasEnumField());
   });
 
-  it('testMessageRegistration', function() {
+  it('testMessageRegistration', /** @suppress {visibility} */ function() {
     // goog.require(SomeResponse) will include its library, which will in
     // turn add SomeResponse to the message registry.
     assertEquals(jspb.Message.registry_['res'], proto.jspb.test.SomeResponse);
@@ -297,47 +297,9 @@ describe('Message test suite', function() {
     var expected = [,,, [], []];
     expected[0] = expected[1] = expected[2] = undefined;
     assertObjectEquals(expected, foo.toArray());
-
-    // Test set(null). We could deprecated this in favor of clear(), but
-    // it's also convenient to have.
-    data = ['str', true, [11], [[22], [33]], ['s1', 's2']];
-    foo = new proto.jspb.test.OptionalFields(data);
-    foo.setAString(null);
-    foo.setABool(null);
-    foo.setANestedMessage(null);
-    foo.setARepeatedMessageList(null);
-    foo.setARepeatedStringList(null);
-    assertEquals('', foo.getAString());
-    assertEquals(false, foo.getABool());
-    assertNull(foo.getANestedMessage());
-    assertFalse(foo.hasAString());
-    assertFalse(foo.hasABool());
-    assertObjectEquals([], foo.getARepeatedMessageList());
-    assertObjectEquals([], foo.getARepeatedStringList());
-    assertObjectEquals([null, null, null, [], []], foo.toArray());
-
-    // Test set(undefined). Again, not something we really need, and not
-    // supported directly by our typing, but it should 'do the right thing'.
-    data = ['str', true, [11], [[22], [33]], ['s1', 's2']];
-    foo = new proto.jspb.test.OptionalFields(data);
-    foo.setAString(undefined);
-    foo.setABool(undefined);
-    foo.setANestedMessage(undefined);
-    foo.setARepeatedMessageList(undefined);
-    foo.setARepeatedStringList(undefined);
-    assertEquals('', foo.getAString());
-    assertEquals(false, foo.getABool());
-    assertUndefined(foo.getANestedMessage());
-    assertFalse(foo.hasAString());
-    assertFalse(foo.hasABool());
-    assertObjectEquals([], foo.getARepeatedMessageList());
-    assertObjectEquals([], foo.getARepeatedStringList());
-    expected = [,,, [], []];
-    expected[0] = expected[1] = expected[2] = undefined;
-    assertObjectEquals(expected, foo.toArray());
   });
 
-  it('testDifferenceRawObject', function() {
+  it('testDifferenceRawObject', /** @suppress {visibility} */ function() {
     var p1 = new proto.jspb.test.HasExtensions(['hi', 'diff', {}]);
     var p2 = new proto.jspb.test.HasExtensions(['hi', 'what',
                                                {1000: 'unique'}]);
@@ -477,7 +439,7 @@ describe('Message test suite', function() {
     var extension = new proto.jspb.test.CloneExtension();
     extension.setExt('e1');
     original.setExtension(proto.jspb.test.IsExtension.extField, extension);
-    var clone = original.cloneMessage();
+    var clone = original.clone();
     assertArrayEquals(['v1',, ['x1', ['y1', 'z1']],,
       [['x2', ['y2', 'z2']], ['x3', ['y3', 'z3']]], bytes1,, { 100: [, 'e1'] }],
         clone.toArray());
@@ -712,11 +674,12 @@ describe('Message test suite', function() {
     assertArrayEquals([1, 2, 3, {1: 'hi'}], msg.toArray());
   });
 
-  it('testExtendedMessageEnsureObject', function() {
-    var data = new proto.jspb.test.HasExtensions(['str1',
-        {'a_key': 'an_object'}]);
-    assertEquals('an_object', data.extensionObject_['a_key']);
-  });
+  it('testExtendedMessageEnsureObject',
+     /** @suppress {visibility} */ function() {
+       var data =
+           new proto.jspb.test.HasExtensions(['str1', {'a_key': 'an_object'}]);
+       assertEquals('an_object', data.extensionObject_['a_key']);
+     });
 
   it('testToObject_hasExtensionField', function() {
     var data = new proto.jspb.test.HasExtensions(['str1', {100: ['ext1']}]);

js/proto3_test.js

@@ -294,7 +294,7 @@ describe('proto3Test', function() {
     msg.setOptionalForeignEnum(proto.jspb.test.Proto3Enum.PROTO3_BAR);
     msg.setOptionalForeignEnum(proto.jspb.test.Proto3Enum.PROTO3_FOO);
     msg.setOneofUint32(32);
-    msg.setOneofUint32(null);
+    msg.clearOneofUint32();
 
 
     var serialized = msg.serializeBinary();

objectivec/google/protobuf/Duration.pbobjc.h

@@ -92,6 +92,12 @@ typedef GPB_ENUM(GPBDuration_FieldNumber) {
  *       end.seconds += 1;
  *       end.nanos -= 1000000000;
  *     }
+ *
+ * Example 3: Compute Duration from datetime.timedelta in Python.
+ *
+ *     td = datetime.timedelta(days=3, minutes=10)
+ *     duration = Duration()
+ *     duration.FromTimedelta(td)
  **/
 @interface GPBDuration : GPBMessage
 

objectivec/google/protobuf/FieldMask.pbobjc.h

@@ -94,7 +94,7 @@ typedef GPB_ENUM(GPBFieldMask_FieldNumber) {
  *     }
  *
  * A repeated field is not allowed except at the last position of a
- * field mask.
+ * paths string.
  *
  * If a FieldMask object is not present in a get operation, the
  * operation applies to all fields (as if a FieldMask of all fields
@@ -121,8 +121,8 @@ typedef GPB_ENUM(GPBFieldMask_FieldNumber) {
  *
  * If a repeated field is specified for an update operation, the existing
  * repeated values in the target resource will be overwritten by the new values.
- * Note that a repeated field is only allowed in the last position of a field
- * mask.
+ * Note that a repeated field is only allowed in the last position of a `paths`
+ * string.
  *
  * If a sub-message is specified in the last position of the field mask for an
  * update operation, then the existing sub-message in the target resource is

objectivec/google/protobuf/Timestamp.pbobjc.h

@@ -101,16 +101,14 @@ typedef GPB_ENUM(GPBTimestamp_FieldNumber) {
  *
  * Example 5: Compute Timestamp from current time in Python.
  *
- *     now = time.time()
- *     seconds = int(now)
- *     nanos = int((now - seconds) * 10**9)
- *     timestamp = Timestamp(seconds=seconds, nanos=nanos)
+ *     timestamp = Timestamp()
+ *     timestamp.GetCurrentTime()
  **/
 @interface GPBTimestamp : GPBMessage
 
 /**
  * Represents seconds of UTC time since Unix epoch
- * 1970-01-01T00:00:00Z. Must be from from 0001-01-01T00:00:00Z to
+ * 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to
  * 9999-12-31T23:59:59Z inclusive.
  **/
 @property(nonatomic, readwrite) int64_t seconds;

php/tests/test.pb.php

@@ -0,0 +1,1385 @@
+<?php
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: test.proto
+
+namespace Foo;
+
+require_once('test_include.pb.php');
+use Google\Protobuf\Internal\DescriptorPool;
+use Google\Protobuf\Internal\GPBType;
+use Google\Protobuf\Internal\RepeatedField;
+use Google\Protobuf\Internal\GPBUtil;
+
+class TestMessage extends \Google\Protobuf\Internal\Message
+{
+    private $optional_int32 = 0;
+    private $optional_int64 = 0;
+    private $optional_uint32 = 0;
+    private $optional_uint64 = 0;
+    private $optional_sint32 = 0;
+    private $optional_sint64 = 0;
+    private $optional_fixed32 = 0;
+    private $optional_fixed64 = 0;
+    private $optional_sfixed32 = 0;
+    private $optional_sfixed64 = 0;
+    private $optional_float = 0.0;
+    private $optional_double = 0.0;
+    private $optional_bool = false;
+    private $optional_string = '';
+    private $optional_bytes = '';
+    private $optional_enum = 0;
+    private $optional_message = null;
+    private $optional_included_message = null;
+    private $recursive = null;
+    private $repeated_int32;
+    private $repeated_int64;
+    private $repeated_uint32;
+    private $repeated_uint64;
+    private $repeated_sint32;
+    private $repeated_sint64;
+    private $repeated_fixed32;
+    private $repeated_fixed64;
+    private $repeated_sfixed32;
+    private $repeated_sfixed64;
+    private $repeated_float;
+    private $repeated_double;
+    private $repeated_bool;
+    private $repeated_string;
+    private $repeated_bytes;
+    private $repeated_enum;
+    private $repeated_message;
+    private $repeated_recursive;
+    private $map_int32_int32;
+    private $map_int64_int64;
+    private $map_uint32_uint32;
+    private $map_uint64_uint64;
+    private $map_sint32_sint32;
+    private $map_sint64_sint64;
+    private $map_fixed32_fixed32;
+    private $map_fixed64_fixed64;
+    private $map_sfixed32_sfixed32;
+    private $map_sfixed64_sfixed64;
+    private $map_int32_float;
+    private $map_int32_double;
+    private $map_bool_bool;
+    private $map_string_string;
+    private $map_int32_bytes;
+    private $map_int32_enum;
+    private $map_int32_message;
+    private $map_recursive;
+    protected $my_oneof;
+
+    public function getOptionalInt32()
+    {
+        return $this->optional_int32;
+    }
+
+    public function setOptionalInt32($var)
+    {
+        GPBUtil::checkInt32($var);
+        $this->optional_int32 = $var;
+    }
+
+    public function getOptionalInt64()
+    {
+        return $this->optional_int64;
+    }
+
+    public function setOptionalInt64($var)
+    {
+        GPBUtil::checkInt64($var);
+        $this->optional_int64 = $var;
+    }
+
+    public function getOptionalUint32()
+    {
+        return $this->optional_uint32;
+    }
+
+    public function setOptionalUint32($var)
+    {
+        GPBUtil::checkUint32($var);
+        $this->optional_uint32 = $var;
+    }
+
+    public function getOptionalUint64()
+    {
+        return $this->optional_uint64;
+    }
+
+    public function setOptionalUint64($var)
+    {
+        GPBUtil::checkUint64($var);
+        $this->optional_uint64 = $var;
+    }
+
+    public function getOptionalSint32()
+    {
+        return $this->optional_sint32;
+    }
+
+    public function setOptionalSint32($var)
+    {
+        GPBUtil::checkInt32($var);
+        $this->optional_sint32 = $var;
+    }
+
+    public function getOptionalSint64()
+    {
+        return $this->optional_sint64;
+    }
+
+    public function setOptionalSint64($var)
+    {
+        GPBUtil::checkInt64($var);
+        $this->optional_sint64 = $var;
+    }
+
+    public function getOptionalFixed32()
+    {
+        return $this->optional_fixed32;
+    }
+
+    public function setOptionalFixed32($var)
+    {
+        GPBUtil::checkUint32($var);
+        $this->optional_fixed32 = $var;
+    }
+
+    public function getOptionalFixed64()
+    {
+        return $this->optional_fixed64;
+    }
+
+    public function setOptionalFixed64($var)
+    {
+        GPBUtil::checkUint64($var);
+        $this->optional_fixed64 = $var;
+    }
+
+    public function getOptionalSfixed32()
+    {
+        return $this->optional_sfixed32;
+    }
+
+    public function setOptionalSfixed32($var)
+    {
+        GPBUtil::checkInt32($var);
+        $this->optional_sfixed32 = $var;
+    }
+
+    public function getOptionalSfixed64()
+    {
+        return $this->optional_sfixed64;
+    }
+
+    public function setOptionalSfixed64($var)
+    {
+        GPBUtil::checkInt64($var);
+        $this->optional_sfixed64 = $var;
+    }
+
+    public function getOptionalFloat()
+    {
+        return $this->optional_float;
+    }
+
+    public function setOptionalFloat($var)
+    {
+        GPBUtil::checkFloat($var);
+        $this->optional_float = $var;
+    }
+
+    public function getOptionalDouble()
+    {
+        return $this->optional_double;
+    }
+
+    public function setOptionalDouble($var)
+    {
+        GPBUtil::checkDouble($var);
+        $this->optional_double = $var;
+    }
+
+    public function getOptionalBool()
+    {
+        return $this->optional_bool;
+    }
+
+    public function setOptionalBool($var)
+    {
+        GPBUtil::checkBool($var);
+        $this->optional_bool = $var;
+    }
+
+    public function getOptionalString()
+    {
+        return $this->optional_string;
+    }
+
+    public function setOptionalString($var)
+    {
+        GPBUtil::checkString($var, True);
+        $this->optional_string = $var;
+    }
+
+    public function getOptionalBytes()
+    {
+        return $this->optional_bytes;
+    }
+
+    public function setOptionalBytes($var)
+    {
+        GPBUtil::checkString($var, False);
+        $this->optional_bytes = $var;
+    }
+
+    public function getOptionalEnum()
+    {
+        return $this->optional_enum;
+    }
+
+    public function setOptionalEnum($var)
+    {
+        GPBUtil::checkEnum($var, \Foo\TestEnum::class);
+        $this->optional_enum = $var;
+    }
+
+    public function getOptionalMessage()
+    {
+        return $this->optional_message;
+    }
+
+    public function setOptionalMessage(&$var)
+    {
+        GPBUtil::checkMessage($var, \Foo\TestMessage_Sub::class);
+        $this->optional_message = $var;
+    }
+
+    public function getOptionalIncludedMessage()
+    {
+        return $this->optional_included_message;
+    }
+
+    public function setOptionalIncludedMessage(&$var)
+    {
+        GPBUtil::checkMessage($var, \Bar\TestInclude::class);
+        $this->optional_included_message = $var;
+    }
+
+    public function getRecursive()
+    {
+        return $this->recursive;
+    }
+
+    public function setRecursive(&$var)
+    {
+        GPBUtil::checkMessage($var, \Foo\TestMessage::class);
+        $this->recursive = $var;
+    }
+
+    public function getRepeatedInt32()
+    {
+        return $this->repeated_int32;
+    }
+
+    public function setRepeatedInt32(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::INT32);
+        $this->repeated_int32 = $var;
+    }
+
+    public function getRepeatedInt64()
+    {
+        return $this->repeated_int64;
+    }
+
+    public function setRepeatedInt64(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::INT64);
+        $this->repeated_int64 = $var;
+    }
+
+    public function getRepeatedUint32()
+    {
+        return $this->repeated_uint32;
+    }
+
+    public function setRepeatedUint32(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::UINT32);
+        $this->repeated_uint32 = $var;
+    }
+
+    public function getRepeatedUint64()
+    {
+        return $this->repeated_uint64;
+    }
+
+    public function setRepeatedUint64(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::UINT64);
+        $this->repeated_uint64 = $var;
+    }
+
+    public function getRepeatedSint32()
+    {
+        return $this->repeated_sint32;
+    }
+
+    public function setRepeatedSint32(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::SINT32);
+        $this->repeated_sint32 = $var;
+    }
+
+    public function getRepeatedSint64()
+    {
+        return $this->repeated_sint64;
+    }
+
+    public function setRepeatedSint64(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::SINT64);
+        $this->repeated_sint64 = $var;
+    }
+
+    public function getRepeatedFixed32()
+    {
+        return $this->repeated_fixed32;
+    }
+
+    public function setRepeatedFixed32(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::FIXED32);
+        $this->repeated_fixed32 = $var;
+    }
+
+    public function getRepeatedFixed64()
+    {
+        return $this->repeated_fixed64;
+    }
+
+    public function setRepeatedFixed64(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::FIXED64);
+        $this->repeated_fixed64 = $var;
+    }
+
+    public function getRepeatedSfixed32()
+    {
+        return $this->repeated_sfixed32;
+    }
+
+    public function setRepeatedSfixed32(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::SFIXED32);
+        $this->repeated_sfixed32 = $var;
+    }
+
+    public function getRepeatedSfixed64()
+    {
+        return $this->repeated_sfixed64;
+    }
+
+    public function setRepeatedSfixed64(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::SFIXED64);
+        $this->repeated_sfixed64 = $var;
+    }
+
+    public function getRepeatedFloat()
+    {
+        return $this->repeated_float;
+    }
+
+    public function setRepeatedFloat(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::FLOAT);
+        $this->repeated_float = $var;
+    }
+
+    public function getRepeatedDouble()
+    {
+        return $this->repeated_double;
+    }
+
+    public function setRepeatedDouble(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::DOUBLE);
+        $this->repeated_double = $var;
+    }
+
+    public function getRepeatedBool()
+    {
+        return $this->repeated_bool;
+    }
+
+    public function setRepeatedBool(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::BOOL);
+        $this->repeated_bool = $var;
+    }
+
+    public function getRepeatedString()
+    {
+        return $this->repeated_string;
+    }
+
+    public function setRepeatedString(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::STRING);
+        $this->repeated_string = $var;
+    }
+
+    public function getRepeatedBytes()
+    {
+        return $this->repeated_bytes;
+    }
+
+    public function setRepeatedBytes(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::BYTES);
+        $this->repeated_bytes = $var;
+    }
+
+    public function getRepeatedEnum()
+    {
+        return $this->repeated_enum;
+    }
+
+    public function setRepeatedEnum(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::ENUM, Foo\TestEnum::class);
+        $this->repeated_enum = $var;
+    }
+
+    public function getRepeatedMessage()
+    {
+        return $this->repeated_message;
+    }
+
+    public function setRepeatedMessage(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::MESSAGE, \Foo\TestMessage_Sub::class);
+        $this->repeated_message = $var;
+    }
+
+    public function getRepeatedRecursive()
+    {
+        return $this->repeated_recursive;
+    }
+
+    public function setRepeatedRecursive(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::MESSAGE, \Foo\TestMessage::class);
+        $this->repeated_recursive = $var;
+    }
+
+    public function getOneofInt32()
+    {
+        return $this->readOneof(51);
+    }
+
+    public function setOneofInt32($var)
+    {
+        GPBUtil::checkInt32($var);
+        $this->writeOneof(51, $var);
+    }
+
+    public function getOneofInt64()
+    {
+        return $this->readOneof(52);
+    }
+
+    public function setOneofInt64($var)
+    {
+        GPBUtil::checkInt64($var);
+        $this->writeOneof(52, $var);
+    }
+
+    public function getOneofUint32()
+    {
+        return $this->readOneof(53);
+    }
+
+    public function setOneofUint32($var)
+    {
+        GPBUtil::checkUint32($var);
+        $this->writeOneof(53, $var);
+    }
+
+    public function getOneofUint64()
+    {
+        return $this->readOneof(54);
+    }
+
+    public function setOneofUint64($var)
+    {
+        GPBUtil::checkUint64($var);
+        $this->writeOneof(54, $var);
+    }
+
+    public function getOneofSint32()
+    {
+        return $this->readOneof(55);
+    }
+
+    public function setOneofSint32($var)
+    {
+        GPBUtil::checkUint32($var);
+        $this->writeOneof(55, $var);
+    }
+
+    public function getOneofSint64()
+    {
+        return $this->readOneof(56);
+    }
+
+    public function setOneofSint64($var)
+    {
+        GPBUtil::checkUint64($var);
+        $this->writeOneof(56, $var);
+    }
+
+    public function getOneofFixed32()
+    {
+        return $this->readOneof(57);
+    }
+
+    public function setOneofFixed32($var)
+    {
+        GPBUtil::checkUint32($var);
+        $this->writeOneof(57, $var);
+    }
+
+    public function getOneofFixed64()
+    {
+        return $this->readOneof(58);
+    }
+
+    public function setOneofFixed64($var)
+    {
+        GPBUtil::checkUint64($var);
+        $this->writeOneof(58, $var);
+    }
+
+    public function getOneofSfixed32()
+    {
+        return $this->readOneof(59);
+    }
+
+    public function setOneofSfixed32($var)
+    {
+        GPBUtil::checkUint32($var);
+        $this->writeOneof(59, $var);
+    }
+
+    public function getOneofSfixed64()
+    {
+        return $this->readOneof(60);
+    }
+
+    public function setOneofSfixed64($var)
+    {
+        GPBUtil::checkUint64($var);
+        $this->writeOneof(60, $var);
+    }
+
+    public function getOneofDouble()
+    {
+        return $this->readOneof(61);
+    }
+
+    public function setOneofDouble($var)
+    {
+        GPBUtil::checkDouble($var);
+        $this->writeOneof(61, $var);
+    }
+
+    public function getOneofFloat()
+    {
+        return $this->readOneof(62);
+    }
+
+    public function setOneofFloat($var)
+    {
+        GPBUtil::checkFloat($var);
+        $this->writeOneof(62, $var);
+    }
+
+    public function getOneofBool()
+    {
+        return $this->readOneof(63);
+    }
+
+    public function setOneofBool($var)
+    {
+        GPBUtil::checkBool($var);
+        $this->writeOneof(63, $var);
+    }
+
+    public function getOneofString()
+    {
+        return $this->readOneof(64);
+    }
+
+    public function setOneofString($var)
+    {
+        GPBUtil::checkString($var, True);
+        $this->writeOneof(64, $var);
+    }
+
+    public function getOneofBytes()
+    {
+        return $this->readOneof(65);
+    }
+
+    public function setOneofBytes($var)
+    {
+        GPBUtil::checkString($var, False);
+        $this->writeOneof(65, $var);
+    }
+
+    public function getOneofEnum()
+    {
+        return $this->readOneof(66);
+    }
+
+    public function setOneofEnum($var)
+    {
+        GPBUtil::checkEnum($var, \Foo\TestEnum::class);
+        $this->writeOneof(66, $var);
+    }
+
+    public function getOneofMessage()
+    {
+        return $this->readOneof(67);
+    }
+
+    public function setOneofMessage(&$var)
+    {
+        GPBUtil::checkMessage($var, \Foo\TestMessage_Sub::class);
+        $this->writeOneof(67, $var);
+    }
+
+    public function getMapInt32Int32()
+    {
+        return $this->map_int32_int32;
+    }
+
+    public function setMapInt32Int32(&$var)
+    {
+        $this->map_int32_int32 = $var;
+    }
+
+    public function getMapInt64Int64()
+    {
+        return $this->map_int64_int64;
+    }
+
+    public function setMapInt64Int64(&$var)
+    {
+        $this->map_int64_int64 = $var;
+    }
+
+    public function getMapUint32Uint32()
+    {
+        return $this->map_uint32_uint32;
+    }
+
+    public function setMapUint32Uint32(&$var)
+    {
+        $this->map_uint32_uint32 = $var;
+    }
+
+    public function getMapUint64Uint64()
+    {
+        return $this->map_uint64_uint64;
+    }
+
+    public function setMapUint64Uint64(&$var)
+    {
+        $this->map_uint64_uint64 = $var;
+    }
+
+    public function getMapSint32Sint32()
+    {
+        return $this->map_sint32_sint32;
+    }
+
+    public function setMapSint32Sint32(&$var)
+    {
+        $this->map_sint32_sint32 = $var;
+    }
+
+    public function getMapSint64Sint64()
+    {
+        return $this->map_sint64_sint64;
+    }
+
+    public function setMapSint64Sint64(&$var)
+    {
+        $this->map_sint64_sint64 = $var;
+    }
+
+    public function getMapFixed32Fixed32()
+    {
+        return $this->map_fixed32_fixed32;
+    }
+
+    public function setMapFixed32Fixed32(&$var)
+    {
+        $this->map_fixed32_fixed32 = $var;
+    }
+
+    public function getMapFixed64Fixed64()
+    {
+        return $this->map_fixed64_fixed64;
+    }
+
+    public function setMapFixed64Fixed64(&$var)
+    {
+        $this->map_fixed64_fixed64 = $var;
+    }
+
+    public function getMapSfixed32Sfixed32()
+    {
+        return $this->map_sfixed32_sfixed32;
+    }
+
+    public function setMapSfixed32Sfixed32(&$var)
+    {
+        $this->map_sfixed32_sfixed32 = $var;
+    }
+
+    public function getMapSfixed64Sfixed64()
+    {
+        return $this->map_sfixed64_sfixed64;
+    }
+
+    public function setMapSfixed64Sfixed64(&$var)
+    {
+        $this->map_sfixed64_sfixed64 = $var;
+    }
+
+    public function getMapInt32Float()
+    {
+        return $this->map_int32_float;
+    }
+
+    public function setMapInt32Float(&$var)
+    {
+        $this->map_int32_float = $var;
+    }
+
+    public function getMapInt32Double()
+    {
+        return $this->map_int32_double;
+    }
+
+    public function setMapInt32Double(&$var)
+    {
+        $this->map_int32_double = $var;
+    }
+
+    public function getMapBoolBool()
+    {
+        return $this->map_bool_bool;
+    }
+
+    public function setMapBoolBool(&$var)
+    {
+        $this->map_bool_bool = $var;
+    }
+
+    public function getMapStringString()
+    {
+        return $this->map_string_string;
+    }
+
+    public function setMapStringString(&$var)
+    {
+        $this->map_string_string = $var;
+    }
+
+    public function getMapInt32Bytes()
+    {
+        return $this->map_int32_bytes;
+    }
+
+    public function setMapInt32Bytes(&$var)
+    {
+        $this->map_int32_bytes = $var;
+    }
+
+    public function getMapInt32Enum()
+    {
+        return $this->map_int32_enum;
+    }
+
+    public function setMapInt32Enum(&$var)
+    {
+        $this->map_int32_enum = $var;
+    }
+
+    public function getMapInt32Message()
+    {
+        return $this->map_int32_message;
+    }
+
+    public function setMapInt32Message(&$var)
+    {
+        $this->map_int32_message = $var;
+    }
+
+    public function getMapRecursive()
+    {
+        return $this->map_recursive;
+    }
+
+    public function setMapRecursive(&$var)
+    {
+        $this->map_recursive = $var;
+    }
+
+    public function getMyOneof()
+    {
+        return $this->my_oneof;
+    }
+
+}
+
+class TestMessage_Sub extends \Google\Protobuf\Internal\Message
+{
+    private $a = 0;
+
+    public function getA()
+    {
+        return $this->a;
+    }
+
+    public function setA($var)
+    {
+        GPBUtil::checkInt32($var);
+        $this->a = $var;
+    }
+
+}
+
+class TestPackedMessage extends \Google\Protobuf\Internal\Message
+{
+    private $repeated_int32;
+    private $repeated_int64;
+    private $repeated_uint32;
+    private $repeated_uint64;
+    private $repeated_sint32;
+    private $repeated_sint64;
+    private $repeated_fixed32;
+    private $repeated_fixed64;
+    private $repeated_sfixed32;
+    private $repeated_sfixed64;
+    private $repeated_float;
+    private $repeated_double;
+    private $repeated_bool;
+    private $repeated_enum;
+
+    public function getRepeatedInt32()
+    {
+        return $this->repeated_int32;
+    }
+
+    public function setRepeatedInt32(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::INT32);
+        $this->repeated_int32 = $var;
+    }
+
+    public function getRepeatedInt64()
+    {
+        return $this->repeated_int64;
+    }
+
+    public function setRepeatedInt64(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::INT64);
+        $this->repeated_int64 = $var;
+    }
+
+    public function getRepeatedUint32()
+    {
+        return $this->repeated_uint32;
+    }
+
+    public function setRepeatedUint32(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::UINT32);
+        $this->repeated_uint32 = $var;
+    }
+
+    public function getRepeatedUint64()
+    {
+        return $this->repeated_uint64;
+    }
+
+    public function setRepeatedUint64(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::UINT64);
+        $this->repeated_uint64 = $var;
+    }
+
+    public function getRepeatedSint32()
+    {
+        return $this->repeated_sint32;
+    }
+
+    public function setRepeatedSint32(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::SINT32);
+        $this->repeated_sint32 = $var;
+    }
+
+    public function getRepeatedSint64()
+    {
+        return $this->repeated_sint64;
+    }
+
+    public function setRepeatedSint64(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::SINT64);
+        $this->repeated_sint64 = $var;
+    }
+
+    public function getRepeatedFixed32()
+    {
+        return $this->repeated_fixed32;
+    }
+
+    public function setRepeatedFixed32(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::FIXED32);
+        $this->repeated_fixed32 = $var;
+    }
+
+    public function getRepeatedFixed64()
+    {
+        return $this->repeated_fixed64;
+    }
+
+    public function setRepeatedFixed64(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::FIXED64);
+        $this->repeated_fixed64 = $var;
+    }
+
+    public function getRepeatedSfixed32()
+    {
+        return $this->repeated_sfixed32;
+    }
+
+    public function setRepeatedSfixed32(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::SFIXED32);
+        $this->repeated_sfixed32 = $var;
+    }
+
+    public function getRepeatedSfixed64()
+    {
+        return $this->repeated_sfixed64;
+    }
+
+    public function setRepeatedSfixed64(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::SFIXED64);
+        $this->repeated_sfixed64 = $var;
+    }
+
+    public function getRepeatedFloat()
+    {
+        return $this->repeated_float;
+    }
+
+    public function setRepeatedFloat(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::FLOAT);
+        $this->repeated_float = $var;
+    }
+
+    public function getRepeatedDouble()
+    {
+        return $this->repeated_double;
+    }
+
+    public function setRepeatedDouble(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::DOUBLE);
+        $this->repeated_double = $var;
+    }
+
+    public function getRepeatedBool()
+    {
+        return $this->repeated_bool;
+    }
+
+    public function setRepeatedBool(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::BOOL);
+        $this->repeated_bool = $var;
+    }
+
+    public function getRepeatedEnum()
+    {
+        return $this->repeated_enum;
+    }
+
+    public function setRepeatedEnum(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::ENUM, Foo\TestEnum::class);
+        $this->repeated_enum = $var;
+    }
+
+}
+
+class TestUnpackedMessage extends \Google\Protobuf\Internal\Message
+{
+    private $repeated_int32;
+    private $repeated_int64;
+    private $repeated_uint32;
+    private $repeated_uint64;
+    private $repeated_sint32;
+    private $repeated_sint64;
+    private $repeated_fixed32;
+    private $repeated_fixed64;
+    private $repeated_sfixed32;
+    private $repeated_sfixed64;
+    private $repeated_float;
+    private $repeated_double;
+    private $repeated_bool;
+    private $repeated_enum;
+
+    public function getRepeatedInt32()
+    {
+        return $this->repeated_int32;
+    }
+
+    public function setRepeatedInt32(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::INT32);
+        $this->repeated_int32 = $var;
+    }
+
+    public function getRepeatedInt64()
+    {
+        return $this->repeated_int64;
+    }
+
+    public function setRepeatedInt64(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::INT64);
+        $this->repeated_int64 = $var;
+    }
+
+    public function getRepeatedUint32()
+    {
+        return $this->repeated_uint32;
+    }
+
+    public function setRepeatedUint32(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::UINT32);
+        $this->repeated_uint32 = $var;
+    }
+
+    public function getRepeatedUint64()
+    {
+        return $this->repeated_uint64;
+    }
+
+    public function setRepeatedUint64(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::UINT64);
+        $this->repeated_uint64 = $var;
+    }
+
+    public function getRepeatedSint32()
+    {
+        return $this->repeated_sint32;
+    }
+
+    public function setRepeatedSint32(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::SINT32);
+        $this->repeated_sint32 = $var;
+    }
+
+    public function getRepeatedSint64()
+    {
+        return $this->repeated_sint64;
+    }
+
+    public function setRepeatedSint64(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::SINT64);
+        $this->repeated_sint64 = $var;
+    }
+
+    public function getRepeatedFixed32()
+    {
+        return $this->repeated_fixed32;
+    }
+
+    public function setRepeatedFixed32(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::FIXED32);
+        $this->repeated_fixed32 = $var;
+    }
+
+    public function getRepeatedFixed64()
+    {
+        return $this->repeated_fixed64;
+    }
+
+    public function setRepeatedFixed64(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::FIXED64);
+        $this->repeated_fixed64 = $var;
+    }
+
+    public function getRepeatedSfixed32()
+    {
+        return $this->repeated_sfixed32;
+    }
+
+    public function setRepeatedSfixed32(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::SFIXED32);
+        $this->repeated_sfixed32 = $var;
+    }
+
+    public function getRepeatedSfixed64()
+    {
+        return $this->repeated_sfixed64;
+    }
+
+    public function setRepeatedSfixed64(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::SFIXED64);
+        $this->repeated_sfixed64 = $var;
+    }
+
+    public function getRepeatedFloat()
+    {
+        return $this->repeated_float;
+    }
+
+    public function setRepeatedFloat(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::FLOAT);
+        $this->repeated_float = $var;
+    }
+
+    public function getRepeatedDouble()
+    {
+        return $this->repeated_double;
+    }
+
+    public function setRepeatedDouble(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::DOUBLE);
+        $this->repeated_double = $var;
+    }
+
+    public function getRepeatedBool()
+    {
+        return $this->repeated_bool;
+    }
+
+    public function setRepeatedBool(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::BOOL);
+        $this->repeated_bool = $var;
+    }
+
+    public function getRepeatedEnum()
+    {
+        return $this->repeated_enum;
+    }
+
+    public function setRepeatedEnum(&$var)
+    {
+        GPBUtil::checkRepeatedField($var, GPBType::ENUM, Foo\TestEnum::class);
+        $this->repeated_enum = $var;
+    }
+
+}
+
+class TestEnum
+{
+    const ZERO = 0;
+    const ONE = 1;
+}
+
+$pool = DescriptorPool::getGeneratedPool();
+
+$pool->internalAddGeneratedFile(hex2bin(
+    "0a83250a0a746573742e70726f746f1203666f6f1a12746573745f696e63" .
+    "6c7564652e70726f746f22be1d0a0b546573744d65737361676512160a0e" .
+    "6f7074696f6e616c5f696e74333218012001280512160a0e6f7074696f6e" .
+    "616c5f696e74363418022001280312170a0f6f7074696f6e616c5f75696e" .
+    "74333218032001280d12170a0f6f7074696f6e616c5f75696e7436341804" .
+    "2001280412170a0f6f7074696f6e616c5f73696e74333218052001281112" .
+    "170a0f6f7074696f6e616c5f73696e74363418062001281212180a106f70" .
+    "74696f6e616c5f6669786564333218072001280712180a106f7074696f6e" .
+    "616c5f6669786564363418082001280612190a116f7074696f6e616c5f73" .
+    "6669786564333218092001280f12190a116f7074696f6e616c5f73666978" .
+    "65643634180a2001281012160a0e6f7074696f6e616c5f666c6f6174180b" .
+    "2001280212170a0f6f7074696f6e616c5f646f75626c65180c2001280112" .
+    "150a0d6f7074696f6e616c5f626f6f6c180d2001280812170a0f6f707469" .
+    "6f6e616c5f737472696e67180e2001280912160a0e6f7074696f6e616c5f" .
+    "6279746573180f2001280c12240a0d6f7074696f6e616c5f656e756d1810" .
+    "2001280e320d2e666f6f2e54657374456e756d122e0a106f7074696f6e61" .
+    "6c5f6d65737361676518112001280b32142e666f6f2e546573744d657373" .
+    "6167652e53756212330a196f7074696f6e616c5f696e636c756465645f6d" .
+    "65737361676518122001280b32102e6261722e54657374496e636c756465" .
+    "12230a0972656375727369766518132001280b32102e666f6f2e54657374" .
+    "4d65737361676512160a0e72657065617465645f696e743332181f200328" .
+    "0512160a0e72657065617465645f696e74363418202003280312170a0f72" .
+    "657065617465645f75696e74333218212003280d12170a0f726570656174" .
+    "65645f75696e74363418222003280412170a0f72657065617465645f7369" .
+    "6e74333218232003281112170a0f72657065617465645f73696e74363418" .
+    "242003281212180a1072657065617465645f666978656433321825200328" .
+    "0712180a1072657065617465645f6669786564363418262003280612190a" .
+    "1172657065617465645f736669786564333218272003280f12190a117265" .
+    "7065617465645f736669786564363418282003281012160a0e7265706561" .
+    "7465645f666c6f617418292003280212170a0f72657065617465645f646f" .
+    "75626c65182a2003280112150a0d72657065617465645f626f6f6c182b20" .
+    "03280812170a0f72657065617465645f737472696e67182c200328091216" .
+    "0a0e72657065617465645f6279746573182d2003280c12240a0d72657065" .
+    "617465645f656e756d182e2003280e320d2e666f6f2e54657374456e756d" .
+    "122e0a1072657065617465645f6d657373616765182f2003280b32142e66" .
+    "6f6f2e546573744d6573736167652e537562122c0a127265706561746564" .
+    "5f72656375727369766518302003280b32102e666f6f2e546573744d6573" .
+    "7361676512150a0b6f6e656f665f696e743332183320012805480012150a" .
+    "0b6f6e656f665f696e743634183420012803480012160a0c6f6e656f665f" .
+    "75696e74333218352001280d480012160a0c6f6e656f665f75696e743634" .
+    "183620012804480012160a0c6f6e656f665f73696e74333218372001280d" .
+    "480012160a0c6f6e656f665f73696e743634183820012804480012170a0d" .
+    "6f6e656f665f6669786564333218392001280d480012170a0d6f6e656f66" .
+    "5f66697865643634183a20012804480012180a0e6f6e656f665f73666978" .
+    "65643332183b2001280d480012180a0e6f6e656f665f7366697865643634" .
+    "183c20012804480012160a0c6f6e656f665f646f75626c65183d20012801" .
+    "480012150a0b6f6e656f665f666c6f6174183e20012802480012140a0a6f" .
+    "6e656f665f626f6f6c183f20012808480012160a0c6f6e656f665f737472" .
+    "696e67184020012809480012150a0b6f6e656f665f627974657318412001" .
+    "280c480012230a0a6f6e656f665f656e756d18422001280e320d2e666f6f" .
+    "2e54657374456e756d4800122d0a0d6f6e656f665f6d6573736167651843" .
+    "2001280b32142e666f6f2e546573744d6573736167652e5375624800123c" .
+    "0a0f6d61705f696e7433325f696e74333218472003280b32232e666f6f2e" .
+    "546573744d6573736167652e4d6170496e743332496e743332456e747279" .
+    "123c0a0f6d61705f696e7436345f696e74363418482003280b32232e666f" .
+    "6f2e546573744d6573736167652e4d6170496e743634496e743634456e74" .
+    "727912400a116d61705f75696e7433325f75696e74333218492003280b32" .
+    "252e666f6f2e546573744d6573736167652e4d617055696e74333255696e" .
+    "743332456e74727912400a116d61705f75696e7436345f75696e74363418" .
+    "4a2003280b32252e666f6f2e546573744d6573736167652e4d617055696e" .
+    "74363455696e743634456e74727912400a116d61705f73696e7433325f73" .
+    "696e743332184b2003280b32252e666f6f2e546573744d6573736167652e" .
+    "4d617053696e74333253696e743332456e74727912400a116d61705f7369" .
+    "6e7436345f73696e743634184c2003280b32252e666f6f2e546573744d65" .
+    "73736167652e4d617053696e74363453696e743634456e74727912440a13" .
+    "6d61705f666978656433325f66697865643332184d2003280b32272e666f" .
+    "6f2e546573744d6573736167652e4d617046697865643332466978656433" .
+    "32456e74727912440a136d61705f666978656436345f6669786564363418" .
+    "4e2003280b32272e666f6f2e546573744d6573736167652e4d6170466978" .
+    "6564363446697865643634456e74727912480a156d61705f736669786564" .
+    "33325f7366697865643332184f2003280b32292e666f6f2e546573744d65" .
+    "73736167652e4d617053666978656433325366697865643332456e747279" .
+    "12480a156d61705f73666978656436345f73666978656436341850200328" .
+    "0b32292e666f6f2e546573744d6573736167652e4d617053666978656436" .
+    "345366697865643634456e747279123c0a0f6d61705f696e7433325f666c" .
+    "6f617418512003280b32232e666f6f2e546573744d6573736167652e4d61" .
+    "70496e743332466c6f6174456e747279123e0a106d61705f696e7433325f" .
+    "646f75626c6518522003280b32242e666f6f2e546573744d657373616765" .
+    "2e4d6170496e743332446f75626c65456e74727912380a0d6d61705f626f" .
+    "6f6c5f626f6f6c18532003280b32212e666f6f2e546573744d6573736167" .
+    "652e4d6170426f6f6c426f6f6c456e74727912400a116d61705f73747269" .
+    "6e675f737472696e6718542003280b32252e666f6f2e546573744d657373" .
+    "6167652e4d6170537472696e67537472696e67456e747279123c0a0f6d61" .
+    "705f696e7433325f627974657318552003280b32232e666f6f2e54657374" .
+    "4d6573736167652e4d6170496e7433324279746573456e747279123a0a0e" .
+    "6d61705f696e7433325f656e756d18562003280b32222e666f6f2e546573" .
+    "744d6573736167652e4d6170496e743332456e756d456e74727912400a11" .
+    "6d61705f696e7433325f6d65737361676518572003280b32252e666f6f2e" .
+    "546573744d6573736167652e4d6170496e7433324d657373616765456e74" .
+    "727912390a0d6d61705f72656375727369766518582003280b32222e666f" .
+    "6f2e546573744d6573736167652e4d6170526563757273697665456e7472" .
+    "791a340a124d6170496e743332496e743332456e747279120b0a036b6579" .
+    "180120012805120d0a0576616c75651802200128053a0238011a340a124d" .
+    "6170496e743634496e743634456e747279120b0a036b6579180120012803" .
+    "120d0a0576616c75651802200128033a0238011a360a144d617055696e74" .
+    "333255696e743332456e747279120b0a036b657918012001280d120d0a05" .
+    "76616c756518022001280d3a0238011a360a144d617055696e7436345569" .
+    "6e743634456e747279120b0a036b6579180120012804120d0a0576616c75" .
+    "651802200128043a0238011a360a144d617053696e74333253696e743332" .
+    "456e747279120b0a036b6579180120012811120d0a0576616c7565180220" .
+    "0128113a0238011a360a144d617053696e74363453696e743634456e7472" .
+    "79120b0a036b6579180120012812120d0a0576616c75651802200128123a" .
+    "0238011a380a164d61704669786564333246697865643332456e74727912" .
+    "0b0a036b6579180120012807120d0a0576616c75651802200128073a0238" .
+    "011a380a164d61704669786564363446697865643634456e747279120b0a" .
+    "036b6579180120012806120d0a0576616c75651802200128063a0238011a" .
+    "3a0a184d617053666978656433325366697865643332456e747279120b0a" .
+    "036b657918012001280f120d0a0576616c756518022001280f3a0238011a" .
+    "3a0a184d617053666978656436345366697865643634456e747279120b0a" .
+    "036b6579180120012810120d0a0576616c75651802200128103a0238011a" .
+    "340a124d6170496e743332466c6f6174456e747279120b0a036b65791801" .
+    "20012805120d0a0576616c75651802200128023a0238011a350a134d6170" .
+    "496e743332446f75626c65456e747279120b0a036b657918012001280512" .
+    "0d0a0576616c75651802200128013a0238011a320a104d6170426f6f6c42" .
+    "6f6f6c456e747279120b0a036b6579180120012808120d0a0576616c7565" .
+    "1802200128083a0238011a360a144d6170537472696e67537472696e6745" .
+    "6e747279120b0a036b6579180120012809120d0a0576616c756518022001" .
+    "28093a0238011a340a124d6170496e7433324279746573456e747279120b" .
+    "0a036b6579180120012805120d0a0576616c756518022001280c3a023801" .
+    "1a420a114d6170496e743332456e756d456e747279120b0a036b65791801" .
+    "20012805121c0a0576616c756518022001280e320d2e666f6f2e54657374" .
+    "456e756d3a0238011a4c0a144d6170496e7433324d657373616765456e74" .
+    "7279120b0a036b657918012001280512230a0576616c756518022001280b" .
+    "32142e666f6f2e546573744d6573736167652e5375623a0238011a450a11" .
+    "4d6170526563757273697665456e747279120b0a036b6579180120012805" .
+    "121f0a0576616c756518022001280b32102e666f6f2e546573744d657373" .
+    "6167653a0238011a100a0353756212090a0161180120012805420a0a086d" .
+    "795f6f6e656f6622b7030a11546573745061636b65644d65737361676512" .
+    "1a0a0e72657065617465645f696e743332185a2003280542021001121a0a" .
+    "0e72657065617465645f696e743634185b2003280342021001121b0a0f72" .
+    "657065617465645f75696e743332185c2003280d42021001121b0a0f7265" .
+    "7065617465645f75696e743634185d2003280442021001121b0a0f726570" .
+    "65617465645f73696e743332185e2003281142021001121b0a0f72657065" .
+    "617465645f73696e743634185f2003281242021001121c0a107265706561" .
+    "7465645f6669786564333218602003280742021001121c0a107265706561" .
+    "7465645f6669786564363418612003280642021001121d0a117265706561" .
+    "7465645f736669786564333218622003280f42021001121d0a1172657065" .
+    "617465645f736669786564363418632003281042021001121a0a0e726570" .
+    "65617465645f666c6f617418642003280242021001121b0a0f7265706561" .
+    "7465645f646f75626c651865200328014202100112190a0d726570656174" .
+    "65645f626f6f6c1866200328084202100112280a0d72657065617465645f" .
+    "656e756d18672003280e320d2e666f6f2e54657374456e756d4202100122" .
+    "b9030a1354657374556e7061636b65644d657373616765121a0a0e726570" .
+    "65617465645f696e743332185a2003280542021000121a0a0e7265706561" .
+    "7465645f696e743634185b2003280342021000121b0a0f72657065617465" .
+    "645f75696e743332185c2003280d42021000121b0a0f7265706561746564" .
+    "5f75696e743634185d2003280442021000121b0a0f72657065617465645f" .
+    "73696e743332185e2003281142021000121b0a0f72657065617465645f73" .
+    "696e743634185f2003281242021000121c0a1072657065617465645f6669" .
+    "786564333218602003280742021000121c0a1072657065617465645f6669" .
+    "786564363418612003280642021000121d0a1172657065617465645f7366" .
+    "69786564333218622003280f42021000121d0a1172657065617465645f73" .
+    "6669786564363418632003281042021000121a0a0e72657065617465645f" .
+    "666c6f617418642003280242021000121b0a0f72657065617465645f646f" .
+    "75626c651865200328014202100012190a0d72657065617465645f626f6f" .
+    "6c1866200328084202100012280a0d72657065617465645f656e756d1867" .
+    "2003280e320d2e666f6f2e54657374456e756d420210002a1d0a08546573" .
+    "74456e756d12080a045a45524f100012070a034f4e451001620670726f74" .
+    "6f33"
+));
+

php/tests/test_include.pb.php

@@ -0,0 +1,36 @@
+<?php
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: test_include.proto
+
+namespace Bar;
+
+use Google\Protobuf\Internal\DescriptorPool;
+use Google\Protobuf\Internal\GPBType;
+use Google\Protobuf\Internal\RepeatedField;
+use Google\Protobuf\Internal\GPBUtil;
+
+class TestInclude extends \Google\Protobuf\Internal\Message
+{
+    private $a = 0;
+
+    public function getA()
+    {
+        return $this->a;
+    }
+
+    public function setA($var)
+    {
+        GPBUtil::checkInt32($var);
+        $this->a = $var;
+    }
+
+}
+
+$pool = DescriptorPool::getGeneratedPool();
+
+$pool->internalAddGeneratedFile(hex2bin(
+    "0a3b0a12746573745f696e636c7564652e70726f746f120362617222180a" .
+    "0b54657374496e636c75646512090a0161180120012805620670726f746f" .
+    "33"
+));
+

python/google/protobuf/descriptor.py

@@ -171,13 +171,6 @@ class _NestedDescriptorBase(DescriptorBase):
     self._serialized_start = serialized_start
     self._serialized_end = serialized_end
 
-  def GetTopLevelContainingType(self):
-    """Returns the root if this is a nested type, or itself if its the root."""
-    desc = self
-    while desc.containing_type is not None:
-      desc = desc.containing_type
-    return desc
-
   def CopyToProto(self, proto):
     """Copies this to the matching proto in descriptor_pb2.
 
@@ -497,7 +490,7 @@ class FieldDescriptor(DescriptorBase):
     def __new__(cls, name, full_name, index, number, type, cpp_type, label,
                 default_value, message_type, enum_type, containing_type,
                 is_extension, extension_scope, options=None,
-                has_default_value=True, containing_oneof=None):
+                has_default_value=True, containing_oneof=None, json_name=None):
       _message.Message._CheckCalledFromGeneratedFile()
       if is_extension:
         return _message.default_pool.FindExtensionByName(full_name)
@@ -507,7 +500,7 @@ class FieldDescriptor(DescriptorBase):
   def __init__(self, name, full_name, index, number, type, cpp_type, label,
                default_value, message_type, enum_type, containing_type,
                is_extension, extension_scope, options=None,
-               has_default_value=True, containing_oneof=None):
+               has_default_value=True, containing_oneof=None, json_name=None):
     """The arguments are as described in the description of FieldDescriptor
     attributes above.
 
@@ -519,6 +512,10 @@ class FieldDescriptor(DescriptorBase):
     self.name = name
     self.full_name = full_name
     self._camelcase_name = None
+    if json_name is None:
+      self.json_name = _ToJsonName(name)
+    else:
+      self.json_name = json_name
     self.index = index
     self.number = number
     self.type = type
@@ -894,6 +891,31 @@ def _ToCamelCase(name):
   return ''.join(result)
 
 
+def _OptionsOrNone(descriptor_proto):
+  """Returns the value of the field `options`, or None if it is not set."""
+  if descriptor_proto.HasField('options'):
+    return descriptor_proto.options
+  else:
+    return None
+
+
+def _ToJsonName(name):
+  """Converts name to Json name and returns it."""
+  capitalize_next = False
+  result = []
+
+  for c in name:
+    if c == '_':
+      capitalize_next = True
+    elif capitalize_next:
+      result.append(c.upper())
+      capitalize_next = False
+    else:
+      result += c
+
+  return ''.join(result)
+
+
 def MakeDescriptor(desc_proto, package='', build_file_if_cpp=True,
                    syntax=None):
   """Make a protobuf Descriptor given a DescriptorProto protobuf.
@@ -970,6 +992,10 @@ def MakeDescriptor(desc_proto, package='', build_file_if_cpp=True,
     full_name = '.'.join(full_message_name + [field_proto.name])
     enum_desc = None
     nested_desc = None
+    if field_proto.json_name:
+      json_name = field_proto.json_name
+    else:
+      json_name = None
     if field_proto.HasField('type_name'):
       type_name = field_proto.type_name
       full_type_name = '.'.join(full_message_name +
@@ -984,10 +1010,11 @@ def MakeDescriptor(desc_proto, package='', build_file_if_cpp=True,
         field_proto.number, field_proto.type,
         FieldDescriptor.ProtoTypeToCppProtoType(field_proto.type),
         field_proto.label, None, nested_desc, enum_desc, None, False, None,
-        options=field_proto.options, has_default_value=False)
+        options=_OptionsOrNone(field_proto), has_default_value=False,
+        json_name=json_name)
     fields.append(field)
 
   desc_name = '.'.join(full_message_name)
   return Descriptor(desc_proto.name, desc_name, None, None, fields,
                     list(nested_types.values()), list(enum_types.values()), [],
-                    options=desc_proto.options)
+                    options=_OptionsOrNone(desc_proto))

python/google/protobuf/descriptor_pool.py

@@ -62,7 +62,7 @@ from google.protobuf import descriptor_database
 from google.protobuf import text_encoding
 
 
-_USE_C_DESCRIPTORS = descriptor._USE_C_DESCRIPTORS
+_USE_C_DESCRIPTORS = descriptor._USE_C_DESCRIPTORS  # pylint: disable=protected-access
 
 
 def _NormalizeFullyQualifiedName(name):
@@ -80,6 +80,14 @@ def _NormalizeFullyQualifiedName(name):
   return name.lstrip('.')
 
 
+def _OptionsOrNone(descriptor_proto):
+  """Returns the value of the field `options`, or None if it is not set."""
+  if descriptor_proto.HasField('options'):
+    return descriptor_proto.options
+  else:
+    return None
+
+
 class DescriptorPool(object):
   """A collection of protobufs dynamically constructed by descriptor protos."""
 
@@ -326,78 +334,61 @@ class DescriptorPool(object):
           name=file_proto.name,
           package=file_proto.package,
           syntax=file_proto.syntax,
-          options=file_proto.options,
+          options=_OptionsOrNone(file_proto),
           serialized_pb=file_proto.SerializeToString(),
           dependencies=direct_deps,
           public_dependencies=public_deps)
-      if _USE_C_DESCRIPTORS:
-        # When using C++ descriptors, all objects defined in the file were added
-        # to the C++ database when the FileDescriptor was built above.
-        # Just add them to this descriptor pool.
-        def _AddMessageDescriptor(message_desc):
-          self._descriptors[message_desc.full_name] = message_desc
-          for nested in message_desc.nested_types:
-            _AddMessageDescriptor(nested)
-          for enum_type in message_desc.enum_types:
-            _AddEnumDescriptor(enum_type)
-        def _AddEnumDescriptor(enum_desc):
-          self._enum_descriptors[enum_desc.full_name] = enum_desc
-        for message_type in file_descriptor.message_types_by_name.values():
-          _AddMessageDescriptor(message_type)
-        for enum_type in file_descriptor.enum_types_by_name.values():
-          _AddEnumDescriptor(enum_type)
+      scope = {}
+
+      # This loop extracts all the message and enum types from all the
+      # dependencies of the file_proto. This is necessary to create the
+      # scope of available message types when defining the passed in
+      # file proto.
+      for dependency in built_deps:
+        scope.update(self._ExtractSymbols(
+            dependency.message_types_by_name.values()))
+        scope.update((_PrefixWithDot(enum.full_name), enum)
+                     for enum in dependency.enum_types_by_name.values())
+
+      for message_type in file_proto.message_type:
+        message_desc = self._ConvertMessageDescriptor(
+            message_type, file_proto.package, file_descriptor, scope,
+            file_proto.syntax)
+        file_descriptor.message_types_by_name[message_desc.name] = (
+            message_desc)
+
+      for enum_type in file_proto.enum_type:
+        file_descriptor.enum_types_by_name[enum_type.name] = (
+            self._ConvertEnumDescriptor(enum_type, file_proto.package,
+                                        file_descriptor, None, scope))
+
+      for index, extension_proto in enumerate(file_proto.extension):
+        extension_desc = self._MakeFieldDescriptor(
+            extension_proto, file_proto.package, index, is_extension=True)
+        extension_desc.containing_type = self._GetTypeFromScope(
+            file_descriptor.package, extension_proto.extendee, scope)
+        self._SetFieldType(extension_proto, extension_desc,
+                           file_descriptor.package, scope)
+        file_descriptor.extensions_by_name[extension_desc.name] = (
+            extension_desc)
+
+      for desc_proto in file_proto.message_type:
+        self._SetAllFieldTypes(file_proto.package, desc_proto, scope)
+
+      if file_proto.package:
+        desc_proto_prefix = _PrefixWithDot(file_proto.package)
       else:
-        scope = {}
-
-        # This loop extracts all the message and enum types from all the
-        # dependencies of the file_proto. This is necessary to create the
-        # scope of available message types when defining the passed in
-        # file proto.
-        for dependency in built_deps:
-          scope.update(self._ExtractSymbols(
-              dependency.message_types_by_name.values()))
-          scope.update((_PrefixWithDot(enum.full_name), enum)
-                       for enum in dependency.enum_types_by_name.values())
-
-        for message_type in file_proto.message_type:
-          message_desc = self._ConvertMessageDescriptor(
-              message_type, file_proto.package, file_descriptor, scope,
-              file_proto.syntax)
-          file_descriptor.message_types_by_name[message_desc.name] = (
-              message_desc)
-
-        for enum_type in file_proto.enum_type:
-          file_descriptor.enum_types_by_name[enum_type.name] = (
-              self._ConvertEnumDescriptor(enum_type, file_proto.package,
-                                          file_descriptor, None, scope))
-
-        for index, extension_proto in enumerate(file_proto.extension):
-          extension_desc = self._MakeFieldDescriptor(
-              extension_proto, file_proto.package, index, is_extension=True)
-          extension_desc.containing_type = self._GetTypeFromScope(
-              file_descriptor.package, extension_proto.extendee, scope)
-          self._SetFieldType(extension_proto, extension_desc,
-                            file_descriptor.package, scope)
-          file_descriptor.extensions_by_name[extension_desc.name] = (
-              extension_desc)
-
-        for desc_proto in file_proto.message_type:
-          self._SetAllFieldTypes(file_proto.package, desc_proto, scope)
-
-        if file_proto.package:
-          desc_proto_prefix = _PrefixWithDot(file_proto.package)
-        else:
-          desc_proto_prefix = ''
+        desc_proto_prefix = ''
 
-        for desc_proto in file_proto.message_type:
-          desc = self._GetTypeFromScope(
-              desc_proto_prefix, desc_proto.name, scope)
-          file_descriptor.message_types_by_name[desc_proto.name] = desc
+      for desc_proto in file_proto.message_type:
+        desc = self._GetTypeFromScope(
+            desc_proto_prefix, desc_proto.name, scope)
+        file_descriptor.message_types_by_name[desc_proto.name] = desc
 
-        for index, service_proto in enumerate(file_proto.service):
-          file_descriptor.services_by_name[service_proto.name] = (
-              self._MakeServiceDescriptor(service_proto, index, scope,
-                                          file_proto.package, file_descriptor))
+      for index, service_proto in enumerate(file_proto.service):
+        file_descriptor.services_by_name[service_proto.name] = (
+            self._MakeServiceDescriptor(service_proto, index, scope,
+                                        file_proto.package, file_descriptor))
 
       self.Add(file_proto)
       self._file_descriptors[file_proto.name] = file_descriptor
@@ -413,6 +404,7 @@ class DescriptorPool(object):
       package: The package the proto should be located in.
       file_desc: The file containing this message.
       scope: Dict mapping short and full symbols to message and enum types.
+      syntax: string indicating syntax of the file ("proto2" or "proto3")
 
     Returns:
       The added descriptor.
@@ -463,7 +455,7 @@ class DescriptorPool(object):
         nested_types=nested,
         enum_types=enums,
         extensions=extensions,
-        options=desc_proto.options,
+        options=_OptionsOrNone(desc_proto),
         is_extendable=is_extendable,
         extension_ranges=extension_ranges,
         file=file_desc,
@@ -517,7 +509,7 @@ class DescriptorPool(object):
                                      file=file_desc,
                                      values=values,
                                      containing_type=containing_type,
-                                     options=enum_proto.options)
+                                     options=_OptionsOrNone(enum_proto))
     scope['.%s' % enum_name] = desc
     self._enum_descriptors[enum_name] = desc
     return desc
@@ -562,7 +554,7 @@ class DescriptorPool(object):
         default_value=None,
         is_extension=is_extension,
         extension_scope=None,
-        options=field_proto.options)
+        options=_OptionsOrNone(field_proto))
 
   def _SetAllFieldTypes(self, package, desc_proto, scope):
     """Sets all the descriptor's fields's types.
@@ -681,7 +673,7 @@ class DescriptorPool(object):
         name=value_proto.name,
         index=index,
         number=value_proto.number,
-        options=value_proto.options,
+        options=_OptionsOrNone(value_proto),
         type=None)
 
   def _MakeServiceDescriptor(self, service_proto, service_index, scope,
@@ -711,7 +703,7 @@ class DescriptorPool(object):
                                         full_name=service_name,
                                         index=service_index,
                                         methods=methods,
-                                        options=service_proto.options,
+                                        options=_OptionsOrNone(service_proto),
                                         file=file_desc)
     return desc
 
@@ -740,7 +732,7 @@ class DescriptorPool(object):
                                        containing_service=None,
                                        input_type=input_type,
                                        output_type=output_type,
-                                       options=method_proto.options)
+                                       options=_OptionsOrNone(method_proto))
 
   def _ExtractSymbols(self, descriptors):
     """Pulls out all the symbols from descriptor protos.

python/google/protobuf/internal/any_test.proto

@@ -30,13 +30,21 @@
 
 // Author: jieluo@google.com (Jie Luo)
 
-syntax = "proto3";
+syntax = "proto2";
 
 package google.protobuf.internal;
 
 import "google/protobuf/any.proto";
 
 message TestAny {
-  google.protobuf.Any value = 1;
-  int32 int_value = 2;
+  optional google.protobuf.Any value = 1;
+  optional int32 int_value = 2;
+  extensions 10 to max;
+}
+
+message TestAnyExtension1 {
+  extend TestAny {
+    optional TestAnyExtension1 extension1 = 98418603;
+  }
+  optional int32 i = 15;
 }

python/google/protobuf/internal/containers.py

@@ -436,9 +436,11 @@ class ScalarMap(MutableMapping):
   """Simple, type-checked, dict-like container for holding repeated scalars."""
 
   # Disallows assignment to other attributes.
-  __slots__ = ['_key_checker', '_value_checker', '_values', '_message_listener']
+  __slots__ = ['_key_checker', '_value_checker', '_values', '_message_listener',
+               '_entry_descriptor']
 
-  def __init__(self, message_listener, key_checker, value_checker):
+  def __init__(self, message_listener, key_checker, value_checker,
+               entry_descriptor):
     """
     Args:
       message_listener: A MessageListener implementation.
@@ -448,10 +450,12 @@ class ScalarMap(MutableMapping):
         inserted into this container.
       value_checker: A type_checkers.ValueChecker instance to run on values
         inserted into this container.
+      entry_descriptor: The MessageDescriptor of a map entry: key and value.
     """
     self._message_listener = message_listener
     self._key_checker = key_checker
     self._value_checker = value_checker
+    self._entry_descriptor = entry_descriptor
     self._values = {}
 
   def __getitem__(self, key):
@@ -513,6 +517,9 @@ class ScalarMap(MutableMapping):
     self._values.clear()
     self._message_listener.Modified()
 
+  def GetEntryClass(self):
+    return self._entry_descriptor._concrete_class
+
 
 class MessageMap(MutableMapping):
 
@@ -520,9 +527,10 @@ class MessageMap(MutableMapping):
 
   # Disallows assignment to other attributes.
   __slots__ = ['_key_checker', '_values', '_message_listener',
-               '_message_descriptor']
+               '_message_descriptor', '_entry_descriptor']
 
-  def __init__(self, message_listener, message_descriptor, key_checker):
+  def __init__(self, message_listener, message_descriptor, key_checker,
+               entry_descriptor):
     """
     Args:
       message_listener: A MessageListener implementation.
@@ -532,10 +540,12 @@ class MessageMap(MutableMapping):
         inserted into this container.
       value_checker: A type_checkers.ValueChecker instance to run on values
         inserted into this container.
+      entry_descriptor: The MessageDescriptor of a map entry: key and value.
     """
     self._message_listener = message_listener
     self._message_descriptor = message_descriptor
     self._key_checker = key_checker
+    self._entry_descriptor = entry_descriptor
     self._values = {}
 
   def __getitem__(self, key):
@@ -613,3 +623,6 @@ class MessageMap(MutableMapping):
   def clear(self):
     self._values.clear()
     self._message_listener.Modified()
+
+  def GetEntryClass(self):
+    return self._entry_descriptor._concrete_class

python/google/protobuf/internal/descriptor_pool_test.py

@@ -119,6 +119,7 @@ class DescriptorPoolTest(unittest.TestCase):
     self.assertEqual('google.protobuf.python.internal.Factory1Message',
                      msg1.full_name)
     self.assertEqual(None, msg1.containing_type)
+    self.assertFalse(msg1.has_options)
 
     nested_msg1 = msg1.nested_types[0]
     self.assertEqual('NestedFactory1Message', nested_msg1.name)
@@ -202,6 +203,7 @@ class DescriptorPoolTest(unittest.TestCase):
     self.assertIsInstance(enum1, descriptor.EnumDescriptor)
     self.assertEqual(0, enum1.values_by_name['FACTORY_1_VALUE_0'].number)
     self.assertEqual(1, enum1.values_by_name['FACTORY_1_VALUE_1'].number)
+    self.assertFalse(enum1.has_options)
 
     nested_enum1 = self.pool.FindEnumTypeByName(
         'google.protobuf.python.internal.Factory1Message.NestedFactory1Enum')
@@ -234,6 +236,8 @@ class DescriptorPoolTest(unittest.TestCase):
         'google.protobuf.python.internal.Factory1Message.list_value')
     self.assertEqual(field.name, 'list_value')
     self.assertEqual(field.label, field.LABEL_REPEATED)
+    self.assertFalse(field.has_options)
+
     with self.assertRaises(KeyError):
       self.pool.FindFieldByName('Does not exist')
 
@@ -448,6 +452,7 @@ class EnumField(object):
     test.assertTrue(field_desc.has_default_value)
     test.assertEqual(enum_desc.values_by_name[self.default_value].number,
                      field_desc.default_value)
+    test.assertFalse(enum_desc.values_by_name[self.default_value].has_options)
     test.assertEqual(msg_desc, field_desc.containing_type)
     test.assertEqual(enum_desc, field_desc.enum_type)
 

python/google/protobuf/internal/descriptor_test.py

@@ -766,6 +766,8 @@ class MakeDescriptorTest(unittest.TestCase):
                      'Foo2.Sub.bar_field')
     self.assertEqual(result.nested_types[0].fields[0].enum_type,
                      result.nested_types[0].enum_types[0])
+    self.assertFalse(result.has_options)
+    self.assertFalse(result.fields[0].has_options)
 
   def testMakeDescriptorWithUnsignedIntField(self):
     file_descriptor_proto = descriptor_pb2.FileDescriptorProto()
@@ -818,6 +820,23 @@ class MakeDescriptorTest(unittest.TestCase):
       self.assertEqual(result.fields[index].camelcase_name,
                        camelcase_names[index])
 
+  def testJsonName(self):
+    descriptor_proto = descriptor_pb2.DescriptorProto()
+    descriptor_proto.name = 'TestJsonName'
+    names = ['field_name', 'fieldName', 'FieldName',
+             '_field_name', 'FIELD_NAME', 'json_name']
+    json_names = ['fieldName', 'fieldName', 'FieldName',
+                  'FieldName', 'FIELDNAME', '@type']
+    for index in range(len(names)):
+      field = descriptor_proto.field.add()
+      field.number = index + 1
+      field.name = names[index]
+    field.json_name = '@type'
+    result = descriptor.MakeDescriptor(descriptor_proto)
+    for index in range(len(json_names)):
+      self.assertEqual(result.fields[index].json_name,
+                       json_names[index])
+
 
 if __name__ == '__main__':
   unittest.main()