Fixed Python by updating failure lists and fixed a few broken tests.

Python 2.x doesn't detect unpaired surrogates so we have to
do that manually.

conformance/Makefile.am

@@ -268,10 +268,10 @@ test_ruby: protoc_middleman conformance-test-runner $(other_language_protoc_outp
 # These depend on library paths being properly set up.  The easiest way to
 # run them is to just use "tox" from the python dir.
 test_python: protoc_middleman conformance-test-runner
-	./conformance-test-runner --failure_list failure_list_python.txt $(CONFORMANCE_PYTHON_EXTRA_FAILURES) ./conformance_python.py
+	./conformance-test-runner --failure_list failure_list_python.txt ./conformance_python.py
 
 test_python_cpp: protoc_middleman conformance-test-runner
-	./conformance-test-runner --failure_list failure_list_python_cpp.txt $(CONFORMANCE_PYTHON_EXTRA_FAILURES) ./conformance_python.py
+	./conformance-test-runner --failure_list failure_list_python_cpp.txt ./conformance_python.py
 
 if OBJC_CONFORMANCE_TEST
 

conformance/failure_list_csharp.txt

@@ -1,16 +1,11 @@
-JsonInput.AnyWithValueForInteger.JsonOutput
-JsonInput.AnyWithValueForJsonObject.JsonOutput
 JsonInput.FieldNameInLowerCamelCase.Validator
 JsonInput.FieldNameInSnakeCase.JsonOutput
 JsonInput.FieldNameInSnakeCase.ProtobufOutput
 JsonInput.FieldNameWithMixedCases.JsonOutput
 JsonInput.FieldNameWithMixedCases.ProtobufOutput
 JsonInput.FieldNameWithMixedCases.Validator
-JsonInput.Int32FieldMinFloatValue.JsonOutput
-JsonInput.Int32FieldMinValue.JsonOutput
 JsonInput.Int64FieldMaxValueNotQuoted.JsonOutput
 JsonInput.Int64FieldMaxValueNotQuoted.ProtobufOutput
 JsonInput.OriginalProtoFieldName.JsonOutput
-JsonInput.StringFieldSurrogatePair.JsonOutput
 JsonInput.Uint64FieldMaxValueNotQuoted.JsonOutput
 JsonInput.Uint64FieldMaxValueNotQuoted.ProtobufOutput

conformance/failure_list_python.txt

@@ -3,26 +3,7 @@ DurationProtoInputTooSmall.JsonOutput
 FieldMaskNumbersDontRoundTrip.JsonOutput
 FieldMaskPathsDontRoundTrip.JsonOutput
 FieldMaskTooManyUnderscore.JsonOutput
-JsonInput.Any.JsonOutput
-JsonInput.Any.ProtobufOutput
-JsonInput.AnyNested.JsonOutput
-JsonInput.AnyNested.ProtobufOutput
-JsonInput.AnyUnorderedTypeTag.JsonOutput
-JsonInput.AnyUnorderedTypeTag.ProtobufOutput
-JsonInput.AnyWithDuration.JsonOutput
-JsonInput.AnyWithDuration.ProtobufOutput
-JsonInput.AnyWithFieldMask.JsonOutput
 JsonInput.AnyWithFieldMask.ProtobufOutput
-JsonInput.AnyWithInt32ValueWrapper.JsonOutput
-JsonInput.AnyWithInt32ValueWrapper.ProtobufOutput
-JsonInput.AnyWithStruct.JsonOutput
-JsonInput.AnyWithStruct.ProtobufOutput
-JsonInput.AnyWithTimestamp.JsonOutput
-JsonInput.AnyWithTimestamp.ProtobufOutput
-JsonInput.AnyWithValueForInteger.JsonOutput
-JsonInput.AnyWithValueForInteger.ProtobufOutput
-JsonInput.AnyWithValueForJsonObject.JsonOutput
-JsonInput.AnyWithValueForJsonObject.ProtobufOutput
 JsonInput.BytesFieldInvalidBase64Characters
 JsonInput.DoubleFieldInfinityNotQuoted
 JsonInput.DoubleFieldNanNotQuoted
@@ -54,32 +35,29 @@ JsonInput.Int32FieldMaxFloatValue.JsonOutput
 JsonInput.Int32FieldMaxFloatValue.ProtobufOutput
 JsonInput.Int32FieldMinFloatValue.JsonOutput
 JsonInput.Int32FieldMinFloatValue.ProtobufOutput
-JsonInput.Int32FieldMinValue.JsonOutput
 JsonInput.OriginalProtoFieldName.JsonOutput
 JsonInput.OriginalProtoFieldName.ProtobufOutput
-JsonInput.RepeatedFieldMessageElementIsNull
-JsonInput.RepeatedFieldPrimitiveElementIsNull
 JsonInput.RepeatedFieldWrongElementTypeExpectingIntegersGotBool
-JsonInput.StringFieldSurrogatePair.JsonOutput
-JsonInput.StringFieldUnpairedLowSurrogate
-JsonInput.Struct.JsonOutput
-JsonInput.Struct.ProtobufOutput
 JsonInput.TimestampJsonInputLowercaseT
 JsonInput.Uint32FieldMaxFloatValue.JsonOutput
 JsonInput.Uint32FieldMaxFloatValue.ProtobufOutput
-JsonInput.ValueAcceptBool.JsonOutput
-JsonInput.ValueAcceptBool.ProtobufOutput
-JsonInput.ValueAcceptFloat.JsonOutput
-JsonInput.ValueAcceptFloat.ProtobufOutput
-JsonInput.ValueAcceptInteger.JsonOutput
-JsonInput.ValueAcceptInteger.ProtobufOutput
-JsonInput.ValueAcceptList.JsonOutput
-JsonInput.ValueAcceptList.ProtobufOutput
 JsonInput.ValueAcceptNull.JsonOutput
 JsonInput.ValueAcceptNull.ProtobufOutput
-JsonInput.ValueAcceptObject.JsonOutput
-JsonInput.ValueAcceptObject.ProtobufOutput
-JsonInput.ValueAcceptString.JsonOutput
-JsonInput.ValueAcceptString.ProtobufOutput
+ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE
+ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE
+ProtobufInput.PrematureEofInPackedField.BOOL
+ProtobufInput.PrematureEofInPackedField.DOUBLE
+ProtobufInput.PrematureEofInPackedField.ENUM
+ProtobufInput.PrematureEofInPackedField.FIXED32
+ProtobufInput.PrematureEofInPackedField.FIXED64
+ProtobufInput.PrematureEofInPackedField.FLOAT
+ProtobufInput.PrematureEofInPackedField.INT32
+ProtobufInput.PrematureEofInPackedField.INT64
+ProtobufInput.PrematureEofInPackedField.SFIXED32
+ProtobufInput.PrematureEofInPackedField.SFIXED64
+ProtobufInput.PrematureEofInPackedField.SINT32
+ProtobufInput.PrematureEofInPackedField.SINT64
+ProtobufInput.PrematureEofInPackedField.UINT32
+ProtobufInput.PrematureEofInPackedField.UINT64
 TimestampProtoInputTooLarge.JsonOutput
 TimestampProtoInputTooSmall.JsonOutput

conformance/failure_list_python_cpp.txt

@@ -12,26 +12,7 @@ DurationProtoInputTooSmall.JsonOutput
 FieldMaskNumbersDontRoundTrip.JsonOutput
 FieldMaskPathsDontRoundTrip.JsonOutput
 FieldMaskTooManyUnderscore.JsonOutput
-JsonInput.Any.JsonOutput
-JsonInput.Any.ProtobufOutput
-JsonInput.AnyNested.JsonOutput
-JsonInput.AnyNested.ProtobufOutput
-JsonInput.AnyUnorderedTypeTag.JsonOutput
-JsonInput.AnyUnorderedTypeTag.ProtobufOutput
-JsonInput.AnyWithDuration.JsonOutput
-JsonInput.AnyWithDuration.ProtobufOutput
-JsonInput.AnyWithFieldMask.JsonOutput
 JsonInput.AnyWithFieldMask.ProtobufOutput
-JsonInput.AnyWithInt32ValueWrapper.JsonOutput
-JsonInput.AnyWithInt32ValueWrapper.ProtobufOutput
-JsonInput.AnyWithStruct.JsonOutput
-JsonInput.AnyWithStruct.ProtobufOutput
-JsonInput.AnyWithTimestamp.JsonOutput
-JsonInput.AnyWithTimestamp.ProtobufOutput
-JsonInput.AnyWithValueForInteger.JsonOutput
-JsonInput.AnyWithValueForInteger.ProtobufOutput
-JsonInput.AnyWithValueForJsonObject.JsonOutput
-JsonInput.AnyWithValueForJsonObject.ProtobufOutput
 JsonInput.BytesFieldInvalidBase64Characters
 JsonInput.DoubleFieldInfinityNotQuoted
 JsonInput.DoubleFieldNanNotQuoted
@@ -63,33 +44,14 @@ JsonInput.Int32FieldMaxFloatValue.JsonOutput
 JsonInput.Int32FieldMaxFloatValue.ProtobufOutput
 JsonInput.Int32FieldMinFloatValue.JsonOutput
 JsonInput.Int32FieldMinFloatValue.ProtobufOutput
-JsonInput.Int32FieldMinValue.JsonOutput
 JsonInput.OriginalProtoFieldName.JsonOutput
 JsonInput.OriginalProtoFieldName.ProtobufOutput
-JsonInput.RepeatedFieldMessageElementIsNull
-JsonInput.RepeatedFieldPrimitiveElementIsNull
 JsonInput.RepeatedFieldWrongElementTypeExpectingIntegersGotBool
-JsonInput.StringFieldSurrogatePair.JsonOutput
-JsonInput.StringFieldUnpairedLowSurrogate
-JsonInput.Struct.JsonOutput
-JsonInput.Struct.ProtobufOutput
 JsonInput.TimestampJsonInputLowercaseT
 JsonInput.Uint32FieldMaxFloatValue.JsonOutput
 JsonInput.Uint32FieldMaxFloatValue.ProtobufOutput
-JsonInput.ValueAcceptBool.JsonOutput
-JsonInput.ValueAcceptBool.ProtobufOutput
-JsonInput.ValueAcceptFloat.JsonOutput
-JsonInput.ValueAcceptFloat.ProtobufOutput
-JsonInput.ValueAcceptInteger.JsonOutput
-JsonInput.ValueAcceptInteger.ProtobufOutput
-JsonInput.ValueAcceptList.JsonOutput
-JsonInput.ValueAcceptList.ProtobufOutput
 JsonInput.ValueAcceptNull.JsonOutput
 JsonInput.ValueAcceptNull.ProtobufOutput
-JsonInput.ValueAcceptObject.JsonOutput
-JsonInput.ValueAcceptObject.ProtobufOutput
-JsonInput.ValueAcceptString.JsonOutput
-JsonInput.ValueAcceptString.ProtobufOutput
 ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE
 ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE
 ProtobufInput.PrematureEofInPackedField.BOOL

python/google/protobuf/json_format.py

@@ -49,6 +49,7 @@ except ImportError:
 import base64
 import json
 import math
+import re
 import six
 import sys
 
@@ -68,6 +69,10 @@ _INFINITY = 'Infinity'
 _NEG_INFINITY = '-Infinity'
 _NAN = 'NaN'
 
+if sys.version_info < (3, 0):
+  _UNPAIRED_SURROGATE_PATTERN = re.compile(six.u(
+      r'[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]'
+  ))
 
 class Error(Exception):
   """Top-level module error for json_format."""
@@ -554,6 +559,11 @@ def _ConvertScalarFieldValue(value, field, require_str=False):
   elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING:
     if field.type == descriptor.FieldDescriptor.TYPE_BYTES:
       return base64.b64decode(value)
+    elif sys.version_info < (3, 0):
+      # Python 2.x does not detect unpaired surrogates when JSON parsing.
+      if _UNPAIRED_SURROGATE_PATTERN.search(value):
+        raise ParseError('Unpaired surrogate')
+      return value
     else:
       return value
   elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM: