Adds the ability to ignore unknown fields on parse

This adds the ability for the MRI Ruby library to optionally pass in a
ignore_unknown_fields option when decoding JSON. The functionality was
added upstream in upb, this change exposes that option.

conformance/failure_list_ruby.txt

@@ -5,8 +5,6 @@ Recommended.Proto3.JsonInput.BytesFieldBase64Url.JsonOutput
 Recommended.Proto3.JsonInput.BytesFieldBase64Url.ProtobufOutput
 Recommended.Proto3.JsonInput.DurationHas3FractionalDigits.Validator
 Recommended.Proto3.JsonInput.DurationHas6FractionalDigits.Validator
-Recommended.Proto3.JsonInput.DurationHas9FractionalDigits.Validator
-Recommended.Proto3.JsonInput.DurationHasZeroFractionalDigit.Validator
 Recommended.Proto3.JsonInput.Int64FieldBeString.Validator
 Recommended.Proto3.JsonInput.MapFieldValueIsNull
 Recommended.Proto3.JsonInput.RepeatedFieldMessageElementIsNull
@@ -17,9 +15,6 @@ Recommended.Proto3.JsonInput.StringFieldUnpairedHighSurrogate
 Recommended.Proto3.JsonInput.StringFieldUnpairedLowSurrogate
 Recommended.Proto3.JsonInput.TimestampHas3FractionalDigits.Validator
 Recommended.Proto3.JsonInput.TimestampHas6FractionalDigits.Validator
-Recommended.Proto3.JsonInput.TimestampHas9FractionalDigits.Validator
-Recommended.Proto3.JsonInput.TimestampHasZeroFractionalDigit.Validator
-Recommended.Proto3.JsonInput.TimestampZeroNormalized.Validator
 Recommended.Proto3.JsonInput.Uint64FieldBeString.Validator
 Required.DurationProtoInputTooLarge.JsonOutput
 Required.DurationProtoInputTooSmall.JsonOutput
@@ -48,12 +43,8 @@ Required.Proto3.JsonInput.DoubleFieldMaxNegativeValue.ProtobufOutput
 Required.Proto3.JsonInput.DoubleFieldMinPositiveValue.JsonOutput
 Required.Proto3.JsonInput.DoubleFieldMinPositiveValue.ProtobufOutput
 Required.Proto3.JsonInput.DoubleFieldNan.JsonOutput
-Required.Proto3.JsonInput.DurationMaxValue.JsonOutput
-Required.Proto3.JsonInput.DurationMaxValue.ProtobufOutput
 Required.Proto3.JsonInput.DurationMinValue.JsonOutput
-Required.Proto3.JsonInput.DurationMinValue.ProtobufOutput
 Required.Proto3.JsonInput.DurationRepeatedValue.JsonOutput
-Required.Proto3.JsonInput.DurationRepeatedValue.ProtobufOutput
 Required.Proto3.JsonInput.FieldMask.JsonOutput
 Required.Proto3.JsonInput.FieldMask.ProtobufOutput
 Required.Proto3.JsonInput.FloatFieldInfinity.JsonOutput
@@ -61,71 +52,32 @@ Required.Proto3.JsonInput.FloatFieldNan.JsonOutput
 Required.Proto3.JsonInput.FloatFieldNegativeInfinity.JsonOutput
 Required.Proto3.JsonInput.OneofFieldDuplicate
 Required.Proto3.JsonInput.OptionalBoolWrapper.JsonOutput
-Required.Proto3.JsonInput.OptionalBoolWrapper.ProtobufOutput
 Required.Proto3.JsonInput.OptionalBytesWrapper.JsonOutput
-Required.Proto3.JsonInput.OptionalBytesWrapper.ProtobufOutput
 Required.Proto3.JsonInput.OptionalDoubleWrapper.JsonOutput
-Required.Proto3.JsonInput.OptionalDoubleWrapper.ProtobufOutput
 Required.Proto3.JsonInput.OptionalFloatWrapper.JsonOutput
-Required.Proto3.JsonInput.OptionalFloatWrapper.ProtobufOutput
 Required.Proto3.JsonInput.OptionalInt32Wrapper.JsonOutput
-Required.Proto3.JsonInput.OptionalInt32Wrapper.ProtobufOutput
 Required.Proto3.JsonInput.OptionalInt64Wrapper.JsonOutput
-Required.Proto3.JsonInput.OptionalInt64Wrapper.ProtobufOutput
 Required.Proto3.JsonInput.OptionalStringWrapper.JsonOutput
-Required.Proto3.JsonInput.OptionalStringWrapper.ProtobufOutput
 Required.Proto3.JsonInput.OptionalUint32Wrapper.JsonOutput
-Required.Proto3.JsonInput.OptionalUint32Wrapper.ProtobufOutput
 Required.Proto3.JsonInput.OptionalUint64Wrapper.JsonOutput
-Required.Proto3.JsonInput.OptionalUint64Wrapper.ProtobufOutput
 Required.Proto3.JsonInput.OptionalWrapperTypesWithNonDefaultValue.JsonOutput
 Required.Proto3.JsonInput.OptionalWrapperTypesWithNonDefaultValue.ProtobufOutput
 Required.Proto3.JsonInput.RepeatedBoolWrapper.JsonOutput
-Required.Proto3.JsonInput.RepeatedBoolWrapper.ProtobufOutput
 Required.Proto3.JsonInput.RepeatedBytesWrapper.JsonOutput
-Required.Proto3.JsonInput.RepeatedBytesWrapper.ProtobufOutput
 Required.Proto3.JsonInput.RepeatedDoubleWrapper.JsonOutput
-Required.Proto3.JsonInput.RepeatedDoubleWrapper.ProtobufOutput
 Required.Proto3.JsonInput.RepeatedFloatWrapper.JsonOutput
-Required.Proto3.JsonInput.RepeatedFloatWrapper.ProtobufOutput
 Required.Proto3.JsonInput.RepeatedInt32Wrapper.JsonOutput
-Required.Proto3.JsonInput.RepeatedInt32Wrapper.ProtobufOutput
 Required.Proto3.JsonInput.RepeatedInt64Wrapper.JsonOutput
-Required.Proto3.JsonInput.RepeatedInt64Wrapper.ProtobufOutput
 Required.Proto3.JsonInput.RepeatedStringWrapper.JsonOutput
-Required.Proto3.JsonInput.RepeatedStringWrapper.ProtobufOutput
 Required.Proto3.JsonInput.RepeatedUint32Wrapper.JsonOutput
-Required.Proto3.JsonInput.RepeatedUint32Wrapper.ProtobufOutput
 Required.Proto3.JsonInput.RepeatedUint64Wrapper.JsonOutput
-Required.Proto3.JsonInput.RepeatedUint64Wrapper.ProtobufOutput
 Required.Proto3.JsonInput.StringFieldSurrogatePair.JsonOutput
 Required.Proto3.JsonInput.StringFieldSurrogatePair.ProtobufOutput
-Required.Proto3.JsonInput.Struct.JsonOutput
-Required.Proto3.JsonInput.Struct.ProtobufOutput
-Required.Proto3.JsonInput.TimestampMaxValue.JsonOutput
-Required.Proto3.JsonInput.TimestampMaxValue.ProtobufOutput
+Required.Proto3.JsonInput.TimestampJsonInputTooSmall
 Required.Proto3.JsonInput.TimestampMinValue.JsonOutput
 Required.Proto3.JsonInput.TimestampMinValue.ProtobufOutput
 Required.Proto3.JsonInput.TimestampRepeatedValue.JsonOutput
 Required.Proto3.JsonInput.TimestampRepeatedValue.ProtobufOutput
-Required.Proto3.JsonInput.TimestampWithNegativeOffset.JsonOutput
-Required.Proto3.JsonInput.TimestampWithNegativeOffset.ProtobufOutput
-Required.Proto3.JsonInput.TimestampWithPositiveOffset.JsonOutput
-Required.Proto3.JsonInput.TimestampWithPositiveOffset.ProtobufOutput
-Required.Proto3.JsonInput.ValueAcceptBool.JsonOutput
-Required.Proto3.JsonInput.ValueAcceptBool.ProtobufOutput
-Required.Proto3.JsonInput.ValueAcceptFloat.JsonOutput
-Required.Proto3.JsonInput.ValueAcceptFloat.ProtobufOutput
-Required.Proto3.JsonInput.ValueAcceptInteger.JsonOutput
-Required.Proto3.JsonInput.ValueAcceptInteger.ProtobufOutput
-Required.Proto3.JsonInput.ValueAcceptList.JsonOutput
-Required.Proto3.JsonInput.ValueAcceptList.ProtobufOutput
-Required.Proto3.JsonInput.ValueAcceptNull.JsonOutput
-Required.Proto3.JsonInput.ValueAcceptNull.ProtobufOutput
-Required.Proto3.JsonInput.ValueAcceptObject.JsonOutput
-Required.Proto3.JsonInput.ValueAcceptObject.ProtobufOutput
-Required.Proto3.JsonInput.ValueAcceptString.JsonOutput
-Required.Proto3.JsonInput.ValueAcceptString.ProtobufOutput
 Required.Proto3.ProtobufInput.DoubleFieldNormalizeQuietNan.JsonOutput
 Required.Proto3.ProtobufInput.DoubleFieldNormalizeSignalingNan.JsonOutput
 Required.Proto3.ProtobufInput.FloatFieldNormalizeQuietNan.JsonOutput

ruby/ext/google/protobuf_c/encode_decode.c

@@ -891,19 +891,38 @@ VALUE Message_decode(VALUE klass, VALUE data) {
 
 /*
  * call-seq:
- *     MessageClass.decode_json(data) => message
+ *     MessageClass.decode_json(data, options = {}) => message
  *
  * Decodes the given data (as a string containing bytes in protocol buffers wire
  * format) under the interpretration given by this message class's definition
  * and returns a message object with the corresponding field values.
+ *
+ * @param options [Hash] options for the decoder
+ *   ignore_unknown_fields: set true to ignore unknown fields (default is to raise an error)
  */
-VALUE Message_decode_json(VALUE klass, VALUE data) {
+VALUE Message_decode_json(int argc, VALUE* argv, VALUE klass) {
   VALUE descriptor = rb_ivar_get(klass, descriptor_instancevar_interned);
   Descriptor* desc = ruby_to_Descriptor(descriptor);
   VALUE msgklass = Descriptor_msgclass(descriptor);
   VALUE msg_rb;
+  VALUE data = argv[0];
+  VALUE ignore_unknown_fields = Qfalse;
   MessageHeader* msg;
 
+  if (argc < 1 || argc > 2) {
+    rb_raise(rb_eArgError, "Expected 1 or 2 arguments.");
+  }
+
+  if (argc == 2) {
+    VALUE hash_args = argv[1];
+    if (TYPE(hash_args) != T_HASH) {
+      rb_raise(rb_eArgError, "Expected hash arguments.");
+    }
+
+    ignore_unknown_fields = rb_hash_lookup2(
+        hash_args, ID2SYM(rb_intern("ignore_unknown_fields")), Qfalse);
+  }
+
   if (TYPE(data) != T_STRING) {
     rb_raise(rb_eArgError, "Expected string for JSON data.");
   }
@@ -922,7 +941,7 @@ VALUE Message_decode_json(VALUE klass, VALUE data) {
     stackenv_init(&se, "Error occurred during parsing: %s");
 
     upb_sink_reset(&sink, get_fill_handlers(desc), msg);
-    parser = upb_json_parser_create(&se.env, method, &sink);
+    parser = upb_json_parser_create(&se.env, method, &sink, ignore_unknown_fields);
     upb_bufsrc_putbuf(RSTRING_PTR(data), RSTRING_LEN(data),
                       upb_json_parser_input(parser));
 
@@ -1310,9 +1329,12 @@ VALUE Message_encode(VALUE klass, VALUE msg_rb) {
 
 /*
  * call-seq:
- *     MessageClass.encode_json(msg) => json_string
+ *     MessageClass.encode_json(msg, options = {}) => json_string
  *
  * Encodes the given message object into its serialized JSON representation.
+ * @param options [Hash] options for the decoder
+ *  preserve_proto_fieldnames: set true to use original fieldnames (default is to camelCase)
+ *  emit_defaults: set true to emit 0/false values (default is to omit them)
  */
 VALUE Message_encode_json(int argc, VALUE* argv, VALUE klass) {
   VALUE descriptor = rb_ivar_get(klass, descriptor_instancevar_interned);

ruby/ext/google/protobuf_c/message.c

@@ -628,7 +628,7 @@ VALUE build_class_from_descriptor(Descriptor* desc) {
   rb_define_method(klass, "[]=", Message_index_set, 2);
   rb_define_singleton_method(klass, "decode", Message_decode, 1);
   rb_define_singleton_method(klass, "encode", Message_encode, 1);
-  rb_define_singleton_method(klass, "decode_json", Message_decode_json, 1);
+  rb_define_singleton_method(klass, "decode_json", Message_decode_json, -1);
   rb_define_singleton_method(klass, "encode_json", Message_encode_json, -1);
   rb_define_singleton_method(klass, "descriptor", Message_descriptor, 0);
 

ruby/ext/google/protobuf_c/protobuf.h

@@ -570,7 +570,7 @@ VALUE Message_index_set(VALUE _self, VALUE field_name, VALUE value);
 VALUE Message_descriptor(VALUE klass);
 VALUE Message_decode(VALUE klass, VALUE data);
 VALUE Message_encode(VALUE klass, VALUE msg_rb);
-VALUE Message_decode_json(VALUE klass, VALUE data);
+VALUE Message_decode_json(int argc, VALUE* argv, VALUE klass);
 VALUE Message_encode_json(int argc, VALUE* argv, VALUE klass);
 
 VALUE Google_Protobuf_discard_unknown(VALUE self, VALUE msg_rb);

ruby/lib/google/protobuf.rb

@@ -69,8 +69,8 @@ module Google
       klass.decode(proto)
     end
 
-    def self.decode_json(klass, json)
-      klass.decode_json(json)
+    def self.decode_json(klass, json, options = {})
+      klass.decode_json(json, options)
     end
 
   end

ruby/tests/basic.rb

@@ -254,6 +254,19 @@ module BasicTest
                     "b" => TestMessage2.new(:foo => 2)}
     end
 
+    def test_protobuf_decode_json_ignore_unknown_fields
+      m = TestMessage.decode_json({
+        optional_string: "foo",
+        not_in_message: "some_value"
+      }.to_json, { ignore_unknown_fields: true })
+
+      assert_equal m.optional_string, "foo"
+      e = assert_raise Google::Protobuf::ParseError do
+        TestMessage.decode_json({ not_in_message: "some_value" }.to_json)
+      end
+      assert_match(/No such field: not_in_message/, e.message)
+    end
+
     def test_to_h
       m = TestMessage.new(:optional_bool => true, :optional_double => -10.100001, :optional_string => 'foo', :repeated_string => ['bar1', 'bar2'], :repeated_msg => [TestMessage2.new(:foo => 100)])
       expected_result = {