Fix Any json encoding/decoding for ruby. (#5592)

* Fix Any json encoding/decoding for ruby.

* Revert unnecessary changes

conformance/failure_list_ruby.txt

@@ -5,6 +5,7 @@ Recommended.Proto3.JsonInput.BytesFieldBase64Url.JsonOutput
 Recommended.Proto3.JsonInput.BytesFieldBase64Url.ProtobufOutput
 Recommended.Proto3.JsonInput.DurationHas3FractionalDigits.Validator
 Recommended.Proto3.JsonInput.DurationHas6FractionalDigits.Validator
+Recommended.Proto3.JsonInput.FieldMaskInvalidCharacter
 Recommended.Proto3.JsonInput.Int64FieldBeString.Validator
 Recommended.Proto3.JsonInput.MapFieldValueIsNull
 Recommended.Proto3.JsonInput.RepeatedFieldMessageElementIsNull
@@ -18,26 +19,6 @@ Recommended.Proto3.JsonInput.TimestampHas6FractionalDigits.Validator
 Recommended.Proto3.JsonInput.Uint64FieldBeString.Validator
 Required.DurationProtoInputTooLarge.JsonOutput
 Required.DurationProtoInputTooSmall.JsonOutput
-Required.Proto3.JsonInput.Any.JsonOutput
-Required.Proto3.JsonInput.Any.ProtobufOutput
-Required.Proto3.JsonInput.AnyNested.JsonOutput
-Required.Proto3.JsonInput.AnyNested.ProtobufOutput
-Required.Proto3.JsonInput.AnyUnorderedTypeTag.JsonOutput
-Required.Proto3.JsonInput.AnyUnorderedTypeTag.ProtobufOutput
-Required.Proto3.JsonInput.AnyWithDuration.JsonOutput
-Required.Proto3.JsonInput.AnyWithDuration.ProtobufOutput
-Required.Proto3.JsonInput.AnyWithFieldMask.JsonOutput
-Required.Proto3.JsonInput.AnyWithFieldMask.ProtobufOutput
-Required.Proto3.JsonInput.AnyWithInt32ValueWrapper.JsonOutput
-Required.Proto3.JsonInput.AnyWithInt32ValueWrapper.ProtobufOutput
-Required.Proto3.JsonInput.AnyWithStruct.JsonOutput
-Required.Proto3.JsonInput.AnyWithStruct.ProtobufOutput
-Required.Proto3.JsonInput.AnyWithTimestamp.JsonOutput
-Required.Proto3.JsonInput.AnyWithTimestamp.ProtobufOutput
-Required.Proto3.JsonInput.AnyWithValueForInteger.JsonOutput
-Required.Proto3.JsonInput.AnyWithValueForInteger.ProtobufOutput
-Required.Proto3.JsonInput.AnyWithValueForJsonObject.JsonOutput
-Required.Proto3.JsonInput.AnyWithValueForJsonObject.ProtobufOutput
 Required.Proto3.JsonInput.DoubleFieldMaxNegativeValue.JsonOutput
 Required.Proto3.JsonInput.DoubleFieldMaxNegativeValue.ProtobufOutput
 Required.Proto3.JsonInput.DoubleFieldMinPositiveValue.JsonOutput
@@ -45,8 +26,6 @@ Required.Proto3.JsonInput.DoubleFieldMinPositiveValue.ProtobufOutput
 Required.Proto3.JsonInput.DoubleFieldNan.JsonOutput
 Required.Proto3.JsonInput.DurationMinValue.JsonOutput
 Required.Proto3.JsonInput.DurationRepeatedValue.JsonOutput
-Required.Proto3.JsonInput.FieldMask.JsonOutput
-Required.Proto3.JsonInput.FieldMask.ProtobufOutput
 Required.Proto3.JsonInput.FloatFieldInfinity.JsonOutput
 Required.Proto3.JsonInput.FloatFieldNan.JsonOutput
 Required.Proto3.JsonInput.FloatFieldNegativeInfinity.JsonOutput

ruby/ext/google/protobuf_c/encode_decode.c

@@ -254,18 +254,13 @@ static size_t stringdata_handler(void* closure, const void* hd,
 }
 
 static bool stringdata_end_handler(void* closure, const void* hd) {
-  MessageHeader* msg = closure;
-  const size_t *ofs = hd;
-  VALUE rb_str = DEREF(msg, *ofs, VALUE);
+  VALUE rb_str = closure;
   rb_obj_freeze(rb_str);
   return true;
 }
 
 static bool appendstring_end_handler(void* closure, const void* hd) {
-  VALUE ary = (VALUE)closure;
-  int size = RepeatedField_size(ary);
-  VALUE* last = RepeatedField_index_native(ary, size - 1);
-  VALUE rb_str = *last;
+  VALUE rb_str = closure;
   rb_obj_freeze(rb_str);
   return true;
 }
@@ -481,9 +476,8 @@ static void *oneofbytes_handler(void *closure,
 }
 
 static bool oneofstring_end_handler(void* closure, const void* hd) {
-  MessageHeader* msg = closure;
-  const oneof_handlerdata_t *oneofdata = hd;
-  rb_obj_freeze(DEREF(msg, oneofdata->ofs, VALUE));
+  VALUE rb_str = rb_str_new2("");
+  rb_obj_freeze(rb_str);
   return true;
 }
 
@@ -938,10 +932,12 @@ VALUE Message_decode_json(int argc, VALUE* argv, VALUE klass) {
     stackenv se;
     upb_sink sink;
     upb_json_parser* parser;
+    DescriptorPool* pool = ruby_to_DescriptorPool(generated_pool);
     stackenv_init(&se, "Error occurred during parsing: %s");
 
     upb_sink_reset(&sink, get_fill_handlers(desc), msg);
-    parser = upb_json_parser_create(&se.env, method, &sink, ignore_unknown_fields);
+    parser = upb_json_parser_create(&se.env, method, pool->symtab,
+                                    &sink, ignore_unknown_fields);
     upb_bufsrc_putbuf(RSTRING_PTR(data), RSTRING_LEN(data),
                       upb_json_parser_input(parser));
 
@@ -958,7 +954,8 @@ VALUE Message_decode_json(int argc, VALUE* argv, VALUE klass) {
 /* msgvisitor *****************************************************************/
 
 static void putmsg(VALUE msg, const Descriptor* desc,
-                   upb_sink *sink, int depth, bool emit_defaults);
+                   upb_sink *sink, int depth, bool emit_defaults,
+                   bool is_json, bool open_msg);
 
 static upb_selector_t getsel(const upb_fielddef *f, upb_handlertype_t type) {
   upb_selector_t ret;
@@ -990,7 +987,7 @@ static void putstr(VALUE str, const upb_fielddef *f, upb_sink *sink) {
 }
 
 static void putsubmsg(VALUE submsg, const upb_fielddef *f, upb_sink *sink,
-                      int depth, bool emit_defaults) {
+                      int depth, bool emit_defaults, bool is_json) {
   upb_sink subsink;
   VALUE descriptor;
   Descriptor* subdesc;
@@ -1001,12 +998,12 @@ static void putsubmsg(VALUE submsg, const upb_fielddef *f, upb_sink *sink,
   subdesc = ruby_to_Descriptor(descriptor);
 
   upb_sink_startsubmsg(sink, getsel(f, UPB_HANDLER_STARTSUBMSG), &subsink);
-  putmsg(submsg, subdesc, &subsink, depth + 1, emit_defaults);
+  putmsg(submsg, subdesc, &subsink, depth + 1, emit_defaults, is_json, true);
   upb_sink_endsubmsg(sink, getsel(f, UPB_HANDLER_ENDSUBMSG));
 }
 
 static void putary(VALUE ary, const upb_fielddef *f, upb_sink *sink,
-                   int depth, bool emit_defaults) {
+                   int depth, bool emit_defaults, bool is_json) {
   upb_sink subsink;
   upb_fieldtype_t type = upb_fielddef_type(f);
   upb_selector_t sel = 0;
@@ -1046,7 +1043,8 @@ static void putary(VALUE ary, const upb_fielddef *f, upb_sink *sink,
         putstr(*((VALUE *)memory), f, &subsink);
         break;
       case UPB_TYPE_MESSAGE:
-        putsubmsg(*((VALUE *)memory), f, &subsink, depth, emit_defaults);
+        putsubmsg(*((VALUE *)memory), f, &subsink, depth,
+                  emit_defaults, is_json);
         break;
 
 #undef T
@@ -1061,7 +1059,8 @@ static void put_ruby_value(VALUE value,
                            VALUE type_class,
                            int depth,
                            upb_sink *sink,
-                           bool emit_defaults) {
+                           bool emit_defaults,
+                           bool is_json) {
   upb_selector_t sel = 0;
   if (upb_fielddef_isprimitive(f)) {
     sel = getsel(f, upb_handlers_getprimitivehandlertype(f));
@@ -1101,12 +1100,12 @@ static void put_ruby_value(VALUE value,
       putstr(value, f, sink);
       break;
     case UPB_TYPE_MESSAGE:
-      putsubmsg(value, f, sink, depth, emit_defaults);
+      putsubmsg(value, f, sink, depth, emit_defaults, is_json);
   }
 }
 
 static void putmap(VALUE map, const upb_fielddef *f, upb_sink *sink,
-                   int depth, bool emit_defaults) {
+                   int depth, bool emit_defaults, bool is_json) {
   Map* self;
   upb_sink subsink;
   const upb_fielddef* key_field;
@@ -1134,9 +1133,10 @@ static void putmap(VALUE map, const upb_fielddef *f, upb_sink *sink,
                          &entry_sink);
     upb_sink_startmsg(&entry_sink);
 
-    put_ruby_value(key, key_field, Qnil, depth + 1, &entry_sink, emit_defaults);
+    put_ruby_value(key, key_field, Qnil, depth + 1, &entry_sink,
+                   emit_defaults, is_json);
     put_ruby_value(value, value_field, self->value_type_class, depth + 1,
-                   &entry_sink, emit_defaults);
+                   &entry_sink, emit_defaults, is_json);
 
     upb_sink_endmsg(&entry_sink, &status);
     upb_sink_endsubmsg(&subsink, getsel(f, UPB_HANDLER_ENDSUBMSG));
@@ -1145,13 +1145,108 @@ static void putmap(VALUE map, const upb_fielddef *f, upb_sink *sink,
   upb_sink_endseq(sink, getsel(f, UPB_HANDLER_ENDSEQ));
 }
 
+static const upb_handlers* msgdef_json_serialize_handlers(
+    Descriptor* desc, bool preserve_proto_fieldnames);
+
+static void putjsonany(VALUE msg_rb, const Descriptor* desc,
+                       upb_sink* sink, int depth, bool emit_defaults) {
+  upb_status status;
+  MessageHeader* msg = NULL;
+  const upb_fielddef* type_field = upb_msgdef_itof(desc->msgdef, UPB_ANY_TYPE);
+  const upb_fielddef* value_field = upb_msgdef_itof(desc->msgdef, UPB_ANY_VALUE);
+
+  size_t type_url_offset;
+  VALUE type_url_str_rb;
+  const upb_msgdef *payload_type = NULL;
+
+  TypedData_Get_Struct(msg_rb, MessageHeader, &Message_type, msg);
+
+  upb_sink_startmsg(sink);
+
+  /* Handle type url */
+  type_url_offset = desc->layout->fields[upb_fielddef_index(type_field)].offset;
+  type_url_str_rb = DEREF(Message_data(msg), type_url_offset, VALUE);
+  if (RSTRING_LEN(type_url_str_rb) > 0) {
+    putstr(type_url_str_rb, type_field, sink);
+  }
+
+  {
+    const char* type_url_str = RSTRING_PTR(type_url_str_rb);
+    size_t type_url_len = RSTRING_LEN(type_url_str_rb);
+    DescriptorPool* pool = ruby_to_DescriptorPool(generated_pool);
+
+    if (type_url_len <= 20 ||
+        strncmp(type_url_str, "type.googleapis.com/", 20) != 0) {
+      rb_raise(rb_eRuntimeError, "Invalid type url: %s", type_url_str);
+      return;
+    }
+
+    /* Resolve type url */
+    type_url_str += 20;
+    type_url_len -= 20;
+
+    payload_type = upb_symtab_lookupmsg2(
+        pool->symtab, type_url_str, type_url_len);
+    if (payload_type == NULL) {
+      rb_raise(rb_eRuntimeError, "Unknown type: %s", type_url_str);
+      return;
+    }
+  }
+
+  {
+    uint32_t value_offset;
+    VALUE value_str_rb;
+    const char* value_str;
+    size_t value_len;
+
+    value_offset = desc->layout->fields[upb_fielddef_index(value_field)].offset;
+    value_str_rb = DEREF(Message_data(msg), value_offset, VALUE);
+    value_str = RSTRING_PTR(value_str_rb);
+    value_len = RSTRING_LEN(value_str_rb);
+
+    if (value_len > 0) {
+      VALUE payload_desc_rb = get_def_obj(payload_type);
+      Descriptor* payload_desc = ruby_to_Descriptor(payload_desc_rb);
+      VALUE payload_class = Descriptor_msgclass(payload_desc_rb);
+      upb_sink subsink;
+      bool is_wellknown;
+
+      VALUE payload_msg_rb = Message_decode(payload_class, value_str_rb);
+
+      is_wellknown =
+          upb_msgdef_wellknowntype(payload_desc->msgdef) !=
+              UPB_WELLKNOWN_UNSPECIFIED;
+      if (is_wellknown) {
+        upb_sink_startstr(sink, getsel(value_field, UPB_HANDLER_STARTSTR), 0,
+                          &subsink);
+      }
+
+      subsink.handlers =
+          msgdef_json_serialize_handlers(payload_desc, true);
+      subsink.closure = sink->closure;
+      putmsg(payload_msg_rb, payload_desc, &subsink, depth, emit_defaults, true,
+             is_wellknown);
+    }
+  }
+
+  upb_sink_endmsg(sink, &status);
+}
+
 static void putmsg(VALUE msg_rb, const Descriptor* desc,
-                   upb_sink *sink, int depth, bool emit_defaults) {
+                   upb_sink *sink, int depth, bool emit_defaults,
+                   bool is_json, bool open_msg) {
   MessageHeader* msg;
   upb_msg_field_iter i;
   upb_status status;
 
-  upb_sink_startmsg(sink);
+  if (is_json && upb_msgdef_wellknowntype(desc->msgdef) == UPB_WELLKNOWN_ANY) {
+    putjsonany(msg_rb, desc, sink, depth, emit_defaults);
+    return;
+  }
+
+  if (open_msg) {
+    upb_sink_startmsg(sink);
+  }
 
   // Protect against cycles (possible because users may freely reassign message
   // and repeated fields) by imposing a maximum recursion depth.
@@ -1196,12 +1291,12 @@ static void putmsg(VALUE msg_rb, const Descriptor* desc,
     if (is_map_field(f)) {
       VALUE map = DEREF(msg, offset, VALUE);
       if (map != Qnil || emit_defaults) {
-        putmap(map, f, sink, depth, emit_defaults);
+        putmap(map, f, sink, depth, emit_defaults, is_json);
       }
     } else if (upb_fielddef_isseq(f)) {
       VALUE ary = DEREF(msg, offset, VALUE);
       if (ary != Qnil) {
-        putary(ary, f, sink, depth, emit_defaults);
+        putary(ary, f, sink, depth, emit_defaults, is_json);
       }
     } else if (upb_fielddef_isstring(f)) {
       VALUE str = DEREF(msg, offset, VALUE);
@@ -1217,7 +1312,8 @@ static void putmsg(VALUE msg_rb, const Descriptor* desc,
         putstr(str, f, sink);
       }
     } else if (upb_fielddef_issubmsg(f)) {
-      putsubmsg(DEREF(msg, offset, VALUE), f, sink, depth, emit_defaults);
+      putsubmsg(DEREF(msg, offset, VALUE), f, sink, depth,
+                emit_defaults, is_json);
     } else {
       upb_selector_t sel = getsel(f, upb_handlers_getprimitivehandlertype(f));
 
@@ -1261,7 +1357,9 @@ static void putmsg(VALUE msg_rb, const Descriptor* desc,
     upb_sink_putunknown(sink, unknown->ptr, unknown->len);
   }
 
-  upb_sink_endmsg(sink, &status);
+  if (open_msg) {
+    upb_sink_endmsg(sink, &status);
+  }
 }
 
 static const upb_handlers* msgdef_pb_serialize_handlers(Descriptor* desc) {
@@ -1316,7 +1414,7 @@ VALUE Message_encode(VALUE klass, VALUE msg_rb) {
     stackenv_init(&se, "Error occurred during encoding: %s");
     encoder = upb_pb_encoder_create(&se.env, serialize_handlers, &sink.sink);
 
-    putmsg(msg_rb, desc, upb_pb_encoder_input(encoder), 0, false);
+    putmsg(msg_rb, desc, upb_pb_encoder_input(encoder), 0, false, false, true);
 
     ret = rb_str_new(sink.ptr, sink.len);
 
@@ -1374,7 +1472,8 @@ VALUE Message_encode_json(int argc, VALUE* argv, VALUE klass) {
     stackenv_init(&se, "Error occurred during encoding: %s");
     printer = upb_json_printer_create(&se.env, serialize_handlers, &sink.sink);
 
-    putmsg(msg_rb, desc, upb_json_printer_input(printer), 0, RTEST(emit_defaults));
+    putmsg(msg_rb, desc, upb_json_printer_input(printer), 0,
+           RTEST(emit_defaults), true, true);
 
     ret = rb_enc_str_new(sink.ptr, sink.len, rb_utf8_encoding());
 

ruby/ext/google/protobuf_c/protobuf.h

@@ -196,6 +196,8 @@ VALUE DescriptorPool_build(int argc, VALUE* argv, VALUE _self);
 VALUE DescriptorPool_lookup(VALUE _self, VALUE name);
 VALUE DescriptorPool_generated_pool(VALUE _self);
 
+extern VALUE generated_pool;
+
 void Descriptor_mark(void* _self);
 void Descriptor_free(void* _self);
 VALUE Descriptor_alloc(VALUE klass);

ruby/ext/google/protobuf_c/upb.h

@@ -1,11 +1,5 @@
 // Amalgamated source file
 
-// php.h intentionally defined NDEBUG. We have to define this macro in order to
-// be used together with php.h
-#ifndef NDEBUG
-#define NDEBUG
-#endif
-
 #if UINTPTR_MAX == 0xffffffff
 #define UPB_SIZE(size32, size64) size32
 #else
@@ -2013,6 +2007,8 @@ typedef enum {
  */
 typedef enum {
   UPB_WELLKNOWN_UNSPECIFIED,
+  UPB_WELLKNOWN_ANY,
+  UPB_WELLKNOWN_FIELDMASK,
   UPB_WELLKNOWN_DURATION,
   UPB_WELLKNOWN_TIMESTAMP,
   /* number wrappers */
@@ -2416,6 +2412,10 @@ typedef upb_strtable_iter upb_msg_oneof_iter;
 #define UPB_MAPENTRY_KEY   1
 #define UPB_MAPENTRY_VALUE 2
 
+/* Well-known field tag numbers for Any messages. */
+#define UPB_ANY_TYPE 1
+#define UPB_ANY_VALUE 2
+
 /* Well-known field tag numbers for timestamp messages. */
 #define UPB_DURATION_SECONDS 1
 #define UPB_DURATION_NANOS 2
@@ -3271,6 +3271,8 @@ const upb_def *upb_symtab_resolve(const upb_symtab *s, const char *base,
                                   const char *sym);
 const upb_def *upb_symtab_lookup(const upb_symtab *s, const char *sym);
 const upb_msgdef *upb_symtab_lookupmsg(const upb_symtab *s, const char *sym);
+const upb_msgdef *upb_symtab_lookupmsg2(
+    const upb_symtab *s, const char *sym, size_t len);
 const upb_enumdef *upb_symtab_lookupenum(const upb_symtab *s, const char *sym);
 bool upb_symtab_add(upb_symtab *s, upb_def *const*defs, size_t n,
                     void *ref_donor, upb_status *status);
@@ -6877,7 +6879,7 @@ typedef enum {
 
 extern const upb_msglayout google_protobuf_FileDescriptorSet_msginit;
 UPB_INLINE google_protobuf_FileDescriptorSet *google_protobuf_FileDescriptorSet_new(upb_arena *arena) {
-  return upb_msg_new(&google_protobuf_FileDescriptorSet_msginit, arena);
+  return (google_protobuf_FileDescriptorSet *)upb_msg_new(&google_protobuf_FileDescriptorSet_msginit, arena);
 }
 UPB_INLINE google_protobuf_FileDescriptorSet *google_protobuf_FileDescriptorSet_parsenew(upb_stringview buf, upb_arena *arena) {
   google_protobuf_FileDescriptorSet *ret = google_protobuf_FileDescriptorSet_new(arena);
@@ -6896,7 +6898,7 @@ UPB_INLINE void google_protobuf_FileDescriptorSet_set_file(google_protobuf_FileD
 
 extern const upb_msglayout google_protobuf_FileDescriptorProto_msginit;
 UPB_INLINE google_protobuf_FileDescriptorProto *google_protobuf_FileDescriptorProto_new(upb_arena *arena) {
-  return upb_msg_new(&google_protobuf_FileDescriptorProto_msginit, arena);
+  return (google_protobuf_FileDescriptorProto *)upb_msg_new(&google_protobuf_FileDescriptorProto_msginit, arena);
 }
 UPB_INLINE google_protobuf_FileDescriptorProto *google_protobuf_FileDescriptorProto_parsenew(upb_stringview buf, upb_arena *arena) {
   google_protobuf_FileDescriptorProto *ret = google_protobuf_FileDescriptorProto_new(arena);
@@ -6937,7 +6939,7 @@ UPB_INLINE void google_protobuf_FileDescriptorProto_set_syntax(google_protobuf_F
 
 extern const upb_msglayout google_protobuf_DescriptorProto_msginit;
 UPB_INLINE google_protobuf_DescriptorProto *google_protobuf_DescriptorProto_new(upb_arena *arena) {
-  return upb_msg_new(&google_protobuf_DescriptorProto_msginit, arena);
+  return (google_protobuf_DescriptorProto *)upb_msg_new(&google_protobuf_DescriptorProto_msginit, arena);
 }
 UPB_INLINE google_protobuf_DescriptorProto *google_protobuf_DescriptorProto_parsenew(upb_stringview buf, upb_arena *arena) {
   google_protobuf_DescriptorProto *ret = google_protobuf_DescriptorProto_new(arena);
@@ -6974,7 +6976,7 @@ UPB_INLINE void google_protobuf_DescriptorProto_set_reserved_name(google_protobu
 
 extern const upb_msglayout google_protobuf_DescriptorProto_ExtensionRange_msginit;
 UPB_INLINE google_protobuf_DescriptorProto_ExtensionRange *google_protobuf_DescriptorProto_ExtensionRange_new(upb_arena *arena) {
-  return upb_msg_new(&google_protobuf_DescriptorProto_ExtensionRange_msginit, arena);
+  return (google_protobuf_DescriptorProto_ExtensionRange *)upb_msg_new(&google_protobuf_DescriptorProto_ExtensionRange_msginit, arena);
 }
 UPB_INLINE google_protobuf_DescriptorProto_ExtensionRange *google_protobuf_DescriptorProto_ExtensionRange_parsenew(upb_stringview buf, upb_arena *arena) {
   google_protobuf_DescriptorProto_ExtensionRange *ret = google_protobuf_DescriptorProto_ExtensionRange_new(arena);
@@ -6997,7 +6999,7 @@ UPB_INLINE void google_protobuf_DescriptorProto_ExtensionRange_set_options(googl
 
 extern const upb_msglayout google_protobuf_DescriptorProto_ReservedRange_msginit;
 UPB_INLINE google_protobuf_DescriptorProto_ReservedRange *google_protobuf_DescriptorProto_ReservedRange_new(upb_arena *arena) {
-  return upb_msg_new(&google_protobuf_DescriptorProto_ReservedRange_msginit, arena);
+  return (google_protobuf_DescriptorProto_ReservedRange *)upb_msg_new(&google_protobuf_DescriptorProto_ReservedRange_msginit, arena);
 }
 UPB_INLINE google_protobuf_DescriptorProto_ReservedRange *google_protobuf_DescriptorProto_ReservedRange_parsenew(upb_stringview buf, upb_arena *arena) {
   google_protobuf_DescriptorProto_ReservedRange *ret = google_protobuf_DescriptorProto_ReservedRange_new(arena);
@@ -7018,7 +7020,7 @@ UPB_INLINE void google_protobuf_DescriptorProto_ReservedRange_set_end(google_pro
 
 extern const upb_msglayout google_protobuf_ExtensionRangeOptions_msginit;
 UPB_INLINE google_protobuf_ExtensionRangeOptions *google_protobuf_ExtensionRangeOptions_new(upb_arena *arena) {
-  return upb_msg_new(&google_protobuf_ExtensionRangeOptions_msginit, arena);
+  return (google_protobuf_ExtensionRangeOptions *)upb_msg_new(&google_protobuf_ExtensionRangeOptions_msginit, arena);
 }
 UPB_INLINE google_protobuf_ExtensionRangeOptions *google_protobuf_ExtensionRangeOptions_parsenew(upb_stringview buf, upb_arena *arena) {
   google_protobuf_ExtensionRangeOptions *ret = google_protobuf_ExtensionRangeOptions_new(arena);
@@ -7037,7 +7039,7 @@ UPB_INLINE void google_protobuf_ExtensionRangeOptions_set_uninterpreted_option(g
 
 extern const upb_msglayout google_protobuf_FieldDescriptorProto_msginit;
 UPB_INLINE google_protobuf_FieldDescriptorProto *google_protobuf_FieldDescriptorProto_new(upb_arena *arena) {
-  return upb_msg_new(&google_protobuf_FieldDescriptorProto_msginit, arena);
+  return (google_protobuf_FieldDescriptorProto *)upb_msg_new(&google_protobuf_FieldDescriptorProto_msginit, arena);
 }
 UPB_INLINE google_protobuf_FieldDescriptorProto *google_protobuf_FieldDescriptorProto_parsenew(upb_stringview buf, upb_arena *arena) {
   google_protobuf_FieldDescriptorProto *ret = google_protobuf_FieldDescriptorProto_new(arena);
@@ -7074,7 +7076,7 @@ UPB_INLINE void google_protobuf_FieldDescriptorProto_set_json_name(google_protob
 
 extern const upb_msglayout google_protobuf_OneofDescriptorProto_msginit;
 UPB_INLINE google_protobuf_OneofDescriptorProto *google_protobuf_OneofDescriptorProto_new(upb_arena *arena) {
-  return upb_msg_new(&google_protobuf_OneofDescriptorProto_msginit, arena);
+  return (google_protobuf_OneofDescriptorProto *)upb_msg_new(&google_protobuf_OneofDescriptorProto_msginit, arena);
 }
 UPB_INLINE google_protobuf_OneofDescriptorProto *google_protobuf_OneofDescriptorProto_parsenew(upb_stringview buf, upb_arena *arena) {
   google_protobuf_OneofDescriptorProto *ret = google_protobuf_OneofDescriptorProto_new(arena);
@@ -7095,7 +7097,7 @@ UPB_INLINE void google_protobuf_OneofDescriptorProto_set_options(google_protobuf
 
 extern const upb_msglayout google_protobuf_EnumDescriptorProto_msginit;
 UPB_INLINE google_protobuf_EnumDescriptorProto *google_protobuf_EnumDescriptorProto_new(upb_arena *arena) {
-  return upb_msg_new(&google_protobuf_EnumDescriptorProto_msginit, arena);
+  return (google_protobuf_EnumDescriptorProto *)upb_msg_new(&google_protobuf_EnumDescriptorProto_msginit, arena);
 }
 UPB_INLINE google_protobuf_EnumDescriptorProto *google_protobuf_EnumDescriptorProto_parsenew(upb_stringview buf, upb_arena *arena) {
   google_protobuf_EnumDescriptorProto *ret = google_protobuf_EnumDescriptorProto_new(arena);
@@ -7122,7 +7124,7 @@ UPB_INLINE void google_protobuf_EnumDescriptorProto_set_reserved_name(google_pro
 
 extern const upb_msglayout google_protobuf_EnumDescriptorProto_EnumReservedRange_msginit;
 UPB_INLINE google_protobuf_EnumDescriptorProto_EnumReservedRange *google_protobuf_EnumDescriptorProto_EnumReservedRange_new(upb_arena *arena) {
-  return upb_msg_new(&google_protobuf_EnumDescriptorProto_EnumReservedRange_msginit, arena);
+  return (google_protobuf_EnumDescriptorProto_EnumReservedRange *)upb_msg_new(&google_protobuf_EnumDescriptorProto_EnumReservedRange_msginit, arena);
 }
 UPB_INLINE google_protobuf_EnumDescriptorProto_EnumReservedRange *google_protobuf_EnumDescriptorProto_EnumReservedRange_parsenew(upb_stringview buf, upb_arena *arena) {
   google_protobuf_EnumDescriptorProto_EnumReservedRange *ret = google_protobuf_EnumDescriptorProto_EnumReservedRange_new(arena);
@@ -7143,7 +7145,7 @@ UPB_INLINE void google_protobuf_EnumDescriptorProto_EnumReservedRange_set_end(go
 
 extern const upb_msglayout google_protobuf_EnumValueDescriptorProto_msginit;
 UPB_INLINE google_protobuf_EnumValueDescriptorProto *google_protobuf_EnumValueDescriptorProto_new(upb_arena *arena) {
-  return upb_msg_new(&google_protobuf_EnumValueDescriptorProto_msginit, arena);
+  return (google_protobuf_EnumValueDescriptorProto *)upb_msg_new(&google_protobuf_EnumValueDescriptorProto_msginit, arena);
 }
 UPB_INLINE google_protobuf_EnumValueDescriptorProto *google_protobuf_EnumValueDescriptorProto_parsenew(upb_stringview buf, upb_arena *arena) {
   google_protobuf_EnumValueDescriptorProto *ret = google_protobuf_EnumValueDescriptorProto_new(arena);
@@ -7166,7 +7168,7 @@ UPB_INLINE void google_protobuf_EnumValueDescriptorProto_set_options(google_prot
 
 extern const upb_msglayout google_protobuf_ServiceDescriptorProto_msginit;
 UPB_INLINE google_protobuf_ServiceDescriptorProto *google_protobuf_ServiceDescriptorProto_new(upb_arena *arena) {
-  return upb_msg_new(&google_protobuf_ServiceDescriptorProto_msginit, arena);
+  return (google_protobuf_ServiceDescriptorProto *)upb_msg_new(&google_protobuf_ServiceDescriptorProto_msginit, arena);
 }
 UPB_INLINE google_protobuf_ServiceDescriptorProto *google_protobuf_ServiceDescriptorProto_parsenew(upb_stringview buf, upb_arena *arena) {
   google_protobuf_ServiceDescriptorProto *ret = google_protobuf_ServiceDescriptorProto_new(arena);
@@ -7189,7 +7191,7 @@ UPB_INLINE void google_protobuf_ServiceDescriptorProto_set_options(google_protob
 
 extern const upb_msglayout google_protobuf_MethodDescriptorProto_msginit;
 UPB_INLINE google_protobuf_MethodDescriptorProto *google_protobuf_MethodDescriptorProto_new(upb_arena *arena) {
-  return upb_msg_new(&google_protobuf_MethodDescriptorProto_msginit, arena);
+  return (google_protobuf_MethodDescriptorProto *)upb_msg_new(&google_protobuf_MethodDescriptorProto_msginit, arena);
 }
 UPB_INLINE google_protobuf_MethodDescriptorProto *google_protobuf_MethodDescriptorProto_parsenew(upb_stringview buf, upb_arena *arena) {
   google_protobuf_MethodDescriptorProto *ret = google_protobuf_MethodDescriptorProto_new(arena);
@@ -7218,7 +7220,7 @@ UPB_INLINE void google_protobuf_MethodDescriptorProto_set_server_streaming(googl
 
 extern const upb_msglayout google_protobuf_FileOptions_msginit;
 UPB_INLINE google_protobuf_FileOptions *google_protobuf_FileOptions_new(upb_arena *arena) {
-  return upb_msg_new(&google_protobuf_FileOptions_msginit, arena);
+  return (google_protobuf_FileOptions *)upb_msg_new(&google_protobuf_FileOptions_msginit, arena);
 }
 UPB_INLINE google_protobuf_FileOptions *google_protobuf_FileOptions_parsenew(upb_stringview buf, upb_arena *arena) {
   google_protobuf_FileOptions *ret = google_protobuf_FileOptions_new(arena);
@@ -7273,7 +7275,7 @@ UPB_INLINE void google_protobuf_FileOptions_set_uninterpreted_option(google_prot
 
 extern const upb_msglayout google_protobuf_MessageOptions_msginit;
 UPB_INLINE google_protobuf_MessageOptions *google_protobuf_MessageOptions_new(upb_arena *arena) {
-  return upb_msg_new(&google_protobuf_MessageOptions_msginit, arena);
+  return (google_protobuf_MessageOptions *)upb_msg_new(&google_protobuf_MessageOptions_msginit, arena);
 }
 UPB_INLINE google_protobuf_MessageOptions *google_protobuf_MessageOptions_parsenew(upb_stringview buf, upb_arena *arena) {
   google_protobuf_MessageOptions *ret = google_protobuf_MessageOptions_new(arena);
@@ -7300,7 +7302,7 @@ UPB_INLINE void google_protobuf_MessageOptions_set_uninterpreted_option(google_p
 
 extern const upb_msglayout google_protobuf_FieldOptions_msginit;
 UPB_INLINE google_protobuf_FieldOptions *google_protobuf_FieldOptions_new(upb_arena *arena) {
-  return upb_msg_new(&google_protobuf_FieldOptions_msginit, arena);
+  return (google_protobuf_FieldOptions *)upb_msg_new(&google_protobuf_FieldOptions_msginit, arena);
 }
 UPB_INLINE google_protobuf_FieldOptions *google_protobuf_FieldOptions_parsenew(upb_stringview buf, upb_arena *arena) {
   google_protobuf_FieldOptions *ret = google_protobuf_FieldOptions_new(arena);
@@ -7331,7 +7333,7 @@ UPB_INLINE void google_protobuf_FieldOptions_set_uninterpreted_option(google_pro
 
 extern const upb_msglayout google_protobuf_OneofOptions_msginit;
 UPB_INLINE google_protobuf_OneofOptions *google_protobuf_OneofOptions_new(upb_arena *arena) {
-  return upb_msg_new(&google_protobuf_OneofOptions_msginit, arena);
+  return (google_protobuf_OneofOptions *)upb_msg_new(&google_protobuf_OneofOptions_msginit, arena);
 }
 UPB_INLINE google_protobuf_OneofOptions *google_protobuf_OneofOptions_parsenew(upb_stringview buf, upb_arena *arena) {
   google_protobuf_OneofOptions *ret = google_protobuf_OneofOptions_new(arena);
@@ -7350,7 +7352,7 @@ UPB_INLINE void google_protobuf_OneofOptions_set_uninterpreted_option(google_pro
 
 extern const upb_msglayout google_protobuf_EnumOptions_msginit;
 UPB_INLINE google_protobuf_EnumOptions *google_protobuf_EnumOptions_new(upb_arena *arena) {
-  return upb_msg_new(&google_protobuf_EnumOptions_msginit, arena);
+  return (google_protobuf_EnumOptions *)upb_msg_new(&google_protobuf_EnumOptions_msginit, arena);
 }
 UPB_INLINE google_protobuf_EnumOptions *google_protobuf_EnumOptions_parsenew(upb_stringview buf, upb_arena *arena) {
   google_protobuf_EnumOptions *ret = google_protobuf_EnumOptions_new(arena);
@@ -7373,7 +7375,7 @@ UPB_INLINE void google_protobuf_EnumOptions_set_uninterpreted_option(google_prot
 
 extern const upb_msglayout google_protobuf_EnumValueOptions_msginit;
 UPB_INLINE google_protobuf_EnumValueOptions *google_protobuf_EnumValueOptions_new(upb_arena *arena) {
-  return upb_msg_new(&google_protobuf_EnumValueOptions_msginit, arena);
+  return (google_protobuf_EnumValueOptions *)upb_msg_new(&google_protobuf_EnumValueOptions_msginit, arena);
 }
 UPB_INLINE google_protobuf_EnumValueOptions *google_protobuf_EnumValueOptions_parsenew(upb_stringview buf, upb_arena *arena) {
   google_protobuf_EnumValueOptions *ret = google_protobuf_EnumValueOptions_new(arena);
@@ -7394,7 +7396,7 @@ UPB_INLINE void google_protobuf_EnumValueOptions_set_uninterpreted_option(google
 
 extern const upb_msglayout google_protobuf_ServiceOptions_msginit;
 UPB_INLINE google_protobuf_ServiceOptions *google_protobuf_ServiceOptions_new(upb_arena *arena) {
-  return upb_msg_new(&google_protobuf_ServiceOptions_msginit, arena);
+  return (google_protobuf_ServiceOptions *)upb_msg_new(&google_protobuf_ServiceOptions_msginit, arena);
 }
 UPB_INLINE google_protobuf_ServiceOptions *google_protobuf_ServiceOptions_parsenew(upb_stringview buf, upb_arena *arena) {
   google_protobuf_ServiceOptions *ret = google_protobuf_ServiceOptions_new(arena);
@@ -7415,7 +7417,7 @@ UPB_INLINE void google_protobuf_ServiceOptions_set_uninterpreted_option(google_p
 
 extern const upb_msglayout google_protobuf_MethodOptions_msginit;
 UPB_INLINE google_protobuf_MethodOptions *google_protobuf_MethodOptions_new(upb_arena *arena) {
-  return upb_msg_new(&google_protobuf_MethodOptions_msginit, arena);
+  return (google_protobuf_MethodOptions *)upb_msg_new(&google_protobuf_MethodOptions_msginit, arena);
 }
 UPB_INLINE google_protobuf_MethodOptions *google_protobuf_MethodOptions_parsenew(upb_stringview buf, upb_arena *arena) {
   google_protobuf_MethodOptions *ret = google_protobuf_MethodOptions_new(arena);
@@ -7438,7 +7440,7 @@ UPB_INLINE void google_protobuf_MethodOptions_set_uninterpreted_option(google_pr
 
 extern const upb_msglayout google_protobuf_UninterpretedOption_msginit;
 UPB_INLINE google_protobuf_UninterpretedOption *google_protobuf_UninterpretedOption_new(upb_arena *arena) {
-  return upb_msg_new(&google_protobuf_UninterpretedOption_msginit, arena);
+  return (google_protobuf_UninterpretedOption *)upb_msg_new(&google_protobuf_UninterpretedOption_msginit, arena);
 }
 UPB_INLINE google_protobuf_UninterpretedOption *google_protobuf_UninterpretedOption_parsenew(upb_stringview buf, upb_arena *arena) {
   google_protobuf_UninterpretedOption *ret = google_protobuf_UninterpretedOption_new(arena);
@@ -7469,7 +7471,7 @@ UPB_INLINE void google_protobuf_UninterpretedOption_set_aggregate_value(google_p
 
 extern const upb_msglayout google_protobuf_UninterpretedOption_NamePart_msginit;
 UPB_INLINE google_protobuf_UninterpretedOption_NamePart *google_protobuf_UninterpretedOption_NamePart_new(upb_arena *arena) {
-  return upb_msg_new(&google_protobuf_UninterpretedOption_NamePart_msginit, arena);
+  return (google_protobuf_UninterpretedOption_NamePart *)upb_msg_new(&google_protobuf_UninterpretedOption_NamePart_msginit, arena);
 }
 UPB_INLINE google_protobuf_UninterpretedOption_NamePart *google_protobuf_UninterpretedOption_NamePart_parsenew(upb_stringview buf, upb_arena *arena) {
   google_protobuf_UninterpretedOption_NamePart *ret = google_protobuf_UninterpretedOption_NamePart_new(arena);
@@ -7490,7 +7492,7 @@ UPB_INLINE void google_protobuf_UninterpretedOption_NamePart_set_is_extension(go
 
 extern const upb_msglayout google_protobuf_SourceCodeInfo_msginit;
 UPB_INLINE google_protobuf_SourceCodeInfo *google_protobuf_SourceCodeInfo_new(upb_arena *arena) {
-  return upb_msg_new(&google_protobuf_SourceCodeInfo_msginit, arena);
+  return (google_protobuf_SourceCodeInfo *)upb_msg_new(&google_protobuf_SourceCodeInfo_msginit, arena);
 }
 UPB_INLINE google_protobuf_SourceCodeInfo *google_protobuf_SourceCodeInfo_parsenew(upb_stringview buf, upb_arena *arena) {
   google_protobuf_SourceCodeInfo *ret = google_protobuf_SourceCodeInfo_new(arena);
@@ -7509,7 +7511,7 @@ UPB_INLINE void google_protobuf_SourceCodeInfo_set_location(google_protobuf_Sour
 
 extern const upb_msglayout google_protobuf_SourceCodeInfo_Location_msginit;
 UPB_INLINE google_protobuf_SourceCodeInfo_Location *google_protobuf_SourceCodeInfo_Location_new(upb_arena *arena) {
-  return upb_msg_new(&google_protobuf_SourceCodeInfo_Location_msginit, arena);
+  return (google_protobuf_SourceCodeInfo_Location *)upb_msg_new(&google_protobuf_SourceCodeInfo_Location_msginit, arena);
 }
 UPB_INLINE google_protobuf_SourceCodeInfo_Location *google_protobuf_SourceCodeInfo_Location_parsenew(upb_stringview buf, upb_arena *arena) {
   google_protobuf_SourceCodeInfo_Location *ret = google_protobuf_SourceCodeInfo_Location_new(arena);
@@ -7536,7 +7538,7 @@ UPB_INLINE void google_protobuf_SourceCodeInfo_Location_set_leading_detached_com
 
 extern const upb_msglayout google_protobuf_GeneratedCodeInfo_msginit;
 UPB_INLINE google_protobuf_GeneratedCodeInfo *google_protobuf_GeneratedCodeInfo_new(upb_arena *arena) {
-  return upb_msg_new(&google_protobuf_GeneratedCodeInfo_msginit, arena);
+  return (google_protobuf_GeneratedCodeInfo *)upb_msg_new(&google_protobuf_GeneratedCodeInfo_msginit, arena);
 }
 UPB_INLINE google_protobuf_GeneratedCodeInfo *google_protobuf_GeneratedCodeInfo_parsenew(upb_stringview buf, upb_arena *arena) {
   google_protobuf_GeneratedCodeInfo *ret = google_protobuf_GeneratedCodeInfo_new(arena);
@@ -7555,7 +7557,7 @@ UPB_INLINE void google_protobuf_GeneratedCodeInfo_set_annotation(google_protobuf
 
 extern const upb_msglayout google_protobuf_GeneratedCodeInfo_Annotation_msginit;
 UPB_INLINE google_protobuf_GeneratedCodeInfo_Annotation *google_protobuf_GeneratedCodeInfo_Annotation_new(upb_arena *arena) {
-  return upb_msg_new(&google_protobuf_GeneratedCodeInfo_Annotation_msginit, arena);
+  return (google_protobuf_GeneratedCodeInfo_Annotation *)upb_msg_new(&google_protobuf_GeneratedCodeInfo_Annotation_msginit, arena);
 }
 UPB_INLINE google_protobuf_GeneratedCodeInfo_Annotation *google_protobuf_GeneratedCodeInfo_Annotation_parsenew(upb_stringview buf, upb_arena *arena) {
   google_protobuf_GeneratedCodeInfo_Annotation *ret = google_protobuf_GeneratedCodeInfo_Annotation_new(arena);
@@ -9567,7 +9569,7 @@ UPB_DECLARE_DERIVED_TYPE(upb::json::ParserMethod, upb::RefCounted,
  * constructed.  This hint may be an overestimate for some build configurations.
  * But if the parser library is upgraded without recompiling the application,
  * it may be an underestimate. */
-#define UPB_JSON_PARSER_SIZE 4160
+#define UPB_JSON_PARSER_SIZE 5712
 
 #ifdef __cplusplus
 
@@ -9576,6 +9578,7 @@ UPB_DECLARE_DERIVED_TYPE(upb::json::ParserMethod, upb::RefCounted,
 class upb::json::Parser {
  public:
   static Parser* Create(Environment* env, const ParserMethod* method,
+                        const SymbolTable* symtab,
                         Sink* output, bool ignore_json_unknown);
 
   BytesSink* input();
@@ -9610,6 +9613,7 @@ UPB_BEGIN_EXTERN_C
 
 upb_json_parser* upb_json_parser_create(upb_env* e,
                                         const upb_json_parsermethod* m,
+                                        const upb_symtab* symtab,
                                         upb_sink* output,
                                         bool ignore_json_unknown);
 upb_bytessink *upb_json_parser_input(upb_json_parser *p);
@@ -9631,8 +9635,10 @@ UPB_END_EXTERN_C
 namespace upb {
 namespace json {
 inline Parser* Parser::Create(Environment* env, const ParserMethod* method,
+                              const SymbolTable* symtab,
                               Sink* output, bool ignore_json_unknown) {
-  return upb_json_parser_create(env, method, output, ignore_json_unknown);
+  return upb_json_parser_create(
+      env, method, symtab, output, ignore_json_unknown);
 }
 inline BytesSink* Parser::input() {
   return upb_json_parser_input(this);

ruby/tests/encode_decode_test.rb

@@ -22,8 +22,8 @@ class EncodeDecodeTest < Test::Unit::TestCase
 
     # Test discard unknown for singular message field.
     unknown_msg = A::B::C::TestUnknown.new(
-	    :optional_unknown =>
-	    A::B::C::TestUnknown.new(:unknown_field => 1))
+            :optional_unknown =>
+            A::B::C::TestUnknown.new(:unknown_field => 1))
     from = A::B::C::TestUnknown.encode(unknown_msg)
     m = A::B::C::TestMessage.decode(from)
     Google::Protobuf.discard_unknown(m)
@@ -32,8 +32,8 @@ class EncodeDecodeTest < Test::Unit::TestCase
 
     # Test discard unknown for repeated message field.
     unknown_msg = A::B::C::TestUnknown.new(
-	    :repeated_unknown =>
-	    [A::B::C::TestUnknown.new(:unknown_field => 1)])
+            :repeated_unknown =>
+            [A::B::C::TestUnknown.new(:unknown_field => 1)])
     from = A::B::C::TestUnknown.encode(unknown_msg)
     m = A::B::C::TestMessage.decode(from)
     Google::Protobuf.discard_unknown(m)
@@ -42,8 +42,8 @@ class EncodeDecodeTest < Test::Unit::TestCase
 
     # Test discard unknown for map value message field.
     unknown_msg = A::B::C::TestUnknown.new(
-	    :map_unknown =>
-	    {"" => A::B::C::TestUnknown.new(:unknown_field => 1)})
+            :map_unknown =>
+            {"" => A::B::C::TestUnknown.new(:unknown_field => 1)})
     from = A::B::C::TestUnknown.encode(unknown_msg)
     m = A::B::C::TestMessage.decode(from)
     Google::Protobuf.discard_unknown(m)
@@ -52,8 +52,8 @@ class EncodeDecodeTest < Test::Unit::TestCase
 
     # Test discard unknown for oneof message field.
     unknown_msg = A::B::C::TestUnknown.new(
-	    :oneof_unknown =>
-	    A::B::C::TestUnknown.new(:unknown_field => 1))
+            :oneof_unknown =>
+            A::B::C::TestUnknown.new(:unknown_field => 1))
     from = A::B::C::TestUnknown.encode(unknown_msg)
     m = A::B::C::TestMessage.decode(from)
     Google::Protobuf.discard_unknown(m)