Merge branch 'master' into layout_mark

ruby/compatibility_tests/v3.0.0/tests/basic.rb

@@ -67,7 +67,6 @@ module BasicTest
     add_message "BadFieldNames" do
       optional :dup, :int32, 1
       optional :class, :int32, 2
-      optional :"a.b", :int32, 3
     end
 
     add_message "MapMessage" do
@@ -1067,8 +1066,6 @@ module BasicTest
       assert m['class'] == 2
       m['dup'] = 3
       assert m['dup'] == 3
-      m['a.b'] = 4
-      assert m['a.b'] == 4
     end
 
     def test_int_ranges

ruby/ext/google/protobuf_c/encode_decode.c

@@ -117,18 +117,18 @@ static const void* newhandlerdata(upb_handlers* h, uint32_t ofs, int32_t hasbit)
 typedef struct {
   size_t ofs;
   int32_t hasbit;
-  const upb_msgdef *md;
+  VALUE subklass;
 } submsg_handlerdata_t;
 
 // Creates a handlerdata that contains offset and submessage type information.
 static const void *newsubmsghandlerdata(upb_handlers* h,
                                         uint32_t ofs,
                                         int32_t hasbit,
-                                        const upb_fielddef* f) {
+                                        VALUE subklass) {
   submsg_handlerdata_t *hd = ALLOC(submsg_handlerdata_t);
   hd->ofs = ofs;
   hd->hasbit = hasbit;
-  hd->md = upb_fielddef_msgsubdef(f);
+  hd->subklass = subklass;
   upb_handlers_addcleanup(h, hd, xfree);
   return hd;
 }
@@ -137,13 +137,14 @@ typedef struct {
   size_t ofs;              // union data slot
   size_t case_ofs;         // oneof_case field
   uint32_t oneof_case_num; // oneof-case number to place in oneof_case field
-  const upb_msgdef *md;    // msgdef, for oneof submessage handler
+  VALUE subklass;
 } oneof_handlerdata_t;
 
 static const void *newoneofhandlerdata(upb_handlers *h,
                                        uint32_t ofs,
                                        uint32_t case_ofs,
-                                       const upb_fielddef *f) {
+                                       const upb_fielddef *f,
+                                       const Descriptor* desc) {
   oneof_handlerdata_t *hd = ALLOC(oneof_handlerdata_t);
   hd->ofs = ofs;
   hd->case_ofs = case_ofs;
@@ -154,11 +155,7 @@ static const void *newoneofhandlerdata(upb_handlers *h,
   // create a separate ID space. In addition, using the field tag number here
   // lets us easily look up the field in the oneof accessor.
   hd->oneof_case_num = upb_fielddef_number(f);
-  if (upb_fielddef_type(f) == UPB_TYPE_MESSAGE) {
-    hd->md = upb_fielddef_msgsubdef(f);
-  } else {
-    hd->md = NULL;
-  }
+  hd->subklass = field_type_class(desc->layout, f);
   upb_handlers_addcleanup(h, hd, xfree);
   return hd;
 }
@@ -254,13 +251,13 @@ static size_t stringdata_handler(void* closure, const void* hd,
 }
 
 static bool stringdata_end_handler(void* closure, const void* hd) {
-  VALUE rb_str = closure;
+  VALUE rb_str = (VALUE)closure;
   rb_obj_freeze(rb_str);
   return true;
 }
 
 static bool appendstring_end_handler(void* closure, const void* hd) {
-  VALUE rb_str = closure;
+  VALUE rb_str = (VALUE)closure;
   rb_obj_freeze(rb_str);
   return true;
 }
@@ -269,12 +266,9 @@ static bool appendstring_end_handler(void* closure, const void* hd) {
 static void *appendsubmsg_handler(void *closure, const void *hd) {
   VALUE ary = (VALUE)closure;
   const submsg_handlerdata_t *submsgdata = hd;
-  VALUE subdesc =
-      get_def_obj((void*)submsgdata->md);
-  VALUE subklass = Descriptor_msgclass(subdesc);
   MessageHeader* submsg;
 
-  VALUE submsg_rb = rb_class_new_instance(0, NULL, subklass);
+  VALUE submsg_rb = rb_class_new_instance(0, NULL, submsgdata->subklass);
   RepeatedField_push(ary, submsg_rb);
 
   TypedData_Get_Struct(submsg_rb, MessageHeader, &Message_type, submsg);
@@ -285,15 +279,12 @@ static void *appendsubmsg_handler(void *closure, const void *hd) {
 static void *submsg_handler(void *closure, const void *hd) {
   MessageHeader* msg = closure;
   const submsg_handlerdata_t* submsgdata = hd;
-  VALUE subdesc =
-      get_def_obj((void*)submsgdata->md);
-  VALUE subklass = Descriptor_msgclass(subdesc);
   VALUE submsg_rb;
   MessageHeader* submsg;
 
   if (DEREF(msg, submsgdata->ofs, VALUE) == Qnil) {
     DEREF(msg, submsgdata->ofs, VALUE) =
-        rb_class_new_instance(0, NULL, subklass);
+        rb_class_new_instance(0, NULL, submsgdata->subklass);
   }
 
   set_hasbit(closure, submsgdata->hasbit);
@@ -309,11 +300,7 @@ typedef struct {
   size_t ofs;
   upb_fieldtype_t key_field_type;
   upb_fieldtype_t value_field_type;
-
-  // We know that we can hold this reference because the handlerdata has the
-  // same lifetime as the upb_handlers struct, and the upb_handlers struct holds
-  // a reference to the upb_msgdef, which in turn has references to its subdefs.
-  const upb_def* value_field_subdef;
+  VALUE subklass;
 } map_handlerdata_t;
 
 // Temporary frame for map parsing: at the beginning of a map entry message, a
@@ -383,19 +370,8 @@ static bool endmap_handler(void *closure, const void *hd, upb_status* s) {
       mapdata->key_field_type, Qnil,
       &frame->key_storage);
 
-  VALUE value_field_typeclass = Qnil;
-  VALUE value;
-
-  if (mapdata->value_field_type == UPB_TYPE_MESSAGE ||
-      mapdata->value_field_type == UPB_TYPE_ENUM) {
-    value_field_typeclass = get_def_obj(mapdata->value_field_subdef);
-    if (mapdata->value_field_type == UPB_TYPE_ENUM) {
-      value_field_typeclass = EnumDescriptor_enummodule(value_field_typeclass);
-    }
-  }
-
-  value = native_slot_get(
-      mapdata->value_field_type, value_field_typeclass,
+  VALUE value = native_slot_get(
+      mapdata->value_field_type, mapdata->subklass,
       &frame->value_storage);
 
   Map_index_set(frame->map, key, value);
@@ -414,7 +390,7 @@ static bool endmap_handler(void *closure, const void *hd, upb_status* s) {
 static map_handlerdata_t* new_map_handlerdata(
     size_t ofs,
     const upb_msgdef* mapentry_def,
-    Descriptor* desc) {
+    const Descriptor* desc) {
   const upb_fielddef* key_field;
   const upb_fielddef* value_field;
   map_handlerdata_t* hd = ALLOC(map_handlerdata_t);
@@ -425,7 +401,7 @@ static map_handlerdata_t* new_map_handlerdata(
   value_field = upb_msgdef_itof(mapentry_def, MAP_VALUE_FIELD);
   assert(value_field != NULL);
   hd->value_field_type = upb_fielddef_type(value_field);
-  hd->value_field_subdef = upb_fielddef_subdef(value_field);
+  hd->subklass = field_type_class(desc->layout, value_field);
 
   return hd;
 }
@@ -491,16 +467,13 @@ static void *oneofsubmsg_handler(void *closure,
   const oneof_handlerdata_t *oneofdata = hd;
   uint32_t oldcase = DEREF(msg, oneofdata->case_ofs, uint32_t);
 
-  VALUE subdesc =
-      get_def_obj((void*)oneofdata->md);
-  VALUE subklass = Descriptor_msgclass(subdesc);
   VALUE submsg_rb;
   MessageHeader* submsg;
 
   if (oldcase != oneofdata->oneof_case_num ||
       DEREF(msg, oneofdata->ofs, VALUE) == Qnil) {
     DEREF(msg, oneofdata->ofs, VALUE) =
-        rb_class_new_instance(0, NULL, subklass);
+        rb_class_new_instance(0, NULL, oneofdata->subklass);
   }
   // Set the oneof case *after* allocating the new class instance -- otherwise,
   // if the Ruby GC is invoked as part of a call into the VM, it might invoke
@@ -518,12 +491,12 @@ static void *oneofsubmsg_handler(void *closure,
 
 // Set up handlers for a repeated field.
 static void add_handlers_for_repeated_field(upb_handlers *h,
+                                            const Descriptor* desc,
                                             const upb_fielddef *f,
                                             size_t offset) {
-  upb_handlerattr attr = UPB_HANDLERATTR_INITIALIZER;
-  upb_handlerattr_sethandlerdata(&attr, newhandlerdata(h, offset, -1));
+  upb_handlerattr attr = UPB_HANDLERATTR_INIT;
+  attr.handler_data = newhandlerdata(h, offset, -1);
   upb_handlers_setstartseq(h, f, startseq_handler, &attr);
-  upb_handlerattr_uninit(&attr);
 
   switch (upb_fielddef_type(f)) {
 
@@ -554,20 +527,20 @@ static void add_handlers_for_repeated_field(upb_handlers *h,
       break;
     }
     case UPB_TYPE_MESSAGE: {
-      upb_handlerattr attr = UPB_HANDLERATTR_INITIALIZER;
-      upb_handlerattr_sethandlerdata(&attr, newsubmsghandlerdata(h, 0, -1, f));
+      VALUE subklass = field_type_class(desc->layout, f);
+      upb_handlerattr attr = UPB_HANDLERATTR_INIT;
+      attr.handler_data = newsubmsghandlerdata(h, 0, -1, subklass);
       upb_handlers_setstartsubmsg(h, f, appendsubmsg_handler, &attr);
-      upb_handlerattr_uninit(&attr);
       break;
     }
   }
 }
 
 // Set up handlers for a singular field.
-static void add_handlers_for_singular_field(upb_handlers *h,
-                                            const upb_fielddef *f,
-                                            size_t offset,
-                                            size_t hasbit_off) {
+static void add_handlers_for_singular_field(const Descriptor* desc,
+                                            upb_handlers* h,
+                                            const upb_fielddef* f,
+                                            size_t offset, size_t hasbit_off) {
   // The offset we pass to UPB points to the start of the Message,
   // rather than the start of where our data is stored.
   int32_t hasbit = -1;
@@ -589,23 +562,20 @@ static void add_handlers_for_singular_field(upb_handlers *h,
     case UPB_TYPE_STRING:
     case UPB_TYPE_BYTES: {
       bool is_bytes = upb_fielddef_type(f) == UPB_TYPE_BYTES;
-      upb_handlerattr attr = UPB_HANDLERATTR_INITIALIZER;
-      upb_handlerattr_sethandlerdata(&attr, newhandlerdata(h, offset, hasbit));
+      upb_handlerattr attr = UPB_HANDLERATTR_INIT;
+      attr.handler_data = newhandlerdata(h, offset, hasbit);
       upb_handlers_setstartstr(h, f,
                                is_bytes ? bytes_handler : str_handler,
                                &attr);
       upb_handlers_setstring(h, f, stringdata_handler, &attr);
       upb_handlers_setendstr(h, f, stringdata_end_handler, &attr);
-      upb_handlerattr_uninit(&attr);
       break;
     }
     case UPB_TYPE_MESSAGE: {
-      upb_handlerattr attr = UPB_HANDLERATTR_INITIALIZER;
-      upb_handlerattr_sethandlerdata(&attr,
-				     newsubmsghandlerdata(h, offset,
-							  hasbit, f));
+      upb_handlerattr attr = UPB_HANDLERATTR_INIT;
+      attr.handler_data = newsubmsghandlerdata(
+          h, offset, hasbit, field_type_class(desc->layout, f));
       upb_handlers_setstartsubmsg(h, f, submsg_handler, &attr);
-      upb_handlerattr_uninit(&attr);
       break;
     }
   }
@@ -615,36 +585,34 @@ static void add_handlers_for_singular_field(upb_handlers *h,
 static void add_handlers_for_mapfield(upb_handlers* h,
                                       const upb_fielddef* fielddef,
                                       size_t offset,
-                                      Descriptor* desc) {
+                                      const Descriptor* desc) {
   const upb_msgdef* map_msgdef = upb_fielddef_msgsubdef(fielddef);
   map_handlerdata_t* hd = new_map_handlerdata(offset, map_msgdef, desc);
-  upb_handlerattr attr = UPB_HANDLERATTR_INITIALIZER;
+  upb_handlerattr attr = UPB_HANDLERATTR_INIT;
 
   upb_handlers_addcleanup(h, hd, xfree);
-  upb_handlerattr_sethandlerdata(&attr, hd);
+  attr.handler_data = hd;
   upb_handlers_setstartsubmsg(h, fielddef, startmapentry_handler, &attr);
-  upb_handlerattr_uninit(&attr);
 }
 
 // Adds handlers to a map-entry msgdef.
-static void add_handlers_for_mapentry(const upb_msgdef* msgdef,
-                                      upb_handlers* h,
-                                      Descriptor* desc) {
+static void add_handlers_for_mapentry(const upb_msgdef* msgdef, upb_handlers* h,
+                                      const Descriptor* desc) {
   const upb_fielddef* key_field = map_entry_key(msgdef);
   const upb_fielddef* value_field = map_entry_value(msgdef);
   map_handlerdata_t* hd = new_map_handlerdata(0, msgdef, desc);
-  upb_handlerattr attr = UPB_HANDLERATTR_INITIALIZER;
+  upb_handlerattr attr = UPB_HANDLERATTR_INIT;
 
   upb_handlers_addcleanup(h, hd, xfree);
-  upb_handlerattr_sethandlerdata(&attr, hd);
+  attr.handler_data = hd;
   upb_handlers_setendmsg(h, endmap_handler, &attr);
 
   add_handlers_for_singular_field(
-      h, key_field,
+      desc, h, key_field,
       offsetof(map_parse_frame_t, key_storage),
       MESSAGE_FIELD_NO_HASBIT);
   add_handlers_for_singular_field(
-      h, value_field,
+      desc, h, value_field,
       offsetof(map_parse_frame_t, value_storage),
       MESSAGE_FIELD_NO_HASBIT);
 }
@@ -653,11 +621,11 @@ static void add_handlers_for_mapentry(const upb_msgdef* msgdef,
 static void add_handlers_for_oneof_field(upb_handlers *h,
                                          const upb_fielddef *f,
                                          size_t offset,
-                                         size_t oneof_case_offset) {
-
-  upb_handlerattr attr = UPB_HANDLERATTR_INITIALIZER;
-  upb_handlerattr_sethandlerdata(
-      &attr, newoneofhandlerdata(h, offset, oneof_case_offset, f));
+                                         size_t oneof_case_offset,
+                                         const Descriptor* desc) {
+  upb_handlerattr attr = UPB_HANDLERATTR_INIT;
+  attr.handler_data =
+      newoneofhandlerdata(h, offset, oneof_case_offset, f, desc);
 
   switch (upb_fielddef_type(f)) {
 
@@ -692,15 +660,13 @@ static void add_handlers_for_oneof_field(upb_handlers *h,
       break;
     }
   }
-
-  upb_handlerattr_uninit(&attr);
 }
 
 static bool unknown_field_handler(void* closure, const void* hd,
                                   const char* buf, size_t size) {
+  MessageHeader* msg = (MessageHeader*)closure;
   UPB_UNUSED(hd);
 
-  MessageHeader* msg = (MessageHeader*)closure;
   if (msg->unknown_fields == NULL) {
     msg->unknown_fields = malloc(sizeof(stringsink));
     stringsink_init(msg->unknown_fields);
@@ -711,27 +677,29 @@ static bool unknown_field_handler(void* closure, const void* hd,
   return true;
 }
 
-static void add_handlers_for_message(const void *closure, upb_handlers *h) {
+void add_handlers_for_message(const void *closure, upb_handlers *h) {
+  const VALUE descriptor_pool = (VALUE)closure;
   const upb_msgdef* msgdef = upb_handlers_msgdef(h);
-  Descriptor* desc = ruby_to_Descriptor(get_def_obj((void*)msgdef));
+  Descriptor* desc =
+      ruby_to_Descriptor(get_msgdef_obj(descriptor_pool, msgdef));
   upb_msg_field_iter i;
-
-  // If this is a mapentry message type, set up a special set of handlers and
-  // bail out of the normal (user-defined) message type handling.
-  if (upb_msgdef_mapentry(msgdef)) {
-    add_handlers_for_mapentry(msgdef, h, desc);
-    return;
-  }
+  upb_handlerattr attr = UPB_HANDLERATTR_INIT;
 
   // Ensure layout exists. We may be invoked to create handlers for a given
   // message if we are included as a submsg of another message type before our
   // class is actually built, so to work around this, we just create the layout
   // (and handlers, in the class-building function) on-demand.
   if (desc->layout == NULL) {
-    desc->layout = create_layout(desc->msgdef);
+    desc->layout = create_layout(desc);
+  }
+
+  // If this is a mapentry message type, set up a special set of handlers and
+  // bail out of the normal (user-defined) message type handling.
+  if (upb_msgdef_mapentry(msgdef)) {
+    add_handlers_for_mapentry(msgdef, h, desc);
+    return;
   }
 
-  upb_handlerattr attr = UPB_HANDLERATTR_INITIALIZER;
   upb_handlers_setunknown(h, unknown_field_handler, &attr);
 
   for (upb_msg_field_begin(&i, desc->msgdef);
@@ -746,64 +714,51 @@ static void add_handlers_for_message(const void *closure, upb_handlers *h) {
       size_t oneof_case_offset =
           desc->layout->oneofs[upb_oneofdef_index(oneof)].case_offset +
           sizeof(MessageHeader);
-      add_handlers_for_oneof_field(h, f, offset, oneof_case_offset);
+      add_handlers_for_oneof_field(h, f, offset, oneof_case_offset, desc);
     } else if (is_map_field(f)) {
       add_handlers_for_mapfield(h, f, offset, desc);
     } else if (upb_fielddef_isseq(f)) {
-      add_handlers_for_repeated_field(h, f, offset);
+      add_handlers_for_repeated_field(h, desc, f, offset);
     } else {
       add_handlers_for_singular_field(
-          h, f, offset, desc->layout->fields[upb_fielddef_index(f)].hasbit);
+          desc, h, f, offset,
+          desc->layout->fields[upb_fielddef_index(f)].hasbit);
     }
   }
 }
 
-// Creates upb handlers for populating a message.
-static const upb_handlers *new_fill_handlers(Descriptor* desc,
-                                             const void* owner) {
-  // TODO(cfallin, haberman): once upb gets a caching/memoization layer for
-  // handlers, reuse subdef handlers so that e.g. if we already parse
-  // B-with-field-of-type-C, we don't have to rebuild the whole hierarchy to
-  // parse A-with-field-of-type-B-with-field-of-type-C.
-  return upb_handlers_newfrozen(desc->msgdef, owner,
-                                add_handlers_for_message, NULL);
-}
-
 // Constructs the handlers for filling a message's data into an in-memory
 // object.
 const upb_handlers* get_fill_handlers(Descriptor* desc) {
-  if (!desc->fill_handlers) {
-    desc->fill_handlers =
-        new_fill_handlers(desc, &desc->fill_handlers);
-  }
-  return desc->fill_handlers;
+  DescriptorPool* pool = ruby_to_DescriptorPool(desc->descriptor_pool);
+  return upb_handlercache_get(pool->fill_handler_cache, desc->msgdef);
 }
 
-// Constructs the upb decoder method for parsing messages of this type.
-// This is called from the message class creation code.
-const upb_pbdecodermethod *new_fillmsg_decodermethod(Descriptor* desc,
-                                                     const void* owner) {
-  const upb_handlers* handlers = get_fill_handlers(desc);
-  upb_pbdecodermethodopts opts;
-  upb_pbdecodermethodopts_init(&opts, handlers);
+static const upb_pbdecodermethod *msgdef_decodermethod(Descriptor* desc) {
+  DescriptorPool* pool = ruby_to_DescriptorPool(desc->descriptor_pool);
+  return upb_pbcodecache_get(pool->fill_method_cache, desc->msgdef);
+}
 
-  return upb_pbdecodermethod_new(&opts, owner);
+static const upb_json_parsermethod *msgdef_jsonparsermethod(Descriptor* desc) {
+  DescriptorPool* pool = ruby_to_DescriptorPool(desc->descriptor_pool);
+  return upb_json_codecache_get(pool->json_fill_method_cache, desc->msgdef);
 }
 
-static const upb_pbdecodermethod *msgdef_decodermethod(Descriptor* desc) {
-  if (desc->fill_method == NULL) {
-    desc->fill_method = new_fillmsg_decodermethod(
-        desc, &desc->fill_method);
-  }
-  return desc->fill_method;
+static const upb_handlers* msgdef_pb_serialize_handlers(Descriptor* desc) {
+  DescriptorPool* pool = ruby_to_DescriptorPool(desc->descriptor_pool);
+  return upb_handlercache_get(pool->pb_serialize_handler_cache, desc->msgdef);
 }
 
-static const upb_json_parsermethod *msgdef_jsonparsermethod(Descriptor* desc) {
-  if (desc->json_fill_method == NULL) {
-    desc->json_fill_method =
-        upb_json_parsermethod_new(desc->msgdef, &desc->json_fill_method);
+static const upb_handlers* msgdef_json_serialize_handlers(
+    Descriptor* desc, bool preserve_proto_fieldnames) {
+  DescriptorPool* pool = ruby_to_DescriptorPool(desc->descriptor_pool);
+  if (preserve_proto_fieldnames) {
+    return upb_handlercache_get(pool->json_serialize_handler_preserve_cache,
+                                desc->msgdef);
+  } else {
+    return upb_handlercache_get(pool->json_serialize_handler_cache,
+                                desc->msgdef);
   }
-  return desc->json_fill_method;
 }
 
 
@@ -813,7 +768,8 @@ static const upb_json_parsermethod *msgdef_jsonparsermethod(Descriptor* desc) {
 // if any error occurs.
 #define STACK_ENV_STACKBYTES 4096
 typedef struct {
-  upb_env env;
+  upb_arena *arena;
+  upb_status status;
   const char* ruby_error_template;
   char allocbuf[STACK_ENV_STACKBYTES];
 } stackenv;
@@ -821,29 +777,22 @@ typedef struct {
 static void stackenv_init(stackenv* se, const char* errmsg);
 static void stackenv_uninit(stackenv* se);
 
-// Callback invoked by upb if any error occurs during parsing or serialization.
-static bool env_error_func(void* ud, const upb_status* status) {
-  stackenv* se = ud;
-  // Free the env -- rb_raise will longjmp up the stack past the encode/decode
-  // function so it would not otherwise have been freed.
-  stackenv_uninit(se);
-
-  // TODO(haberman): have a way to verify that this is actually a parse error,
-  // instead of just throwing "parse error" unconditionally.
-  rb_raise(cParseError, se->ruby_error_template, upb_status_errmsg(status));
-  // Never reached: rb_raise() always longjmp()s up the stack, past all of our
-  // code, back to Ruby.
-  return false;
-}
-
 static void stackenv_init(stackenv* se, const char* errmsg) {
   se->ruby_error_template = errmsg;
-  upb_env_init2(&se->env, se->allocbuf, sizeof(se->allocbuf), NULL);
-  upb_env_seterrorfunc(&se->env, env_error_func, se);
+  se->arena =
+      upb_arena_init(se->allocbuf, sizeof(se->allocbuf), &upb_alloc_global);
+  upb_status_clear(&se->status);
 }
 
 static void stackenv_uninit(stackenv* se) {
-  upb_env_uninit(&se->env);
+  upb_arena_free(se->arena);
+
+  if (!upb_ok(&se->status)) {
+    // TODO(haberman): have a way to verify that this is actually a parse error,
+    // instead of just throwing "parse error" unconditionally.
+    VALUE errmsg = rb_str_new2(upb_status_errmsg(&se->status));
+    rb_raise(cParseError, se->ruby_error_template, errmsg);
+  }
 }
 
 /*
@@ -874,10 +823,10 @@ VALUE Message_decode(VALUE klass, VALUE data) {
     stackenv se;
     upb_sink sink;
     upb_pbdecoder* decoder;
-    stackenv_init(&se, "Error occurred during parsing: %s");
+    stackenv_init(&se, "Error occurred during parsing: %" PRIsVALUE);
 
     upb_sink_reset(&sink, h, msg);
-    decoder = upb_pbdecoder_create(&se.env, method, &sink);
+    decoder = upb_pbdecoder_create(se.arena, method, sink, &se.status);
     upb_bufsrc_putbuf(RSTRING_PTR(data), RSTRING_LEN(data),
                       upb_pbdecoder_input(decoder));
 
@@ -895,8 +844,9 @@ VALUE Message_decode(VALUE klass, VALUE data) {
  * format) under the interpretration given by this message class's definition
  * and returns a message object with the corresponding field values.
  *
- * @param options [Hash] options for the decoder
- *   ignore_unknown_fields: set true to ignore unknown fields (default is to raise an error)
+ *  @param options [Hash] options for the decoder
+ *   ignore_unknown_fields: set true to ignore unknown fields (default is to
+ *   raise an error)
  */
 VALUE Message_decode_json(int argc, VALUE* argv, VALUE klass) {
   VALUE descriptor = rb_ivar_get(klass, descriptor_instancevar_interned);
@@ -924,6 +874,7 @@ VALUE Message_decode_json(int argc, VALUE* argv, VALUE klass) {
   if (TYPE(data) != T_STRING) {
     rb_raise(rb_eArgError, "Expected string for JSON data.");
   }
+
   // TODO(cfallin): Check and respect string encoding. If not UTF-8, we need to
   // convert, because string handlers pass data directly to message string
   // fields.
@@ -937,11 +888,11 @@ VALUE Message_decode_json(int argc, VALUE* argv, VALUE klass) {
     upb_sink sink;
     upb_json_parser* parser;
     DescriptorPool* pool = ruby_to_DescriptorPool(generated_pool);
-    stackenv_init(&se, "Error occurred during parsing: %s");
+    stackenv_init(&se, "Error occurred during parsing: %" PRIsVALUE);
 
     upb_sink_reset(&sink, get_fill_handlers(desc), msg);
-    parser = upb_json_parser_create(&se.env, method, pool->symtab,
-                                    &sink, ignore_unknown_fields);
+    parser = upb_json_parser_create(se.arena, method, pool->symtab, sink,
+                                    &se.status, RTEST(ignore_unknown_fields));
     upb_bufsrc_putbuf(RSTRING_PTR(data), RSTRING_LEN(data),
                       upb_json_parser_input(parser));
 
@@ -957,9 +908,8 @@ VALUE Message_decode_json(int argc, VALUE* argv, VALUE klass) {
 
 /* msgvisitor *****************************************************************/
 
-static void putmsg(VALUE msg, const Descriptor* desc,
-                   upb_sink *sink, int depth, bool emit_defaults,
-                   bool is_json, bool open_msg);
+static void putmsg(VALUE msg, const Descriptor* desc, upb_sink sink, int depth,
+                   bool emit_defaults, bool is_json, bool open_msg);
 
 static upb_selector_t getsel(const upb_fielddef *f, upb_handlertype_t type) {
   upb_selector_t ret;
@@ -968,7 +918,7 @@ static upb_selector_t getsel(const upb_fielddef *f, upb_handlertype_t type) {
   return ret;
 }
 
-static void putstr(VALUE str, const upb_fielddef *f, upb_sink *sink) {
+static void putstr(VALUE str, const upb_fielddef *f, upb_sink sink) {
   upb_sink subsink;
 
   if (str == Qnil) return;
@@ -985,12 +935,12 @@ static void putstr(VALUE str, const upb_fielddef *f, upb_sink *sink) {
 
   upb_sink_startstr(sink, getsel(f, UPB_HANDLER_STARTSTR), RSTRING_LEN(str),
                     &subsink);
-  upb_sink_putstring(&subsink, getsel(f, UPB_HANDLER_STRING), RSTRING_PTR(str),
+  upb_sink_putstring(subsink, getsel(f, UPB_HANDLER_STRING), RSTRING_PTR(str),
                      RSTRING_LEN(str), NULL);
   upb_sink_endstr(sink, getsel(f, UPB_HANDLER_ENDSTR));
 }
 
-static void putsubmsg(VALUE submsg, const upb_fielddef *f, upb_sink *sink,
+static void putsubmsg(VALUE submsg, const upb_fielddef *f, upb_sink sink,
                       int depth, bool emit_defaults, bool is_json) {
   upb_sink subsink;
   VALUE descriptor;
@@ -1002,16 +952,17 @@ static void putsubmsg(VALUE submsg, const upb_fielddef *f, upb_sink *sink,
   subdesc = ruby_to_Descriptor(descriptor);
 
   upb_sink_startsubmsg(sink, getsel(f, UPB_HANDLER_STARTSUBMSG), &subsink);
-  putmsg(submsg, subdesc, &subsink, depth + 1, emit_defaults, is_json, true);
+  putmsg(submsg, subdesc, subsink, depth + 1, emit_defaults, is_json, true);
   upb_sink_endsubmsg(sink, getsel(f, UPB_HANDLER_ENDSUBMSG));
 }
 
-static void putary(VALUE ary, const upb_fielddef *f, upb_sink *sink,
-                   int depth, bool emit_defaults, bool is_json) {
+static void putary(VALUE ary, const upb_fielddef* f, upb_sink sink, int depth,
+                   bool emit_defaults, bool is_json) {
   upb_sink subsink;
   upb_fieldtype_t type = upb_fielddef_type(f);
   upb_selector_t sel = 0;
   int size;
+  int i;
 
   if (ary == Qnil) return;
   if (!emit_defaults && NUM2INT(RepeatedField_length(ary)) == 0) return;
@@ -1025,12 +976,12 @@ static void putary(VALUE ary, const upb_fielddef *f, upb_sink *sink,
     sel = getsel(f, upb_handlers_getprimitivehandlertype(f));
   }
 
-  for (int i = 0; i < size; i++) {
+  for (i = 0; i < size; i++) {
     void* memory = RepeatedField_index_native(ary, i);
     switch (type) {
-#define T(upbtypeconst, upbtype, ctype)                         \
-  case upbtypeconst:                                            \
-    upb_sink_put##upbtype(&subsink, sel, *((ctype *)memory));   \
+#define T(upbtypeconst, upbtype, ctype)                     \
+  case upbtypeconst:                                        \
+    upb_sink_put##upbtype(subsink, sel, *((ctype*)memory)); \
     break;
 
       T(UPB_TYPE_FLOAT,  float,  float)
@@ -1044,11 +995,10 @@ static void putary(VALUE ary, const upb_fielddef *f, upb_sink *sink,
 
       case UPB_TYPE_STRING:
       case UPB_TYPE_BYTES:
-        putstr(*((VALUE *)memory), f, &subsink);
+        putstr(*((VALUE *)memory), f, subsink);
         break;
       case UPB_TYPE_MESSAGE:
-        putsubmsg(*((VALUE *)memory), f, &subsink, depth,
-                  emit_defaults, is_json);
+        putsubmsg(*((VALUE*)memory), f, subsink, depth, emit_defaults, is_json);
         break;
 
 #undef T
@@ -1058,19 +1008,16 @@ static void putary(VALUE ary, const upb_fielddef *f, upb_sink *sink,
   upb_sink_endseq(sink, getsel(f, UPB_HANDLER_ENDSEQ));
 }
 
-static void put_ruby_value(VALUE value,
-                           const upb_fielddef *f,
-                           VALUE type_class,
-                           int depth,
-                           upb_sink *sink,
-                           bool emit_defaults,
+static void put_ruby_value(VALUE value, const upb_fielddef* f, VALUE type_class,
+                           int depth, upb_sink sink, bool emit_defaults,
                            bool is_json) {
+  upb_selector_t sel = 0;
+
   if (depth > ENCODE_MAX_NESTING) {
     rb_raise(rb_eRuntimeError,
              "Maximum recursion depth exceeded during encoding.");
   }
 
-  upb_selector_t sel = 0;
   if (upb_fielddef_isprimitive(f)) {
     sel = getsel(f, upb_handlers_getprimitivehandlertype(f));
   }
@@ -1113,8 +1060,8 @@ static void put_ruby_value(VALUE value,
   }
 }
 
-static void putmap(VALUE map, const upb_fielddef *f, upb_sink *sink,
-                   int depth, bool emit_defaults, bool is_json) {
+static void putmap(VALUE map, const upb_fielddef* f, upb_sink sink, int depth,
+                   bool emit_defaults, bool is_json) {
   Map* self;
   upb_sink subsink;
   const upb_fielddef* key_field;
@@ -1138,17 +1085,17 @@ static void putmap(VALUE map, const upb_fielddef *f, upb_sink *sink,
     upb_status status;
 
     upb_sink entry_sink;
-    upb_sink_startsubmsg(&subsink, getsel(f, UPB_HANDLER_STARTSUBMSG),
+    upb_sink_startsubmsg(subsink, getsel(f, UPB_HANDLER_STARTSUBMSG),
                          &entry_sink);
-    upb_sink_startmsg(&entry_sink);
+    upb_sink_startmsg(entry_sink);
 
-    put_ruby_value(key, key_field, Qnil, depth + 1, &entry_sink,
-                   emit_defaults, is_json);
+    put_ruby_value(key, key_field, Qnil, depth + 1, entry_sink, emit_defaults,
+                   is_json);
     put_ruby_value(value, value_field, self->value_type_class, depth + 1,
-                   &entry_sink, emit_defaults, is_json);
+                   entry_sink, emit_defaults, is_json);
 
-    upb_sink_endmsg(&entry_sink, &status);
-    upb_sink_endsubmsg(&subsink, getsel(f, UPB_HANDLER_ENDSUBMSG));
+    upb_sink_endmsg(entry_sink, &status);
+    upb_sink_endsubmsg(subsink, getsel(f, UPB_HANDLER_ENDSUBMSG));
   }
 
   upb_sink_endseq(sink, getsel(f, UPB_HANDLER_ENDSEQ));
@@ -1157,8 +1104,8 @@ static void putmap(VALUE map, const upb_fielddef *f, upb_sink *sink,
 static const upb_handlers* msgdef_json_serialize_handlers(
     Descriptor* desc, bool preserve_proto_fieldnames);
 
-static void putjsonany(VALUE msg_rb, const Descriptor* desc,
-                       upb_sink* sink, int depth, bool emit_defaults) {
+static void putjsonany(VALUE msg_rb, const Descriptor* desc, upb_sink sink,
+                       int depth, bool emit_defaults) {
   upb_status status;
   MessageHeader* msg = NULL;
   const upb_fielddef* type_field = upb_msgdef_itof(desc->msgdef, UPB_ANY_TYPE);
@@ -1205,16 +1152,14 @@ static void putjsonany(VALUE msg_rb, const Descriptor* desc,
   {
     uint32_t value_offset;
     VALUE value_str_rb;
-    const char* value_str;
     size_t value_len;
 
     value_offset = desc->layout->fields[upb_fielddef_index(value_field)].offset;
     value_str_rb = DEREF(Message_data(msg), value_offset, VALUE);
-    value_str = RSTRING_PTR(value_str_rb);
     value_len = RSTRING_LEN(value_str_rb);
 
     if (value_len > 0) {
-      VALUE payload_desc_rb = get_def_obj(payload_type);
+      VALUE payload_desc_rb = get_msgdef_obj(generated_pool, payload_type);
       Descriptor* payload_desc = ruby_to_Descriptor(payload_desc_rb);
       VALUE payload_class = Descriptor_msgclass(payload_desc_rb);
       upb_sink subsink;
@@ -1232,8 +1177,8 @@ static void putjsonany(VALUE msg_rb, const Descriptor* desc,
 
       subsink.handlers =
           msgdef_json_serialize_handlers(payload_desc, true);
-      subsink.closure = sink->closure;
-      putmsg(payload_msg_rb, payload_desc, &subsink, depth, emit_defaults, true,
+      subsink.closure = sink.closure;
+      putmsg(payload_msg_rb, payload_desc, subsink, depth, emit_defaults, true,
              is_wellknown);
     }
   }
@@ -1243,7 +1188,7 @@ static void putjsonany(VALUE msg_rb, const Descriptor* desc,
 
 static void putjsonlistvalue(
     VALUE msg_rb, const Descriptor* desc,
-    upb_sink* sink, int depth, bool emit_defaults) {
+    upb_sink sink, int depth, bool emit_defaults) {
   upb_status status;
   upb_sink subsink;
   MessageHeader* msg = NULL;
@@ -1270,7 +1215,7 @@ static void putjsonlistvalue(
 }
 
 static void putmsg(VALUE msg_rb, const Descriptor* desc,
-                   upb_sink *sink, int depth, bool emit_defaults,
+                   upb_sink sink, int depth, bool emit_defaults,
                    bool is_json, bool open_msg) {
   MessageHeader* msg;
   upb_msg_field_iter i;
@@ -1360,20 +1305,19 @@ static void putmsg(VALUE msg_rb, const Descriptor* desc,
     } else {
       upb_selector_t sel = getsel(f, upb_handlers_getprimitivehandlertype(f));
 
-#define T(upbtypeconst, upbtype, ctype, default_value)                          \
-  case upbtypeconst: {                                                          \
-      ctype value = DEREF(msg, offset, ctype);                                  \
-      bool is_default = false;                                                  \
-      if (upb_fielddef_haspresence(f)) {                                        \
-        is_default = layout_has(desc->layout, Message_data(msg), f) == Qfalse;  \
-      } else if (upb_msgdef_syntax(desc->msgdef) == UPB_SYNTAX_PROTO3) {        \
-        is_default = default_value == value;                                    \
-      }                                                                         \
-      if (is_matching_oneof || emit_defaults || !is_default) {                  \
-        upb_sink_put##upbtype(sink, sel, value);                                \
-      }                                                                         \
-    }                                                                           \
-    break;
+#define T(upbtypeconst, upbtype, ctype, default_value)                       \
+  case upbtypeconst: {                                                       \
+    ctype value = DEREF(msg, offset, ctype);                                 \
+    bool is_default = false;                                                 \
+    if (upb_fielddef_haspresence(f)) {                                       \
+      is_default = layout_has(desc->layout, Message_data(msg), f) == Qfalse; \
+    } else if (upb_msgdef_syntax(desc->msgdef) == UPB_SYNTAX_PROTO3) {       \
+      is_default = default_value == value;                                   \
+    }                                                                        \
+    if (is_matching_oneof || emit_defaults || !is_default) {                 \
+      upb_sink_put##upbtype(sink, sel, value);                               \
+    }                                                                        \
+  } break;
 
       switch (upb_fielddef_type(f)) {
         T(UPB_TYPE_FLOAT,  float,  float, 0.0)
@@ -1395,9 +1339,11 @@ static void putmsg(VALUE msg_rb, const Descriptor* desc,
     }
   }
 
-  stringsink* unknown = msg->unknown_fields;
-  if (unknown != NULL) {
-    upb_sink_putunknown(sink, unknown->ptr, unknown->len);
+  {
+    stringsink* unknown = msg->unknown_fields;
+    if (unknown != NULL) {
+      upb_sink_putunknown(sink, unknown->ptr, unknown->len);
+    }
   }
 
   if (open_msg) {
@@ -1405,33 +1351,6 @@ static void putmsg(VALUE msg_rb, const Descriptor* desc,
   }
 }
 
-static const upb_handlers* msgdef_pb_serialize_handlers(Descriptor* desc) {
-  if (desc->pb_serialize_handlers == NULL) {
-    desc->pb_serialize_handlers =
-        upb_pb_encoder_newhandlers(desc->msgdef, &desc->pb_serialize_handlers);
-  }
-  return desc->pb_serialize_handlers;
-}
-
-static const upb_handlers* msgdef_json_serialize_handlers(
-    Descriptor* desc, bool preserve_proto_fieldnames) {
-  if (preserve_proto_fieldnames) {
-    if (desc->json_serialize_handlers == NULL) {
-      desc->json_serialize_handlers =
-          upb_json_printer_newhandlers(
-              desc->msgdef, true, &desc->json_serialize_handlers);
-    }
-    return desc->json_serialize_handlers;
-  } else {
-    if (desc->json_serialize_handlers_preserve == NULL) {
-      desc->json_serialize_handlers_preserve =
-          upb_json_printer_newhandlers(
-              desc->msgdef, false, &desc->json_serialize_handlers_preserve);
-    }
-    return desc->json_serialize_handlers_preserve;
-  }
-}
-
 /*
  * call-seq:
  *     MessageClass.encode(msg) => bytes
@@ -1454,8 +1373,8 @@ VALUE Message_encode(VALUE klass, VALUE msg_rb) {
     upb_pb_encoder* encoder;
     VALUE ret;
 
-    stackenv_init(&se, "Error occurred during encoding: %s");
-    encoder = upb_pb_encoder_create(&se.env, serialize_handlers, &sink.sink);
+    stackenv_init(&se, "Error occurred during encoding: %" PRIsVALUE);
+    encoder = upb_pb_encoder_create(se.arena, serialize_handlers, sink.sink);
 
     putmsg(msg_rb, desc, upb_pb_encoder_input(encoder), 0, false, false, true);
 
@@ -1512,8 +1431,8 @@ VALUE Message_encode_json(int argc, VALUE* argv, VALUE klass) {
     stackenv se;
     VALUE ret;
 
-    stackenv_init(&se, "Error occurred during encoding: %s");
-    printer = upb_json_printer_create(&se.env, serialize_handlers, &sink.sink);
+    stackenv_init(&se, "Error occurred during encoding: %" PRIsVALUE);
+    printer = upb_json_printer_create(se.arena, serialize_handlers, sink.sink);
 
     putmsg(msg_rb, desc, upb_json_printer_input(printer), 0,
            RTEST(emit_defaults), true, true);
@@ -1533,10 +1452,12 @@ static void discard_unknown(VALUE msg_rb, const Descriptor* desc) {
 
   TypedData_Get_Struct(msg_rb, MessageHeader, &Message_type, msg);
 
-  stringsink* unknown = msg->unknown_fields;
-  if (unknown != NULL) {
-    stringsink_uninit(unknown);
-    msg->unknown_fields = NULL;
+  {
+    stringsink* unknown = msg->unknown_fields;
+    if (unknown != NULL) {
+      stringsink_uninit(unknown);
+      msg->unknown_fields = NULL;
+    }
   }
 
   for (upb_msg_field_begin(&it, desc->msgdef);
@@ -1565,10 +1486,12 @@ static void discard_unknown(VALUE msg_rb, const Descriptor* desc) {
     }
 
     if (is_map_field(f)) {
+      VALUE map;
+      Map_iter map_it;
+
       if (!upb_fielddef_issubmsg(map_field_value(f))) continue;
-      VALUE map = DEREF(msg, offset, VALUE);
+      map = DEREF(msg, offset, VALUE);
       if (map == Qnil) continue;
-      Map_iter map_it;
       for (Map_begin(map, &map_it); !Map_done(&map_it); Map_next(&map_it)) {
         VALUE submsg = Map_iter_value(&map_it);
         VALUE descriptor = rb_ivar_get(submsg, descriptor_instancevar_interned);
@@ -1577,9 +1500,12 @@ static void discard_unknown(VALUE msg_rb, const Descriptor* desc) {
       }
     } else if (upb_fielddef_isseq(f)) {
       VALUE ary = DEREF(msg, offset, VALUE);
+      int size;
+      int i;
+
       if (ary == Qnil) continue;
-      int size = NUM2INT(RepeatedField_length(ary));
-      for (int i = 0; i < size; i++) {
+      size = NUM2INT(RepeatedField_length(ary));
+      for (i = 0; i < size; i++) {
         void* memory = RepeatedField_index_native(ary, i);
         VALUE submsg = *((VALUE *)memory);
         VALUE descriptor = rb_ivar_get(submsg, descriptor_instancevar_interned);
@@ -1588,9 +1514,12 @@ static void discard_unknown(VALUE msg_rb, const Descriptor* desc) {
       }
     } else {
       VALUE submsg = DEREF(msg, offset, VALUE);
+      VALUE descriptor;
+      const Descriptor* subdesc;
+
       if (submsg == Qnil) continue;
-      VALUE descriptor = rb_ivar_get(submsg, descriptor_instancevar_interned);
-      const Descriptor* subdesc = ruby_to_Descriptor(descriptor);
+      descriptor = rb_ivar_get(submsg, descriptor_instancevar_interned);
+      subdesc = ruby_to_Descriptor(descriptor);
       discard_unknown(submsg, subdesc);
     }
   }

ruby/ext/google/protobuf_c/extconf.rb

@@ -3,11 +3,9 @@
 require 'mkmf'
 
 if RUBY_PLATFORM =~ /darwin/ || RUBY_PLATFORM =~ /linux/
-  # XOPEN_SOURCE needed for strptime:
-  # https://stackoverflow.com/questions/35234152/strptime-giving-implicit-declaration-and-undefined-reference
-  $CFLAGS += " -std=c99 -O3 -DNDEBUG -D_XOPEN_SOURCE=700"
+  $CFLAGS += " -std=gnu90 -O3 -DNDEBUG -Wall -Wdeclaration-after-statement -Wsign-compare"
 else
-  $CFLAGS += " -std=c99 -O3 -DNDEBUG"
+  $CFLAGS += " -std=gnu90 -O3 -DNDEBUG"
 end
 
 

ruby/ext/google/protobuf_c/map.c

@@ -389,10 +389,7 @@ VALUE Map_index(VALUE _self, VALUE key) {
  * was just inserted.
  */
 VALUE Map_index_set(VALUE _self, VALUE key, VALUE value) {
-  rb_check_frozen(_self);
-
   Map* self = ruby_to_Map(_self);
-
   char keybuf[TABLE_KEY_BUF_LENGTH];
   const char* keyval = NULL;
   size_t length = 0;
@@ -400,6 +397,8 @@ VALUE Map_index_set(VALUE _self, VALUE key, VALUE value) {
   void* mem;
   key = table_key(self, key, keybuf, &keyval, &length);
 
+  rb_check_frozen(_self);
+
   if (TYPE(value) == T_HASH) {
     VALUE args[1] = { value };
     value = rb_class_new_instance(1, args, self->value_type_class);
@@ -448,16 +447,15 @@ VALUE Map_has_key(VALUE _self, VALUE key) {
  * nil if none was present. Throws an exception if the key is of the wrong type.
  */
 VALUE Map_delete(VALUE _self, VALUE key) {
-  rb_check_frozen(_self);
-
   Map* self = ruby_to_Map(_self);
-
   char keybuf[TABLE_KEY_BUF_LENGTH];
   const char* keyval = NULL;
   size_t length = 0;
   upb_value v;
   key = table_key(self, key, keybuf, &keyval, &length);
 
+  rb_check_frozen(_self);
+
   if (upb_strtable_remove2(&self->table, keyval, length, &v)) {
     void* mem = value_memory(&v);
     return native_slot_get(self->value_type, self->value_type_class, mem);
@@ -473,10 +471,10 @@ VALUE Map_delete(VALUE _self, VALUE key) {
  * Removes all entries from the map.
  */
 VALUE Map_clear(VALUE _self) {
-  rb_check_frozen(_self);
-
   Map* self = ruby_to_Map(_self);
 
+  rb_check_frozen(_self);
+
   // Uninit and reinit the table -- this is faster than iterating and doing a
   // delete-lookup on each key.
   upb_strtable_uninit(&self->table);

ruby/ext/google/protobuf_c/message.c

@@ -60,10 +60,16 @@ rb_data_type_t Message_type = {
 VALUE Message_alloc(VALUE klass) {
   VALUE descriptor = rb_ivar_get(klass, descriptor_instancevar_interned);
   Descriptor* desc = ruby_to_Descriptor(descriptor);
-  MessageHeader* msg = (MessageHeader*)ALLOC_N(
-      uint8_t, sizeof(MessageHeader) + desc->layout->size);
+  MessageHeader* msg;
   VALUE ret;
 
+  if (desc->layout == NULL) {
+    desc->layout = create_layout(desc);
+  }
+
+  msg = (MessageHeader*)ALLOC_N(uint8_t,
+                                sizeof(MessageHeader) + desc->layout->size);
+
   memset(Message_data(msg), 0, desc->layout->size);
 
   // We wrap first so that everything in the message object is GC-rooted in case
@@ -80,10 +86,7 @@ VALUE Message_alloc(VALUE klass) {
 }
 
 static const upb_fielddef* which_oneof_field(MessageHeader* self, const upb_oneofdef* o) {
-  upb_oneof_iter it;
-  size_t case_ofs;
   uint32_t oneof_case;
-  const upb_fielddef* first_field;
   const upb_fielddef* f;
 
   oneof_case =
@@ -112,8 +115,9 @@ enum {
 };
 
 // Check if the field is a well known wrapper type
-static bool is_wrapper_type_field(const upb_fielddef* field) {
-  char* field_type_name = rb_class2name(field_type_class(field));
+static bool is_wrapper_type_field(const MessageLayout* layout,
+                                  const upb_fielddef* field) {
+  const char* field_type_name = rb_class2name(field_type_class(layout, field));
 
   return strcmp(field_type_name, "Google::Protobuf::DoubleValue") == 0 ||
          strcmp(field_type_name, "Google::Protobuf::FloatValue") == 0 ||
@@ -127,26 +131,34 @@ static bool is_wrapper_type_field(const upb_fielddef* field) {
 }
 
 // Get a new Ruby wrapper type and set the initial value
-static VALUE ruby_wrapper_type(const upb_fielddef* field, const VALUE* value) {
-  if (is_wrapper_type_field(field) && value != Qnil) {
+static VALUE ruby_wrapper_type(const MessageLayout* layout,
+                               const upb_fielddef* field, const VALUE value) {
+  if (is_wrapper_type_field(layout, field) && value != Qnil) {
     VALUE hash = rb_hash_new();
     rb_hash_aset(hash, rb_str_new2("value"), value);
-    VALUE args[1] = { hash };
-    return rb_class_new_instance(1, args, field_type_class(field));
+    {
+      VALUE args[1] = {hash};
+      return rb_class_new_instance(1, args, field_type_class(layout, field));
+    }
   }
   return Qnil;
 }
 
 static int extract_method_call(VALUE method_name, MessageHeader* self,
-			       const upb_fielddef **f, const upb_oneofdef **o) {
-  Check_Type(method_name, T_SYMBOL);
-
-  VALUE method_str = rb_id2str(SYM2ID(method_name));
-  char* name = RSTRING_PTR(method_str);
-  size_t name_len = RSTRING_LEN(method_str);
+                            const upb_fielddef **f, const upb_oneofdef **o) {
+  VALUE method_str;
+  char* name;
+  size_t name_len;
   int accessor_type;
   const upb_oneofdef* test_o;
   const upb_fielddef* test_f;
+  bool has_field;
+
+  Check_Type(method_name, T_SYMBOL);
+
+  method_str = rb_id2str(SYM2ID(method_name));
+  name = RSTRING_PTR(method_str);
+  name_len = RSTRING_LEN(method_str);
 
   if (name[name_len - 1] == '=') {
     accessor_type = METHOD_SETTER;
@@ -155,13 +167,13 @@ static int extract_method_call(VALUE method_name, MessageHeader* self,
     // we don't strip the prefix.
   } else if (strncmp("clear_", name, 6) == 0 &&
              !upb_msgdef_lookupname(self->descriptor->msgdef, name, name_len,
-				    &test_f, &test_o)) {
+                                    &test_f, &test_o)) {
     accessor_type = METHOD_CLEAR;
     name = name + 6;
     name_len = name_len - 6;
   } else if (strncmp("has_", name, 4) == 0 && name[name_len - 1] == '?' &&
              !upb_msgdef_lookupname(self->descriptor->msgdef, name, name_len,
-				    &test_f, &test_o)) {
+                                    &test_f, &test_o)) {
     accessor_type = METHOD_PRESENCE;
     name = name + 4;
     name_len = name_len - 5;
@@ -169,25 +181,26 @@ static int extract_method_call(VALUE method_name, MessageHeader* self,
     accessor_type = METHOD_GETTER;
   }
 
-  bool has_field = upb_msgdef_lookupname(self->descriptor->msgdef, name, name_len,
-			                                   &test_f, &test_o);
+  has_field = upb_msgdef_lookupname(self->descriptor->msgdef, name, name_len,
+                                    &test_f, &test_o);
 
   // Look for wrapper type accessor of the form <field_name>_as_value
   if (!has_field &&
       (accessor_type == METHOD_GETTER || accessor_type == METHOD_SETTER) &&
       name_len > 9 && strncmp(name + name_len - 9, "_as_value", 9) == 0) {
-    // Find the field name
+    const upb_oneofdef* test_o_wrapper;
+    const upb_fielddef* test_f_wrapper;
     char wrapper_field_name[name_len - 8];
+
+    // Find the field name
     strncpy(wrapper_field_name, name, name_len - 9);
     wrapper_field_name[name_len - 9] = '\0';
 
     // Check if field exists and is a wrapper type
-    const upb_oneofdef* test_o_wrapper;
-    const upb_fielddef* test_f_wrapper;
-    if (upb_msgdef_lookupname(self->descriptor->msgdef, wrapper_field_name, name_len - 9,
-			                        &test_f_wrapper, &test_o_wrapper) &&
+    if (upb_msgdef_lookupname(self->descriptor->msgdef, wrapper_field_name,
+                              name_len - 9, &test_f_wrapper, &test_o_wrapper) &&
         upb_fielddef_type(test_f_wrapper) == UPB_TYPE_MESSAGE &&
-        is_wrapper_type_field(test_f_wrapper)) {
+        is_wrapper_type_field(self->descriptor->layout, test_f_wrapper)) {
       // It does exist!
       has_field = true;
       if (accessor_type == METHOD_SETTER) {
@@ -203,17 +216,17 @@ static int extract_method_call(VALUE method_name, MessageHeader* self,
   // Look for enum accessor of the form <enum_name>_const
   if (!has_field && accessor_type == METHOD_GETTER &&
       name_len > 6 && strncmp(name + name_len - 6, "_const", 6) == 0) {
+    const upb_oneofdef* test_o_enum;
+    const upb_fielddef* test_f_enum;
+    char enum_name[name_len - 5];
 
     // Find enum field name
-    char enum_name[name_len - 5];
     strncpy(enum_name, name, name_len - 6);
     enum_name[name_len - 6] = '\0';
 
     // Check if enum field exists
-    const upb_oneofdef* test_o_enum;
-    const upb_fielddef* test_f_enum;
     if (upb_msgdef_lookupname(self->descriptor->msgdef, enum_name, name_len - 6,
-			                        &test_f_enum, &test_o_enum) &&
+                                             &test_f_enum, &test_o_enum) &&
         upb_fielddef_type(test_f_enum) == UPB_TYPE_ENUM) {
       // It does exist!
       has_field = true;
@@ -272,13 +285,14 @@ VALUE Message_method_missing(int argc, VALUE* argv, VALUE _self) {
   MessageHeader* self;
   const upb_oneofdef* o;
   const upb_fielddef* f;
+  int accessor_type;
 
   TypedData_Get_Struct(_self, MessageHeader, &Message_type, self);
   if (argc < 1) {
     rb_raise(rb_eArgError, "Expected method name as first argument.");
   }
 
-  int accessor_type = extract_method_call(argv[0], self, &f, &o);
+  accessor_type = extract_method_call(argv[0], self, &f, &o);
   if (accessor_type == METHOD_UNKNOWN || (o == NULL && f == NULL) ) {
     return rb_call_super(argc, argv);
   } else if (accessor_type == METHOD_SETTER || accessor_type == METHOD_WRAPPER_SETTER) {
@@ -292,11 +306,12 @@ VALUE Message_method_missing(int argc, VALUE* argv, VALUE _self) {
 
   // Return which of the oneof fields are set
   if (o != NULL) {
+    const upb_fielddef* oneof_field = which_oneof_field(self, o);
+
     if (accessor_type == METHOD_SETTER) {
       rb_raise(rb_eRuntimeError, "Oneof accessors are read-only.");
     }
 
-    const upb_fielddef* oneof_field = which_oneof_field(self, o);
     if (accessor_type == METHOD_PRESENCE) {
       return oneof_field == NULL ? Qfalse : Qtrue;
     } else if (accessor_type == METHOD_CLEAR) {
@@ -325,20 +340,21 @@ VALUE Message_method_missing(int argc, VALUE* argv, VALUE _self) {
     }
     return value;
   } else if (accessor_type == METHOD_WRAPPER_SETTER) {
-    VALUE wrapper = ruby_wrapper_type(f, argv[1]);
+    VALUE wrapper = ruby_wrapper_type(self->descriptor->layout, f, argv[1]);
     layout_set(self->descriptor->layout, Message_data(self), f, wrapper);
     return Qnil;
   } else if (accessor_type == METHOD_ENUM_GETTER) {
-    VALUE enum_type = field_type_class(f);
+    VALUE enum_type = field_type_class(self->descriptor->layout, f);
     VALUE method = rb_intern("const_get");
     VALUE raw_value = layout_get(self->descriptor->layout, Message_data(self), f);
 
     // Map repeated fields to a new type with ints
     if (upb_fielddef_label(f) == UPB_LABEL_REPEATED) {
       int array_size = FIX2INT(rb_funcall(raw_value, rb_intern("length"), 0));
+      int i;
       VALUE array_args[1] = { ID2SYM(rb_intern("int64")) };
       VALUE array = rb_class_new_instance(1, array_args, CLASS_OF(raw_value));
-      for (int i = 0; i < array_size; i++) {
+      for (i = 0; i < array_size; i++) {
         VALUE entry = rb_funcall(enum_type, method, 1, rb_funcall(raw_value,
                                  rb_intern("at"), 1, INT2NUM(i)));
         rb_funcall(array, rb_intern("push"), 1, entry);
@@ -357,13 +373,14 @@ VALUE Message_respond_to_missing(int argc, VALUE* argv, VALUE _self) {
   MessageHeader* self;
   const upb_oneofdef* o;
   const upb_fielddef* f;
+  int accessor_type;
 
   TypedData_Get_Struct(_self, MessageHeader, &Message_type, self);
   if (argc < 1) {
     rb_raise(rb_eArgError, "Expected method name as first argument.");
   }
 
-  int accessor_type = extract_method_call(argv[0], self, &f, &o);
+  accessor_type = extract_method_call(argv[0], self, &f, &o);
   if (accessor_type == METHOD_UNKNOWN) {
     return rb_call_super(argc, argv);
   } else if (o != NULL) {
@@ -373,15 +390,10 @@ VALUE Message_respond_to_missing(int argc, VALUE* argv, VALUE _self) {
   }
 }
 
-VALUE create_submsg_from_hash(const upb_fielddef *f, VALUE hash) {
-  const upb_def *d = upb_fielddef_subdef(f);
-  assert(d != NULL);
-
-  VALUE descriptor = get_def_obj(d);
-  VALUE msgclass = rb_funcall(descriptor, rb_intern("msgclass"), 0, NULL);
-
+VALUE create_submsg_from_hash(const MessageLayout* layout,
+                              const upb_fielddef* f, VALUE hash) {
   VALUE args[1] = { hash };
-  return rb_class_new_instance(1, args, msgclass);
+  return rb_class_new_instance(1, args, field_type_class(layout, f));
 }
 
 int Message_initialize_kwarg(VALUE key, VALUE val, VALUE _self) {
@@ -421,6 +433,7 @@ int Message_initialize_kwarg(VALUE key, VALUE val, VALUE _self) {
     Map_merge_into_self(map, val);
   } else if (upb_fielddef_label(f) == UPB_LABEL_REPEATED) {
     VALUE ary;
+    int i;
 
     if (TYPE(val) != T_ARRAY) {
       rb_raise(rb_eArgError,
@@ -428,17 +441,17 @@ int Message_initialize_kwarg(VALUE key, VALUE val, VALUE _self) {
                name, rb_class2name(CLASS_OF(val)));
     }
     ary = layout_get(self->descriptor->layout, Message_data(self), f);
-    for (int i = 0; i < RARRAY_LEN(val); i++) {
+    for (i = 0; i < RARRAY_LEN(val); i++) {
       VALUE entry = rb_ary_entry(val, i);
       if (TYPE(entry) == T_HASH && upb_fielddef_issubmsg(f)) {
-        entry = create_submsg_from_hash(f, entry);
+        entry = create_submsg_from_hash(self->descriptor->layout, f, entry);
       }
 
       RepeatedField_push(ary, entry);
     }
   } else {
     if (TYPE(val) == T_HASH && upb_fielddef_issubmsg(f)) {
-      val = create_submsg_from_hash(f, val);
+      val = create_submsg_from_hash(self->descriptor->layout, f, val);
     }
 
     layout_set(self->descriptor->layout, Message_data(self), f, val);
@@ -595,17 +608,18 @@ VALUE Message_to_h(VALUE _self) {
        !upb_msg_field_done(&it);
        upb_msg_field_next(&it)) {
     const upb_fielddef* field = upb_msg_iter_field(&it);
+    VALUE msg_value;
+    VALUE msg_key;
 
     // For proto2, do not include fields which are not set.
     if (upb_msgdef_syntax(self->descriptor->msgdef) == UPB_SYNTAX_PROTO2 &&
-	field_contains_hasbit(self->descriptor->layout, field) &&
-	!layout_has(self->descriptor->layout, Message_data(self), field)) {
+       field_contains_hasbit(self->descriptor->layout, field) &&
+       !layout_has(self->descriptor->layout, Message_data(self), field)) {
       continue;
     }
 
-    VALUE msg_value = layout_get(self->descriptor->layout, Message_data(self),
-                                 field);
-    VALUE msg_key   = ID2SYM(rb_intern(upb_fielddef_name(field)));
+    msg_value = layout_get(self->descriptor->layout, Message_data(self), field);
+    msg_key = ID2SYM(rb_intern(upb_fielddef_name(field)));
     if (is_map_field(field)) {
       msg_value = Map_to_h(msg_value);
     } else if (upb_fielddef_label(field) == UPB_LABEL_REPEATED) {
@@ -616,7 +630,8 @@ VALUE Message_to_h(VALUE _self) {
       }
 
       if (upb_fielddef_type(field) == UPB_TYPE_MESSAGE) {
-        for (int i = 0; i < RARRAY_LEN(msg_value); i++) {
+        int i;
+        for (i = 0; i < RARRAY_LEN(msg_value); i++) {
           VALUE elem = rb_ary_entry(msg_value, i);
           rb_ary_store(msg_value, i, Message_to_h(elem));
         }
@@ -683,17 +698,11 @@ VALUE Message_descriptor(VALUE klass) {
   return rb_ivar_get(klass, descriptor_instancevar_interned);
 }
 
-VALUE build_class_from_descriptor(Descriptor* desc) {
+VALUE build_class_from_descriptor(VALUE descriptor) {
+  Descriptor* desc = ruby_to_Descriptor(descriptor);
   const char *name;
   VALUE klass;
 
-  if (desc->layout == NULL) {
-    desc->layout = create_layout(desc->msgdef);
-  }
-  if (desc->fill_method == NULL) {
-    desc->fill_method = new_fillmsg_decodermethod(desc, &desc->fill_method);
-  }
-
   name = upb_msgdef_fullname(desc->msgdef);
   if (name == NULL) {
     rb_raise(rb_eRuntimeError, "Descriptor does not have assigned name.");
@@ -704,8 +713,7 @@ VALUE build_class_from_descriptor(Descriptor* desc) {
       // their own toplevel constant class name.
       rb_intern("Message"),
       rb_cObject);
-  rb_ivar_set(klass, descriptor_instancevar_interned,
-              get_def_obj(desc->msgdef));
+  rb_ivar_set(klass, descriptor_instancevar_interned, descriptor);
   rb_define_alloc_func(klass, Message_alloc);
   rb_require("google/protobuf/message_exts");
   rb_include_module(klass, rb_eval_string("::Google::Protobuf::MessageExts"));
@@ -789,7 +797,8 @@ VALUE enum_descriptor(VALUE self) {
   return rb_ivar_get(self, descriptor_instancevar_interned);
 }
 
-VALUE build_module_from_enumdesc(EnumDescriptor* enumdesc) {
+VALUE build_module_from_enumdesc(VALUE _enumdesc) {
+  EnumDescriptor* enumdesc = ruby_to_EnumDescriptor(_enumdesc);
   VALUE mod = rb_define_module_id(
       rb_intern(upb_enumdef_fullname(enumdesc->enumdef)));
 
@@ -810,8 +819,7 @@ VALUE build_module_from_enumdesc(EnumDescriptor* enumdesc) {
   rb_define_singleton_method(mod, "lookup", enum_lookup, 1);
   rb_define_singleton_method(mod, "resolve", enum_resolve, 1);
   rb_define_singleton_method(mod, "descriptor", enum_descriptor, 0);
-  rb_ivar_set(mod, descriptor_instancevar_interned,
-              get_def_obj(enumdesc->enumdef));
+  rb_ivar_set(mod, descriptor_instancevar_interned, _enumdesc);
 
   return mod;
 }

ruby/ext/google/protobuf_c/protobuf.c

@@ -30,26 +30,10 @@
 
 #include "protobuf.h"
 
-// -----------------------------------------------------------------------------
-// Global map from upb {msg,enum}defs to wrapper Descriptor/EnumDescriptor
-// instances.
-// -----------------------------------------------------------------------------
-
-// This is a hash table from def objects (encoded by converting pointers to
-// Ruby integers) to MessageDef/EnumDef instances (as Ruby values).
-VALUE upb_def_to_ruby_obj_map;
-
 VALUE cError;
 VALUE cParseError;
 VALUE cTypeError;
-
-void add_def_obj(const void* def, VALUE value) {
-  rb_hash_aset(upb_def_to_ruby_obj_map, ULL2NUM((intptr_t)def), value);
-}
-
-VALUE get_def_obj(const void* def) {
-  return rb_hash_aref(upb_def_to_ruby_obj_map, ULL2NUM((intptr_t)def));
-}
+VALUE c_only_cookie = Qnil;
 
 static VALUE cached_empty_string = Qnil;
 static VALUE cached_empty_bytes = Qnil;
@@ -142,8 +126,8 @@ void Init_protobuf_c() {
   kRubyStringASCIIEncoding = rb_usascii_encoding();
   kRubyString8bitEncoding = rb_ascii8bit_encoding();
 
-  rb_gc_register_address(&upb_def_to_ruby_obj_map);
-  upb_def_to_ruby_obj_map = rb_hash_new();
+  rb_gc_register_address(&c_only_cookie);
+  c_only_cookie = rb_class_new_instance(0, NULL, rb_cObject);
 
   rb_gc_register_address(&cached_empty_string);
   rb_gc_register_address(&cached_empty_bytes);

ruby/ext/google/protobuf_c/protobuf.h

@@ -108,62 +108,68 @@ typedef struct Builder Builder;
 // -----------------------------------------------------------------------------
 
 struct DescriptorPool {
+  VALUE def_to_descriptor;  // Hash table of def* -> Ruby descriptor.
   upb_symtab* symtab;
+  upb_handlercache* fill_handler_cache;
+  upb_handlercache* pb_serialize_handler_cache;
+  upb_handlercache* json_serialize_handler_cache;
+  upb_handlercache* json_serialize_handler_preserve_cache;
+  upb_pbcodecache* fill_method_cache;
+  upb_json_codecache* json_fill_method_cache;
 };
 
 struct Descriptor {
   const upb_msgdef* msgdef;
   MessageLayout* layout;
-  VALUE klass;  // begins as nil
-  const upb_handlers* fill_handlers;
-  const upb_pbdecodermethod* fill_method;
-  const upb_json_parsermethod* json_fill_method;
-  const upb_handlers* pb_serialize_handlers;
-  const upb_handlers* json_serialize_handlers;
-  const upb_handlers* json_serialize_handlers_preserve;
+  VALUE klass;
+  VALUE descriptor_pool;
 };
 
 struct FileDescriptor {
   const upb_filedef* filedef;
+  VALUE descriptor_pool;  // Owns the upb_filedef.
 };
 
 struct FieldDescriptor {
   const upb_fielddef* fielddef;
+  VALUE descriptor_pool;  // Owns the upb_fielddef.
 };
 
 struct OneofDescriptor {
   const upb_oneofdef* oneofdef;
+  VALUE descriptor_pool;  // Owns the upb_oneofdef.
 };
 
 struct EnumDescriptor {
   const upb_enumdef* enumdef;
   VALUE module;  // begins as nil
+  VALUE descriptor_pool;  // Owns the upb_enumdef.
 };
 
 struct MessageBuilderContext {
-  VALUE descriptor;
-  VALUE builder;
+  google_protobuf_DescriptorProto* msg_proto;
+  VALUE file_builder;
 };
 
 struct OneofBuilderContext {
-  VALUE descriptor;
-  VALUE builder;
+  int oneof_index;
+  VALUE message_builder;
 };
 
 struct EnumBuilderContext {
-  VALUE enumdesc;
+  google_protobuf_EnumDescriptorProto* enum_proto;
+  VALUE file_builder;
 };
 
 struct FileBuilderContext {
-  VALUE pending_list;
-  VALUE file_descriptor;
-  VALUE builder;
+  upb_arena *arena;
+  google_protobuf_FileDescriptorProto* file_proto;
+  VALUE descriptor_pool;
 };
 
 struct Builder {
-  VALUE pending_list;
-  VALUE default_file_descriptor;
-  upb_def** defs;  // used only while finalizing
+  VALUE descriptor_pool;
+  VALUE default_file_builder;
 };
 
 extern VALUE cDescriptorPool;
@@ -192,7 +198,6 @@ void DescriptorPool_free(void* _self);
 VALUE DescriptorPool_alloc(VALUE klass);
 void DescriptorPool_register(VALUE module);
 DescriptorPool* ruby_to_DescriptorPool(VALUE value);
-VALUE DescriptorPool_add(VALUE _self, VALUE def);
 VALUE DescriptorPool_build(int argc, VALUE* argv, VALUE _self);
 VALUE DescriptorPool_lookup(VALUE _self, VALUE name);
 VALUE DescriptorPool_generated_pool(VALUE _self);
@@ -204,13 +209,11 @@ void Descriptor_free(void* _self);
 VALUE Descriptor_alloc(VALUE klass);
 void Descriptor_register(VALUE module);
 Descriptor* ruby_to_Descriptor(VALUE value);
-VALUE Descriptor_initialize(VALUE _self, VALUE file_descriptor_rb);
+VALUE Descriptor_initialize(VALUE _self, VALUE cookie, VALUE descriptor_pool,
+                            VALUE ptr);
 VALUE Descriptor_name(VALUE _self);
-VALUE Descriptor_name_set(VALUE _self, VALUE str);
 VALUE Descriptor_each(VALUE _self);
 VALUE Descriptor_lookup(VALUE _self, VALUE name);
-VALUE Descriptor_add_field(VALUE _self, VALUE obj);
-VALUE Descriptor_add_oneof(VALUE _self, VALUE obj);
 VALUE Descriptor_each_oneof(VALUE _self);
 VALUE Descriptor_lookup_oneof(VALUE _self, VALUE name);
 VALUE Descriptor_msgclass(VALUE _self);
@@ -222,28 +225,24 @@ void FileDescriptor_free(void* _self);
 VALUE FileDescriptor_alloc(VALUE klass);
 void FileDescriptor_register(VALUE module);
 FileDescriptor* ruby_to_FileDescriptor(VALUE value);
-VALUE FileDescriptor_initialize(int argc, VALUE* argv, VALUE _self);
+VALUE FileDescriptor_initialize(VALUE _self, VALUE cookie,
+                                VALUE descriptor_pool, VALUE ptr);
 VALUE FileDescriptor_name(VALUE _self);
 VALUE FileDescriptor_syntax(VALUE _self);
-VALUE FileDescriptor_syntax_set(VALUE _self, VALUE syntax);
 
 void FieldDescriptor_mark(void* _self);
 void FieldDescriptor_free(void* _self);
 VALUE FieldDescriptor_alloc(VALUE klass);
 void FieldDescriptor_register(VALUE module);
 FieldDescriptor* ruby_to_FieldDescriptor(VALUE value);
+VALUE FieldDescriptor_initialize(VALUE _self, VALUE cookie,
+                                 VALUE descriptor_pool, VALUE ptr);
 VALUE FieldDescriptor_name(VALUE _self);
-VALUE FieldDescriptor_name_set(VALUE _self, VALUE str);
 VALUE FieldDescriptor_type(VALUE _self);
-VALUE FieldDescriptor_type_set(VALUE _self, VALUE type);
 VALUE FieldDescriptor_default(VALUE _self);
-VALUE FieldDescriptor_default_set(VALUE _self, VALUE default_value);
 VALUE FieldDescriptor_label(VALUE _self);
-VALUE FieldDescriptor_label_set(VALUE _self, VALUE label);
 VALUE FieldDescriptor_number(VALUE _self);
-VALUE FieldDescriptor_number_set(VALUE _self, VALUE number);
 VALUE FieldDescriptor_submsg_name(VALUE _self);
-VALUE FieldDescriptor_submsg_name_set(VALUE _self, VALUE value);
 VALUE FieldDescriptor_subtype(VALUE _self);
 VALUE FieldDescriptor_has(VALUE _self, VALUE msg_rb);
 VALUE FieldDescriptor_clear(VALUE _self, VALUE msg_rb);
@@ -257,21 +256,20 @@ void OneofDescriptor_free(void* _self);
 VALUE OneofDescriptor_alloc(VALUE klass);
 void OneofDescriptor_register(VALUE module);
 OneofDescriptor* ruby_to_OneofDescriptor(VALUE value);
+VALUE OneofDescriptor_initialize(VALUE _self, VALUE cookie,
+                                 VALUE descriptor_pool, VALUE ptr);
 VALUE OneofDescriptor_name(VALUE _self);
-VALUE OneofDescriptor_name_set(VALUE _self, VALUE value);
-VALUE OneofDescriptor_add_field(VALUE _self, VALUE field);
 VALUE OneofDescriptor_each(VALUE _self, VALUE field);
 
 void EnumDescriptor_mark(void* _self);
 void EnumDescriptor_free(void* _self);
 VALUE EnumDescriptor_alloc(VALUE klass);
+VALUE EnumDescriptor_initialize(VALUE _self, VALUE cookie,
+                                VALUE descriptor_pool, VALUE ptr);
 void EnumDescriptor_register(VALUE module);
 EnumDescriptor* ruby_to_EnumDescriptor(VALUE value);
-VALUE EnumDescriptor_initialize(VALUE _self, VALUE file_descriptor_rb);
 VALUE EnumDescriptor_file_descriptor(VALUE _self);
 VALUE EnumDescriptor_name(VALUE _self);
-VALUE EnumDescriptor_name_set(VALUE _self, VALUE str);
-VALUE EnumDescriptor_add_value(VALUE _self, VALUE name, VALUE number);
 VALUE EnumDescriptor_lookup_name(VALUE _self, VALUE name);
 VALUE EnumDescriptor_lookup_value(VALUE _self, VALUE number);
 VALUE EnumDescriptor_each(VALUE _self);
@@ -284,8 +282,8 @@ VALUE MessageBuilderContext_alloc(VALUE klass);
 void MessageBuilderContext_register(VALUE module);
 MessageBuilderContext* ruby_to_MessageBuilderContext(VALUE value);
 VALUE MessageBuilderContext_initialize(VALUE _self,
-                                       VALUE descriptor,
-                                       VALUE builder);
+                                       VALUE _file_builder,
+                                       VALUE name);
 VALUE MessageBuilderContext_optional(int argc, VALUE* argv, VALUE _self);
 VALUE MessageBuilderContext_required(int argc, VALUE* argv, VALUE _self);
 VALUE MessageBuilderContext_repeated(int argc, VALUE* argv, VALUE _self);
@@ -307,15 +305,20 @@ void EnumBuilderContext_free(void* _self);
 VALUE EnumBuilderContext_alloc(VALUE klass);
 void EnumBuilderContext_register(VALUE module);
 EnumBuilderContext* ruby_to_EnumBuilderContext(VALUE value);
-VALUE EnumBuilderContext_initialize(VALUE _self, VALUE enumdesc);
+VALUE EnumBuilderContext_initialize(VALUE _self, VALUE _file_builder,
+                                    VALUE name);
 VALUE EnumBuilderContext_value(VALUE _self, VALUE name, VALUE number);
 
 void FileBuilderContext_mark(void* _self);
 void FileBuilderContext_free(void* _self);
 VALUE FileBuilderContext_alloc(VALUE klass);
 void FileBuilderContext_register(VALUE module);
-VALUE FileBuilderContext_initialize(VALUE _self, VALUE file_descriptor,
-				    VALUE builder);
+FileBuilderContext* ruby_to_FileBuilderContext(VALUE _self);
+upb_strview FileBuilderContext_strdup(VALUE _self, VALUE rb_str);
+upb_strview FileBuilderContext_strdup_name(VALUE _self, VALUE rb_str);
+upb_strview FileBuilderContext_strdup_sym(VALUE _self, VALUE rb_sym);
+VALUE FileBuilderContext_initialize(VALUE _self, VALUE descriptor_pool,
+                                    VALUE name, VALUE options);
 VALUE FileBuilderContext_add_message(VALUE _self, VALUE name);
 VALUE FileBuilderContext_add_enum(VALUE _self, VALUE name);
 VALUE FileBuilderContext_pending_descriptors(VALUE _self);
@@ -325,7 +328,8 @@ void Builder_free(void* _self);
 VALUE Builder_alloc(VALUE klass);
 void Builder_register(VALUE module);
 Builder* ruby_to_Builder(VALUE value);
-VALUE Builder_initialize(VALUE _self);
+VALUE Builder_build(VALUE _self);
+VALUE Builder_initialize(VALUE _self, VALUE descriptor_pool);
 VALUE Builder_add_file(int argc, VALUE *argv, VALUE _self);
 VALUE Builder_add_message(VALUE _self, VALUE name);
 VALUE Builder_add_enum(VALUE _self, VALUE name);
@@ -371,7 +375,7 @@ extern rb_encoding* kRubyStringUtf8Encoding;
 extern rb_encoding* kRubyStringASCIIEncoding;
 extern rb_encoding* kRubyString8bitEncoding;
 
-VALUE field_type_class(const upb_fielddef* field);
+VALUE field_type_class(const MessageLayout* layout, const upb_fielddef* field);
 
 #define MAP_KEY_FIELD 1
 #define MAP_VALUE_FIELD 2
@@ -506,14 +510,16 @@ struct MessageOneof {
   uint32_t case_offset;
 };
 
+// MessageLayout is owned by the enclosing Descriptor, which must outlive us.
 struct MessageLayout {
+  const Descriptor* desc;
   const upb_msgdef* msgdef;
   MessageField* fields;
   MessageOneof* oneofs;
   size_t size;
 };
 
-MessageLayout* create_layout(const upb_msgdef* msgdef);
+MessageLayout* create_layout(const Descriptor* desc);
 void free_layout(MessageLayout* layout);
 bool field_contains_hasbit(MessageLayout* layout,
                  const upb_fielddef* field);
@@ -562,7 +568,7 @@ struct MessageHeader {
 
 extern rb_data_type_t Message_type;
 
-VALUE build_class_from_descriptor(Descriptor* descriptor);
+VALUE build_class_from_descriptor(VALUE descriptor);
 void* Message_data(void* msg);
 void Message_mark(void* self);
 void Message_free(void* self);
@@ -586,12 +592,14 @@ VALUE Message_encode_json(int argc, VALUE* argv, VALUE klass);
 VALUE Google_Protobuf_discard_unknown(VALUE self, VALUE msg_rb);
 VALUE Google_Protobuf_deep_copy(VALUE self, VALUE obj);
 
-VALUE build_module_from_enumdesc(EnumDescriptor* enumdef);
+VALUE build_module_from_enumdesc(VALUE _enumdesc);
 VALUE enum_lookup(VALUE self, VALUE number);
 VALUE enum_resolve(VALUE self, VALUE sym);
+VALUE enum_descriptor(VALUE self);
 
 const upb_pbdecodermethod *new_fillmsg_decodermethod(
     Descriptor* descriptor, const void *owner);
+void add_handlers_for_message(const void *closure, upb_handlers *h);
 
 // Maximum depth allowed during encoding, to avoid stack overflows due to
 // cycles.
@@ -606,8 +614,11 @@ VALUE get_frozen_string(const char* data, size_t size, bool binary);
 // Global map from upb {msg,enum}defs to wrapper Descriptor/EnumDescriptor
 // instances.
 // -----------------------------------------------------------------------------
-void add_def_obj(const void* def, VALUE value);
-VALUE get_def_obj(const void* def);
+VALUE get_msgdef_obj(VALUE descriptor_pool, const upb_msgdef* def);
+VALUE get_enumdef_obj(VALUE descriptor_pool, const upb_enumdef* def);
+VALUE get_fielddef_obj(VALUE descriptor_pool, const upb_fielddef* def);
+VALUE get_filedef_obj(VALUE descriptor_pool, const upb_filedef* def);
+VALUE get_oneofdef_obj(VALUE descriptor_pool, const upb_oneofdef* def);
 
 // -----------------------------------------------------------------------------
 // Utilities.
@@ -623,4 +634,17 @@ void check_upb_status(const upb_status* status, const char* msg);
 
 extern ID descriptor_instancevar_interned;
 
+// A distinct object that is not accessible from Ruby.  We use this as a
+// constructor argument to enforce that certain objects cannot be created from
+// Ruby.
+extern VALUE c_only_cookie;
+
+#ifdef NDEBUG
+#define UPB_ASSERT(expr) do {} while (false && (expr))
+#else
+#define UPB_ASSERT(expr) assert(expr)
+#endif
+
+#define UPB_UNUSED(var) (void)var
+
 #endif  // __GOOGLE_PROTOBUF_RUBY_PROTOBUF_H__

ruby/ext/google/protobuf_c/repeated_field.c

@@ -64,10 +64,11 @@ VALUE RepeatedField_subarray(VALUE _self, long beg, long len) {
   int element_size = native_slot_size(self->field_type);
   upb_fieldtype_t field_type = self->field_type;
   VALUE field_type_class = self->field_type_class;
-
   size_t off = beg * element_size;
   VALUE ary = rb_ary_new2(len);
-  for (int i = beg; i < beg + len; i++, off += element_size) {
+  int i;
+
+  for (i = beg; i < beg + len; i++, off += element_size) {
     void* mem = ((uint8_t *)self->elements) + off;
     VALUE elem = native_slot_get(field_type, field_type_class, mem);
     rb_ary_push(ary, elem);
@@ -88,9 +89,10 @@ VALUE RepeatedField_each(VALUE _self) {
   upb_fieldtype_t field_type = self->field_type;
   VALUE field_type_class = self->field_type_class;
   int element_size = native_slot_size(field_type);
-
   size_t off = 0;
-  for (int i = 0; i < self->size; i++, off += element_size) {
+  int i;
+
+  for (i = 0; i < self->size; i++, off += element_size) {
     void* memory = (void *) (((uint8_t *)self->elements) + off);
     VALUE val = native_slot_get(field_type, field_type_class, memory);
     rb_yield(val);
@@ -169,8 +171,10 @@ VALUE RepeatedField_index_set(VALUE _self, VALUE _index, VALUE val) {
   if (index >= self->size) {
     upb_fieldtype_t field_type = self->field_type;
     int element_size = native_slot_size(field_type);
+    int i;
+
     RepeatedField_reserve(self, index + 1);
-    for (int i = self->size; i <= index; i++) {
+    for (i = self->size; i <= index; i++) {
       void* elem = RepeatedField_memoryat(self, i, element_size);
       native_slot_init(field_type, elem);
     }
@@ -224,7 +228,8 @@ VALUE RepeatedField_push(VALUE _self, VALUE val) {
 }
 
 VALUE RepeatedField_push_vararg(VALUE _self, VALUE args) {
-  for (int i = 0; i < RARRAY_LEN(args); i++) {
+  int i;
+  for (i = 0; i < RARRAY_LEN(args); i++) {
     RepeatedField_push(_self, rb_ary_entry(args, i));
   }
   return _self;
@@ -285,9 +290,11 @@ VALUE RepeatedField_pop_one(VALUE _self) {
  */
 VALUE RepeatedField_replace(VALUE _self, VALUE list) {
   RepeatedField* self = ruby_to_RepeatedField(_self);
+  int i;
+
   Check_Type(list, T_ARRAY);
   self->size = 0;
-  for (int i = 0; i < RARRAY_LEN(list); i++) {
+  for (i = 0; i < RARRAY_LEN(list); i++) {
     RepeatedField_push(_self, rb_ary_entry(list, i));
   }
   return list;
@@ -344,8 +351,10 @@ VALUE RepeatedField_dup(VALUE _self) {
   upb_fieldtype_t field_type = self->field_type;
   size_t elem_size = native_slot_size(field_type);
   size_t off = 0;
+  int i;
+
   RepeatedField_reserve(new_rptfield_self, self->size);
-  for (int i = 0; i < self->size; i++, off += elem_size) {
+  for (i = 0; i < self->size; i++, off += elem_size) {
     void* to_mem = (uint8_t *)new_rptfield_self->elements + off;
     void* from_mem = (uint8_t *)self->elements + off;
     native_slot_dup(field_type, to_mem, from_mem);
@@ -363,8 +372,10 @@ VALUE RepeatedField_deep_copy(VALUE _self) {
   upb_fieldtype_t field_type = self->field_type;
   size_t elem_size = native_slot_size(field_type);
   size_t off = 0;
+  int i;
+
   RepeatedField_reserve(new_rptfield_self, self->size);
-  for (int i = 0; i < self->size; i++, off += elem_size) {
+  for (i = 0; i < self->size; i++, off += elem_size) {
     void* to_mem = (uint8_t *)new_rptfield_self->elements + off;
     void* from_mem = (uint8_t *)self->elements + off;
     native_slot_deep_copy(field_type, to_mem, from_mem);
@@ -384,11 +395,12 @@ VALUE RepeatedField_deep_copy(VALUE _self) {
 VALUE RepeatedField_to_ary(VALUE _self) {
   RepeatedField* self = ruby_to_RepeatedField(_self);
   upb_fieldtype_t field_type = self->field_type;
-
   size_t elem_size = native_slot_size(field_type);
   size_t off = 0;
   VALUE ary = rb_ary_new2(self->size);
-  for (int i = 0; i < self->size; i++, off += elem_size) {
+  int i;
+
+  for (i = 0; i < self->size; i++, off += elem_size) {
     void* mem = ((uint8_t *)self->elements) + off;
     VALUE elem = native_slot_get(field_type, self->field_type_class, mem);
     rb_ary_push(ary, elem);
@@ -434,7 +446,9 @@ VALUE RepeatedField_eq(VALUE _self, VALUE _other) {
     upb_fieldtype_t field_type = self->field_type;
     size_t elem_size = native_slot_size(field_type);
     size_t off = 0;
-    for (int i = 0; i < self->size; i++, off += elem_size) {
+    int i;
+
+    for (i = 0; i < self->size; i++, off += elem_size) {
       void* self_mem = ((uint8_t *)self->elements) + off;
       void* other_mem = ((uint8_t *)other->elements) + off;
       if (!native_slot_eq(field_type, self_mem, other_mem)) {
@@ -459,7 +473,9 @@ VALUE RepeatedField_hash(VALUE _self) {
   VALUE field_type_class = self->field_type_class;
   size_t elem_size = native_slot_size(field_type);
   size_t off = 0;
-  for (int i = 0; i < self->size; i++, off += elem_size) {
+  int i;
+
+  for (i = 0; i < self->size; i++, off += elem_size) {
     void* mem = ((uint8_t *)self->elements) + off;
     VALUE elem = native_slot_get(field_type, field_type_class, mem);
     h = rb_hash_uint(h, NUM2LONG(rb_funcall(elem, hash_sym, 0)));
@@ -481,7 +497,8 @@ VALUE RepeatedField_plus(VALUE _self, VALUE list) {
   VALUE dupped = RepeatedField_dup(_self);
 
   if (TYPE(list) == T_ARRAY) {
-    for (int i = 0; i < RARRAY_LEN(list); i++) {
+    int i;
+    for (i = 0; i < RARRAY_LEN(list); i++) {
       VALUE elem = rb_ary_entry(list, i);
       RepeatedField_push(dupped, elem);
     }
@@ -489,12 +506,14 @@ VALUE RepeatedField_plus(VALUE _self, VALUE list) {
              RTYPEDDATA_TYPE(list) == &RepeatedField_type) {
     RepeatedField* self = ruby_to_RepeatedField(_self);
     RepeatedField* list_rptfield = ruby_to_RepeatedField(list);
+    int i;
+
     if (self->field_type != list_rptfield->field_type ||
         self->field_type_class != list_rptfield->field_type_class) {
       rb_raise(rb_eArgError,
                "Attempt to append RepeatedField with different element type.");
     }
-    for (int i = 0; i < list_rptfield->size; i++) {
+    for (i = 0; i < list_rptfield->size; i++) {
       void* mem = RepeatedField_index_native(list, i);
       RepeatedField_push_native(dupped, mem);
     }
@@ -512,8 +531,10 @@ VALUE RepeatedField_plus(VALUE _self, VALUE list) {
  * concats the passed in array to self.  Returns a Ruby array.
  */
 VALUE RepeatedField_concat(VALUE _self, VALUE list) {
+  int i;
+
   Check_Type(list, T_ARRAY);
-  for (int i = 0; i < RARRAY_LEN(list); i++) {
+  for (i = 0; i < RARRAY_LEN(list); i++) {
     RepeatedField_push(_self, rb_ary_entry(list, i));
   }
   return _self;
@@ -574,10 +595,12 @@ void RepeatedField_init_args(int argc, VALUE* argv,
   }
 
   if (ary != Qnil) {
+    int i;
+
     if (!RB_TYPE_P(ary, T_ARRAY)) {
       rb_raise(rb_eArgError, "Expected array as initialize argument");
     }
-    for (int i = 0; i < RARRAY_LEN(ary); i++) {
+    for (i = 0; i < RARRAY_LEN(ary); i++) {
       RepeatedField_push(_self, rb_ary_entry(ary, i));
     }
   }
@@ -589,8 +612,10 @@ void RepeatedField_mark(void* _self) {
   RepeatedField* self = (RepeatedField*)_self;
   upb_fieldtype_t field_type = self->field_type;
   int element_size = native_slot_size(field_type);
+  int i;
+
   rb_gc_mark(self->field_type_class);
-  for (int i = 0; i < self->size; i++) {
+  for (i = 0; i < self->size; i++) {
     void* memory = (((uint8_t *)self->elements) + i * element_size);
     native_slot_mark(self->field_type, memory);
   }

ruby/ext/google/protobuf_c/storage.c

@@ -181,32 +181,43 @@ void native_slot_set_value_and_case(const char* name,
         value = Qnil;
       } else if (CLASS_OF(value) != type_class) {
         // check for possible implicit conversions
-        VALUE converted_value = NULL;
-        char* field_type_name = rb_class2name(type_class);
+        VALUE converted_value = Qnil;
+        const char* field_type_name = rb_class2name(type_class);
 
         if (strcmp(field_type_name, "Google::Protobuf::Timestamp") == 0 &&
             rb_obj_is_kind_of(value, rb_cTime)) {
           // Time -> Google::Protobuf::Timestamp
           VALUE hash = rb_hash_new();
-          rb_hash_aset(hash, rb_str_new2("seconds"), rb_funcall(value, rb_intern("to_i"), 0));
-          rb_hash_aset(hash, rb_str_new2("nanos"), rb_funcall(value, rb_intern("nsec"), 0));
-          VALUE args[1] = { hash };
-          converted_value = rb_class_new_instance(1, args, type_class);
+          rb_hash_aset(hash, rb_str_new2("seconds"),
+                       rb_funcall(value, rb_intern("to_i"), 0));
+          rb_hash_aset(hash, rb_str_new2("nanos"),
+                       rb_funcall(value, rb_intern("nsec"), 0));
+          {
+            VALUE args[1] = {hash};
+            converted_value = rb_class_new_instance(1, args, type_class);
+          }
         } else if (strcmp(field_type_name, "Google::Protobuf::Duration") == 0 &&
                    rb_obj_is_kind_of(value, rb_cNumeric)) {
           // Numeric -> Google::Protobuf::Duration
           VALUE hash = rb_hash_new();
-          rb_hash_aset(hash, rb_str_new2("seconds"), rb_funcall(value, rb_intern("to_i"), 0));
-          VALUE n_value = rb_funcall(value, rb_intern("remainder"), 1, INT2NUM(1));
-          n_value = rb_funcall(n_value, rb_intern("*"), 1, INT2NUM(1000000000));
-          n_value = rb_funcall(n_value, rb_intern("round"), 0);
-          rb_hash_aset(hash, rb_str_new2("nanos"), n_value);
-          VALUE args[1] = { hash };
-          converted_value = rb_class_new_instance(1, args, type_class);
+          rb_hash_aset(hash, rb_str_new2("seconds"),
+                       rb_funcall(value, rb_intern("to_i"), 0));
+          {
+            VALUE n_value =
+                rb_funcall(value, rb_intern("remainder"), 1, INT2NUM(1));
+            n_value =
+                rb_funcall(n_value, rb_intern("*"), 1, INT2NUM(1000000000));
+            n_value = rb_funcall(n_value, rb_intern("round"), 0);
+            rb_hash_aset(hash, rb_str_new2("nanos"), n_value);
+          }
+          {
+            VALUE args[1] = { hash };
+            converted_value = rb_class_new_instance(1, args, type_class);
+          }
         }
 
         // raise if no suitable conversaion could be found
-        if (converted_value == NULL) {
+        if (converted_value == Qnil) {
           rb_raise(cTypeError,
                    "Invalid type %s to assign to submessage field '%s'.",
                   rb_class2name(CLASS_OF(value)), name);
@@ -462,14 +473,17 @@ static size_t align_up_to(size_t offset, size_t granularity) {
   return (offset + granularity - 1) & ~(granularity - 1);
 }
 
-MessageLayout* create_layout(const upb_msgdef* msgdef) {
+MessageLayout* create_layout(const Descriptor* desc) {
+  const upb_msgdef *msgdef = desc->msgdef;
   MessageLayout* layout = ALLOC(MessageLayout);
   int nfields = upb_msgdef_numfields(msgdef);
   int noneofs = upb_msgdef_numoneofs(msgdef);
   upb_msg_field_iter it;
   upb_msg_oneof_iter oit;
   size_t off = 0;
+  size_t hasbit = 0;
 
+  layout->desc = desc;
   layout->fields = ALLOC_N(MessageField, nfields);
   layout->oneofs = NULL;
 
@@ -477,7 +491,6 @@ MessageLayout* create_layout(const upb_msgdef* msgdef) {
     layout->oneofs = ALLOC_N(MessageOneof, noneofs);
   }
 
-  size_t hasbit = 0;
   for (upb_msg_field_begin(&it, msgdef);
        !upb_msg_field_done(&it);
        upb_msg_field_next(&it)) {
@@ -557,8 +570,6 @@ MessageLayout* create_layout(const upb_msgdef* msgdef) {
        !upb_msg_oneof_done(&oit);
        upb_msg_oneof_next(&oit)) {
     const upb_oneofdef* oneof = upb_msg_iter_oneof(&oit);
-    upb_oneof_iter fit;
-
     size_t field_size = sizeof(uint32_t);
     // Align the offset.
     off = (off + field_size - 1) & ~(field_size - 1);
@@ -567,9 +578,7 @@ MessageLayout* create_layout(const upb_msgdef* msgdef) {
   }
 
   layout->size = off;
-
   layout->msgdef = msgdef;
-  upb_msgdef_ref(layout->msgdef, &layout->msgdef);
 
   return layout;
 }
@@ -577,19 +586,18 @@ MessageLayout* create_layout(const upb_msgdef* msgdef) {
 void free_layout(MessageLayout* layout) {
   xfree(layout->fields);
   xfree(layout->oneofs);
-  upb_msgdef_unref(layout->msgdef, &layout->msgdef);
   xfree(layout);
 }
 
-VALUE field_type_class(const upb_fielddef* field) {
+VALUE field_type_class(const MessageLayout* layout, const upb_fielddef* field) {
   VALUE type_class = Qnil;
   if (upb_fielddef_type(field) == UPB_TYPE_MESSAGE) {
-    VALUE submsgdesc =
-        get_def_obj(upb_fielddef_subdef(field));
+    VALUE submsgdesc = get_msgdef_obj(layout->desc->descriptor_pool,
+                                      upb_fielddef_msgsubdef(field));
     type_class = Descriptor_msgclass(submsgdesc);
   } else if (upb_fielddef_type(field) == UPB_TYPE_ENUM) {
-    VALUE subenumdesc =
-        get_def_obj(upb_fielddef_subdef(field));
+    VALUE subenumdesc = get_enumdef_obj(layout->desc->descriptor_pool,
+                                        upb_fielddef_enumsubdef(field));
     type_class = EnumDescriptor_enummodule(subenumdesc);
   }
   return type_class;
@@ -670,7 +678,7 @@ void layout_clear(MessageLayout* layout,
 
     const upb_fielddef* key_field = map_field_key(field);
     const upb_fielddef* value_field = map_field_value(field);
-    VALUE type_class = field_type_class(value_field);
+    VALUE type_class = field_type_class(layout, value_field);
 
     if (type_class != Qnil) {
       VALUE args[3] = {
@@ -691,7 +699,7 @@ void layout_clear(MessageLayout* layout,
   } else if (upb_fielddef_label(field) == UPB_LABEL_REPEATED) {
     VALUE ary = Qnil;
 
-    VALUE type_class = field_type_class(field);
+    VALUE type_class = field_type_class(layout, field);
 
     if (type_class != Qnil) {
       VALUE args[2] = {
@@ -706,9 +714,9 @@ void layout_clear(MessageLayout* layout,
 
     DEREF(memory, VALUE) = ary;
   } else {
-    native_slot_set(upb_fielddef_name(field),
-                    upb_fielddef_type(field), field_type_class(field),
-                    memory, layout_get_default(field));
+    native_slot_set(upb_fielddef_name(field), upb_fielddef_type(field),
+                    field_type_class(layout, field), memory,
+                    layout_get_default(field));
   }
 }
 
@@ -762,20 +770,19 @@ VALUE layout_get(MessageLayout* layout,
       return layout_get_default(field);
     }
     return native_slot_get(upb_fielddef_type(field),
-                           field_type_class(field),
-                           memory);
+                           field_type_class(layout, field), memory);
   } else if (upb_fielddef_label(field) == UPB_LABEL_REPEATED) {
     return *((VALUE *)memory);
   } else if (!field_set) {
     return layout_get_default(field);
   } else {
     return native_slot_get(upb_fielddef_type(field),
-                           field_type_class(field),
-                           memory);
+                           field_type_class(layout, field), memory);
   }
 }
 
-static void check_repeated_field_type(VALUE val, const upb_fielddef* field) {
+static void check_repeated_field_type(const MessageLayout* layout, VALUE val,
+                                      const upb_fielddef* field) {
   RepeatedField* self;
   assert(upb_fielddef_label(field) == UPB_LABEL_REPEATED);
 
@@ -789,25 +796,13 @@ static void check_repeated_field_type(VALUE val, const upb_fielddef* field) {
     rb_raise(cTypeError, "Repeated field array has wrong element type");
   }
 
-  if (self->field_type == UPB_TYPE_MESSAGE) {
-    if (self->field_type_class !=
-        Descriptor_msgclass(get_def_obj(upb_fielddef_subdef(field)))) {
-      rb_raise(cTypeError,
-               "Repeated field array has wrong message class");
-    }
-  }
-
-
-  if (self->field_type == UPB_TYPE_ENUM) {
-    if (self->field_type_class !=
-        EnumDescriptor_enummodule(get_def_obj(upb_fielddef_subdef(field)))) {
-      rb_raise(cTypeError,
-               "Repeated field array has wrong enum class");
-    }
+  if (self->field_type_class != field_type_class(layout, field)) {
+    rb_raise(cTypeError, "Repeated field array has wrong message/enum class");
   }
 }
 
-static void check_map_field_type(VALUE val, const upb_fielddef* field) {
+static void check_map_field_type(const MessageLayout* layout, VALUE val,
+                                 const upb_fielddef* field) {
   const upb_fielddef* key_field = map_field_key(field);
   const upb_fielddef* value_field = map_field_value(field);
   Map* self;
@@ -824,17 +819,11 @@ static void check_map_field_type(VALUE val, const upb_fielddef* field) {
   if (self->value_type != upb_fielddef_type(value_field)) {
     rb_raise(cTypeError, "Map value type does not match field's value type");
   }
-  if (upb_fielddef_type(value_field) == UPB_TYPE_MESSAGE ||
-      upb_fielddef_type(value_field) == UPB_TYPE_ENUM) {
-    if (self->value_type_class !=
-        get_def_obj(upb_fielddef_subdef(value_field))) {
-      rb_raise(cTypeError,
-               "Map value type has wrong message/enum class");
-    }
+  if (self->value_type_class != field_type_class(layout, value_field)) {
+    rb_raise(cTypeError, "Map value type has wrong message/enum class");
   }
 }
 
-
 void layout_set(MessageLayout* layout,
                 void* storage,
                 const upb_fielddef* field,
@@ -863,20 +852,19 @@ void layout_set(MessageLayout* layout,
       // and case number are altered atomically (w.r.t. the Ruby VM).
       native_slot_set_value_and_case(
           upb_fielddef_name(field),
-          upb_fielddef_type(field), field_type_class(field),
+          upb_fielddef_type(field), field_type_class(layout, field),
           memory, val,
           oneof_case, upb_fielddef_number(field));
     }
   } else if (is_map_field(field)) {
-    check_map_field_type(val, field);
+    check_map_field_type(layout, val, field);
     DEREF(memory, VALUE) = val;
   } else if (upb_fielddef_label(field) == UPB_LABEL_REPEATED) {
-    check_repeated_field_type(val, field);
+    check_repeated_field_type(layout, val, field);
     DEREF(memory, VALUE) = val;
   } else {
-    native_slot_set(upb_fielddef_name(field),
-                    upb_fielddef_type(field), field_type_class(field),
-                    memory, val);
+    native_slot_set(upb_fielddef_name(field), upb_fielddef_type(field),
+                    field_type_class(layout, field), memory, val);
   }
 
   if (layout->fields[upb_fielddef_index(field)].hasbit !=

ruby/lib/google/protobuf.rb

@@ -50,6 +50,72 @@ else
   rescue LoadError
     require 'google/protobuf_c'
   end
+
+  module Google
+    module Protobuf
+      module Internal
+        def self.infer_package(names)
+          # Package is longest common prefix ending in '.', if any.
+          min, max = names.minmax
+          last_common_dot = nil
+          min.size.times { |i|
+            if min[i] != max[i] then break end
+            if min[i] == ?. then last_common_dot = i end
+          }
+          if last_common_dot
+            return min.slice(0, last_common_dot)
+          end
+        end
+
+        class NestingBuilder
+          def initialize(msg_names, enum_names)
+            @to_pos = {nil=>nil}
+            @msg_children = Hash.new { |hash, key| hash[key] = [] }
+            @enum_children = Hash.new { |hash, key| hash[key] = [] }
+
+            msg_names.each_with_index { |name, idx| @to_pos[name] = idx }
+            enum_names.each_with_index { |name, idx| @to_pos[name] = idx }
+
+            msg_names.each { |name| @msg_children[parent(name)] << name }
+            enum_names.each { |name| @enum_children[parent(name)] << name }
+          end
+
+          def build(package)
+            return build_msg(package)
+          end
+
+          private
+          def build_msg(msg)
+            return {
+              :pos => @to_pos[msg],
+              :msgs => @msg_children[msg].map { |child| build_msg(child) },
+              :enums => @enum_children[msg].map { |child| @to_pos[child] },
+            }
+          end
+
+          private
+          def parent(name)
+            idx = name.rindex(?.)
+            if idx
+              return name.slice(0, idx)
+            else
+              return nil
+            end
+          end
+        end
+
+        def self.fixup_descriptor(package, msg_names, enum_names)
+          if package.nil?
+            package = self.infer_package(msg_names + enum_names)
+          end
+
+          nesting = NestingBuilder.new(msg_names, enum_names).build(package)
+
+          return package, nesting
+        end
+      end
+    end
+  end
 end
 
 require 'google/protobuf/repeated_field'

ruby/tests/basic.rb

@@ -17,7 +17,6 @@ module BasicTest
     add_message "BadFieldNames" do
       optional :dup, :int32, 1
       optional :class, :int32, 2
-      optional :"a.b", :int32, 3
     end
   end
 
@@ -285,6 +284,14 @@ module BasicTest
       assert_match(/No such field: not_in_message/, e.message)
     end
 
+    #def test_json_quoted_string
+    #  m = TestMessage.decode_json(%q(
+    #    "optionalInt64": "1",,
+    #  }))
+    #  puts(m)
+    #  assert_equal 1, m.optional_int32
+    #end
+
     def test_to_h
       m = TestMessage.new(:optional_bool => true, :optional_double => -10.100001, :optional_string => 'foo', :repeated_string => ['bar1', 'bar2'], :repeated_msg => [TestMessage2.new(:foo => 100)])
       expected_result = {
@@ -371,11 +378,6 @@ module BasicTest
       assert nil != file_descriptor
       assert_equal "tests/basic_test.proto", file_descriptor.name
       assert_equal :proto3, file_descriptor.syntax
-
-      file_descriptor = BadFieldNames.descriptor.file_descriptor
-      assert nil != file_descriptor
-      assert_equal nil, file_descriptor.name
-      assert_equal :proto3, file_descriptor.syntax
     end
 
     # Ruby 2.5 changed to raise FrozenError instead of RuntimeError

ruby/tests/basic_proto2.rb

@@ -18,7 +18,6 @@ module BasicTestProto2
       add_message "BadFieldNames" do
         optional :dup, :int32, 1
         optional :class, :int32, 2
-        optional :"a.b", :int32, 3
       end
     end
   end

ruby/tests/common_tests.rb

@@ -807,7 +807,7 @@ module CommonTests
                                                         proto_module::TestMessage2.new(:foo => 2)])
     data = proto_module::TestMessage.encode m
     m2 = proto_module::TestMessage.decode data
-    assert m == m2
+    assert_equal m, m2
 
     data = Google::Protobuf.encode m
     m2 = Google::Protobuf.decode(proto_module::TestMessage, data)
@@ -902,8 +902,6 @@ module CommonTests
     assert m['class'] == 2
     m['dup'] = 3
     assert m['dup'] == 3
-    m['a.b'] = 4
-    assert m['a.b'] == 4
   end
 
   def test_int_ranges
@@ -1084,9 +1082,7 @@ module CommonTests
 
     json_text = proto_module::TestMessage.encode_json(m)
     m2 = proto_module::TestMessage.decode_json(json_text)
-    puts m.inspect
-    puts m2.inspect
-    assert m == m2
+    assert_equal m, m2
 
     # Crash case from GitHub issue 283.
     bar = proto_module::Bar.new(msg: "bar")
@@ -1132,7 +1128,7 @@ module CommonTests
 
     actual = proto_module::TestMessage.encode_json(m, :emit_defaults => true)
 
-    assert JSON.parse(actual, :symbolize_names => true) == expected
+    assert_equal expected, JSON.parse(actual, :symbolize_names => true)
   end
 
   def test_json_emit_defaults_submsg
@@ -1167,7 +1163,7 @@ module CommonTests
 
     actual = proto_module::TestMessage.encode_json(m, :emit_defaults => true)
 
-    assert JSON.parse(actual, :symbolize_names => true) == expected
+    assert_equal expected, JSON.parse(actual, :symbolize_names => true)
   end
 
   def test_json_emit_defaults_repeated_submsg
@@ -1201,7 +1197,7 @@ module CommonTests
 
     actual = proto_module::TestMessage.encode_json(m, :emit_defaults => true)
 
-    assert JSON.parse(actual, :symbolize_names => true) == expected
+    assert_equal expected, JSON.parse(actual, :symbolize_names => true)
   end
 
   def value_from_ruby(value)