Roll forward Ruby upb changes now that protobuf Ruby build is fixed (#5866)

* Rolled forward again with "Updated upb from defcleanup branch..."

Revert "Revert "Updated upb from defcleanup branch and modified Ruby to use it (#5539)" (#5848)"

This reverts commit 1568deab40db055a79fa415d21547eaffbd56d57.

* A few more merge fixes.

* Updated for defcleanup2 branch.

* Fixed upb to define upb_decode().

* Fixed names of nested messages.

* Revert submodule.

* Set -std=gnu90 and fixed warnings/errors.

Some of our Kokoro tests seem to run with this level of warnings,
and the source strives to be gnu90 compatible.  Enforcing it for
every build removes the possibility of some errors showing up in
Kokoro/Travis tests only.

* Fixed remaining warnings with gnu90 mode.

I tried to match warning flags with what Ruby appears to do
in our Kokoro tests.

* Initialize values registered by rb_gc_register_address().

* Fixed subtle GC bug.

We need to initialize this marked value before creating the instance.

* Truly fix the GC bug.

* Updated upb for mktime() fix.

* Removed XOPEN_SOURCE as we are not using strptime().

* Removed fixed tests from the conformance failure list for Ruby.

* Fixed memory error related to oneof def names.

* Picked up new upb changes re: JSON printing.

* Uncomment concurrent decoding test.

ruby/compatibility_tests/v3.0.0/tests/basic.rb

@@ -67,7 +67,6 @@ module BasicTest
     add_message "BadFieldNames" do
       optional :dup, :int32, 1
       optional :class, :int32, 2
-      optional :"a.b", :int32, 3
     end
 
     add_message "MapMessage" do
@@ -1067,8 +1066,6 @@ module BasicTest
       assert m['class'] == 2
       m['dup'] = 3
       assert m['dup'] == 3
-      m['a.b'] = 4
-      assert m['a.b'] == 4
     end
 
     def test_int_ranges

ruby/ext/google/protobuf_c/encode_decode.c

@@ -117,18 +117,18 @@ static const void* newhandlerdata(upb_handlers* h, uint32_t ofs, int32_t hasbit)
 typedef struct {
   size_t ofs;
   int32_t hasbit;
-  const upb_msgdef *md;
+  VALUE subklass;
 } submsg_handlerdata_t;
 
 // Creates a handlerdata that contains offset and submessage type information.
 static const void *newsubmsghandlerdata(upb_handlers* h,
                                         uint32_t ofs,
                                         int32_t hasbit,
-                                        const upb_fielddef* f) {
+                                        VALUE subklass) {
   submsg_handlerdata_t *hd = ALLOC(submsg_handlerdata_t);
   hd->ofs = ofs;
   hd->hasbit = hasbit;
-  hd->md = upb_fielddef_msgsubdef(f);
+  hd->subklass = subklass;
   upb_handlers_addcleanup(h, hd, xfree);
   return hd;
 }
@@ -137,13 +137,14 @@ typedef struct {
   size_t ofs;              // union data slot
   size_t case_ofs;         // oneof_case field
   uint32_t oneof_case_num; // oneof-case number to place in oneof_case field
-  const upb_msgdef *md;    // msgdef, for oneof submessage handler
+  VALUE subklass;
 } oneof_handlerdata_t;
 
 static const void *newoneofhandlerdata(upb_handlers *h,
                                        uint32_t ofs,
                                        uint32_t case_ofs,
-                                       const upb_fielddef *f) {
+                                       const upb_fielddef *f,
+                                       const Descriptor* desc) {
   oneof_handlerdata_t *hd = ALLOC(oneof_handlerdata_t);
   hd->ofs = ofs;
   hd->case_ofs = case_ofs;
@@ -154,11 +155,7 @@ static const void *newoneofhandlerdata(upb_handlers *h,
   // create a separate ID space. In addition, using the field tag number here
   // lets us easily look up the field in the oneof accessor.
   hd->oneof_case_num = upb_fielddef_number(f);
-  if (upb_fielddef_type(f) == UPB_TYPE_MESSAGE) {
-    hd->md = upb_fielddef_msgsubdef(f);
-  } else {
-    hd->md = NULL;
-  }
+  hd->subklass = field_type_class(desc->layout, f);
   upb_handlers_addcleanup(h, hd, xfree);
   return hd;
 }
@@ -254,13 +251,13 @@ static size_t stringdata_handler(void* closure, const void* hd,
 }
 
 static bool stringdata_end_handler(void* closure, const void* hd) {
-  VALUE rb_str = closure;
+  VALUE rb_str = (VALUE)closure;
   rb_obj_freeze(rb_str);
   return true;
 }
 
 static bool appendstring_end_handler(void* closure, const void* hd) {
-  VALUE rb_str = closure;
+  VALUE rb_str = (VALUE)closure;
   rb_obj_freeze(rb_str);
   return true;
 }
@@ -269,12 +266,9 @@ static bool appendstring_end_handler(void* closure, const void* hd) {
 static void *appendsubmsg_handler(void *closure, const void *hd) {
   VALUE ary = (VALUE)closure;
   const submsg_handlerdata_t *submsgdata = hd;
-  VALUE subdesc =
-      get_def_obj((void*)submsgdata->md);
-  VALUE subklass = Descriptor_msgclass(subdesc);
   MessageHeader* submsg;
 
-  VALUE submsg_rb = rb_class_new_instance(0, NULL, subklass);
+  VALUE submsg_rb = rb_class_new_instance(0, NULL, submsgdata->subklass);
   RepeatedField_push(ary, submsg_rb);
 
   TypedData_Get_Struct(submsg_rb, MessageHeader, &Message_type, submsg);
@@ -285,15 +279,12 @@ static void *appendsubmsg_handler(void *closure, const void *hd) {
 static void *submsg_handler(void *closure, const void *hd) {
   MessageHeader* msg = closure;
   const submsg_handlerdata_t* submsgdata = hd;
-  VALUE subdesc =
-      get_def_obj((void*)submsgdata->md);
-  VALUE subklass = Descriptor_msgclass(subdesc);
   VALUE submsg_rb;
   MessageHeader* submsg;
 
   if (DEREF(msg, submsgdata->ofs, VALUE) == Qnil) {
     DEREF(msg, submsgdata->ofs, VALUE) =
-        rb_class_new_instance(0, NULL, subklass);
+        rb_class_new_instance(0, NULL, submsgdata->subklass);
   }
 
   set_hasbit(closure, submsgdata->hasbit);
@@ -309,11 +300,7 @@ typedef struct {
   size_t ofs;
   upb_fieldtype_t key_field_type;
   upb_fieldtype_t value_field_type;
-
-  // We know that we can hold this reference because the handlerdata has the
-  // same lifetime as the upb_handlers struct, and the upb_handlers struct holds
-  // a reference to the upb_msgdef, which in turn has references to its subdefs.
-  const upb_def* value_field_subdef;
+  VALUE subklass;
 } map_handlerdata_t;
 
 // Temporary frame for map parsing: at the beginning of a map entry message, a
@@ -383,19 +370,8 @@ static bool endmap_handler(void *closure, const void *hd, upb_status* s) {
       mapdata->key_field_type, Qnil,
       &frame->key_storage);
 
-  VALUE value_field_typeclass = Qnil;
-  VALUE value;
-
-  if (mapdata->value_field_type == UPB_TYPE_MESSAGE ||
-      mapdata->value_field_type == UPB_TYPE_ENUM) {
-    value_field_typeclass = get_def_obj(mapdata->value_field_subdef);
-    if (mapdata->value_field_type == UPB_TYPE_ENUM) {
-      value_field_typeclass = EnumDescriptor_enummodule(value_field_typeclass);
-    }
-  }
-
-  value = native_slot_get(
-      mapdata->value_field_type, value_field_typeclass,
+  VALUE value = native_slot_get(
+      mapdata->value_field_type, mapdata->subklass,
       &frame->value_storage);
 
   Map_index_set(frame->map, key, value);
@@ -414,7 +390,7 @@ static bool endmap_handler(void *closure, const void *hd, upb_status* s) {
 static map_handlerdata_t* new_map_handlerdata(
     size_t ofs,
     const upb_msgdef* mapentry_def,
-    Descriptor* desc) {
+    const Descriptor* desc) {
   const upb_fielddef* key_field;
   const upb_fielddef* value_field;
   map_handlerdata_t* hd = ALLOC(map_handlerdata_t);
@@ -425,7 +401,7 @@ static map_handlerdata_t* new_map_handlerdata(
   value_field = upb_msgdef_itof(mapentry_def, MAP_VALUE_FIELD);
   assert(value_field != NULL);
   hd->value_field_type = upb_fielddef_type(value_field);
-  hd->value_field_subdef = upb_fielddef_subdef(value_field);
+  hd->subklass = field_type_class(desc->layout, value_field);
 
   return hd;
 }
@@ -491,16 +467,13 @@ static void *oneofsubmsg_handler(void *closure,
   const oneof_handlerdata_t *oneofdata = hd;
   uint32_t oldcase = DEREF(msg, oneofdata->case_ofs, uint32_t);
 
-  VALUE subdesc =
-      get_def_obj((void*)oneofdata->md);
-  VALUE subklass = Descriptor_msgclass(subdesc);
   VALUE submsg_rb;
   MessageHeader* submsg;
 
   if (oldcase != oneofdata->oneof_case_num ||
       DEREF(msg, oneofdata->ofs, VALUE) == Qnil) {
     DEREF(msg, oneofdata->ofs, VALUE) =
-        rb_class_new_instance(0, NULL, subklass);
+        rb_class_new_instance(0, NULL, oneofdata->subklass);
   }
   // Set the oneof case *after* allocating the new class instance -- otherwise,
   // if the Ruby GC is invoked as part of a call into the VM, it might invoke
@@ -518,12 +491,12 @@ static void *oneofsubmsg_handler(void *closure,
 
 // Set up handlers for a repeated field.
 static void add_handlers_for_repeated_field(upb_handlers *h,
+                                            const Descriptor* desc,
                                             const upb_fielddef *f,
                                             size_t offset) {
-  upb_handlerattr attr = UPB_HANDLERATTR_INITIALIZER;
-  upb_handlerattr_sethandlerdata(&attr, newhandlerdata(h, offset, -1));
+  upb_handlerattr attr = UPB_HANDLERATTR_INIT;
+  attr.handler_data = newhandlerdata(h, offset, -1);
   upb_handlers_setstartseq(h, f, startseq_handler, &attr);
-  upb_handlerattr_uninit(&attr);
 
   switch (upb_fielddef_type(f)) {
 
@@ -554,20 +527,20 @@ static void add_handlers_for_repeated_field(upb_handlers *h,
       break;
     }
     case UPB_TYPE_MESSAGE: {
-      upb_handlerattr attr = UPB_HANDLERATTR_INITIALIZER;
-      upb_handlerattr_sethandlerdata(&attr, newsubmsghandlerdata(h, 0, -1, f));
+      VALUE subklass = field_type_class(desc->layout, f);
+      upb_handlerattr attr = UPB_HANDLERATTR_INIT;
+      attr.handler_data = newsubmsghandlerdata(h, 0, -1, subklass);
       upb_handlers_setstartsubmsg(h, f, appendsubmsg_handler, &attr);
-      upb_handlerattr_uninit(&attr);
       break;
     }
   }
 }
 
 // Set up handlers for a singular field.
-static void add_handlers_for_singular_field(upb_handlers *h,
-                                            const upb_fielddef *f,
-                                            size_t offset,
-                                            size_t hasbit_off) {
+static void add_handlers_for_singular_field(const Descriptor* desc,
+                                            upb_handlers* h,
+                                            const upb_fielddef* f,
+                                            size_t offset, size_t hasbit_off) {
   // The offset we pass to UPB points to the start of the Message,
   // rather than the start of where our data is stored.
   int32_t hasbit = -1;
@@ -589,23 +562,20 @@ static void add_handlers_for_singular_field(upb_handlers *h,
     case UPB_TYPE_STRING:
     case UPB_TYPE_BYTES: {
       bool is_bytes = upb_fielddef_type(f) == UPB_TYPE_BYTES;
-      upb_handlerattr attr = UPB_HANDLERATTR_INITIALIZER;
-      upb_handlerattr_sethandlerdata(&attr, newhandlerdata(h, offset, hasbit));
+      upb_handlerattr attr = UPB_HANDLERATTR_INIT;
+      attr.handler_data = newhandlerdata(h, offset, hasbit);
       upb_handlers_setstartstr(h, f,
                                is_bytes ? bytes_handler : str_handler,
                                &attr);
       upb_handlers_setstring(h, f, stringdata_handler, &attr);
       upb_handlers_setendstr(h, f, stringdata_end_handler, &attr);
-      upb_handlerattr_uninit(&attr);
       break;
     }
     case UPB_TYPE_MESSAGE: {
-      upb_handlerattr attr = UPB_HANDLERATTR_INITIALIZER;
-      upb_handlerattr_sethandlerdata(&attr,
-				     newsubmsghandlerdata(h, offset,
-							  hasbit, f));
+      upb_handlerattr attr = UPB_HANDLERATTR_INIT;
+      attr.handler_data = newsubmsghandlerdata(
+          h, offset, hasbit, field_type_class(desc->layout, f));
       upb_handlers_setstartsubmsg(h, f, submsg_handler, &attr);
-      upb_handlerattr_uninit(&attr);
       break;
     }
   }
@@ -615,36 +585,34 @@ static void add_handlers_for_singular_field(upb_handlers *h,
 static void add_handlers_for_mapfield(upb_handlers* h,
                                       const upb_fielddef* fielddef,
                                       size_t offset,
-                                      Descriptor* desc) {
+                                      const Descriptor* desc) {
   const upb_msgdef* map_msgdef = upb_fielddef_msgsubdef(fielddef);
   map_handlerdata_t* hd = new_map_handlerdata(offset, map_msgdef, desc);
-  upb_handlerattr attr = UPB_HANDLERATTR_INITIALIZER;
+  upb_handlerattr attr = UPB_HANDLERATTR_INIT;
 
   upb_handlers_addcleanup(h, hd, xfree);
-  upb_handlerattr_sethandlerdata(&attr, hd);
+  attr.handler_data = hd;
   upb_handlers_setstartsubmsg(h, fielddef, startmapentry_handler, &attr);
-  upb_handlerattr_uninit(&attr);
 }
 
 // Adds handlers to a map-entry msgdef.
-static void add_handlers_for_mapentry(const upb_msgdef* msgdef,
-                                      upb_handlers* h,
-                                      Descriptor* desc) {
+static void add_handlers_for_mapentry(const upb_msgdef* msgdef, upb_handlers* h,
+                                      const Descriptor* desc) {
   const upb_fielddef* key_field = map_entry_key(msgdef);
   const upb_fielddef* value_field = map_entry_value(msgdef);
   map_handlerdata_t* hd = new_map_handlerdata(0, msgdef, desc);
-  upb_handlerattr attr = UPB_HANDLERATTR_INITIALIZER;
+  upb_handlerattr attr = UPB_HANDLERATTR_INIT;
 
   upb_handlers_addcleanup(h, hd, xfree);
-  upb_handlerattr_sethandlerdata(&attr, hd);
+  attr.handler_data = hd;
   upb_handlers_setendmsg(h, endmap_handler, &attr);
 
   add_handlers_for_singular_field(
-      h, key_field,
+      desc, h, key_field,
       offsetof(map_parse_frame_t, key_storage),
       MESSAGE_FIELD_NO_HASBIT);
   add_handlers_for_singular_field(
-      h, value_field,
+      desc, h, value_field,
       offsetof(map_parse_frame_t, value_storage),
       MESSAGE_FIELD_NO_HASBIT);
 }
@@ -653,11 +621,11 @@ static void add_handlers_for_mapentry(const upb_msgdef* msgdef,
 static void add_handlers_for_oneof_field(upb_handlers *h,
                                          const upb_fielddef *f,
                                          size_t offset,
-                                         size_t oneof_case_offset) {
-
-  upb_handlerattr attr = UPB_HANDLERATTR_INITIALIZER;
-  upb_handlerattr_sethandlerdata(
-      &attr, newoneofhandlerdata(h, offset, oneof_case_offset, f));
+                                         size_t oneof_case_offset,
+                                         const Descriptor* desc) {
+  upb_handlerattr attr = UPB_HANDLERATTR_INIT;
+  attr.handler_data =
+      newoneofhandlerdata(h, offset, oneof_case_offset, f, desc);
 
   switch (upb_fielddef_type(f)) {
 
@@ -692,15 +660,13 @@ static void add_handlers_for_oneof_field(upb_handlers *h,
       break;
     }
   }
-
-  upb_handlerattr_uninit(&attr);
 }
 
 static bool unknown_field_handler(void* closure, const void* hd,
                                   const char* buf, size_t size) {
+  MessageHeader* msg = (MessageHeader*)closure;
   UPB_UNUSED(hd);
 
-  MessageHeader* msg = (MessageHeader*)closure;
   if (msg->unknown_fields == NULL) {
     msg->unknown_fields = malloc(sizeof(stringsink));
     stringsink_init(msg->unknown_fields);
@@ -711,27 +677,29 @@ static bool unknown_field_handler(void* closure, const void* hd,
   return true;
 }
 
-static void add_handlers_for_message(const void *closure, upb_handlers *h) {
+void add_handlers_for_message(const void *closure, upb_handlers *h) {
+  const VALUE descriptor_pool = (VALUE)closure;
   const upb_msgdef* msgdef = upb_handlers_msgdef(h);
-  Descriptor* desc = ruby_to_Descriptor(get_def_obj((void*)msgdef));
+  Descriptor* desc =
+      ruby_to_Descriptor(get_msgdef_obj(descriptor_pool, msgdef));
   upb_msg_field_iter i;
-
-  // If this is a mapentry message type, set up a special set of handlers and
-  // bail out of the normal (user-defined) message type handling.
-  if (upb_msgdef_mapentry(msgdef)) {
-    add_handlers_for_mapentry(msgdef, h, desc);
-    return;
-  }
+  upb_handlerattr attr = UPB_HANDLERATTR_INIT;
 
   // Ensure layout exists. We may be invoked to create handlers for a given
   // message if we are included as a submsg of another message type before our
   // class is actually built, so to work around this, we just create the layout
   // (and handlers, in the class-building function) on-demand.
   if (desc->layout == NULL) {
-    desc->layout = create_layout(desc->msgdef);
+    desc->layout = create_layout(desc);
+  }
+
+  // If this is a mapentry message type, set up a special set of handlers and
+  // bail out of the normal (user-defined) message type handling.
+  if (upb_msgdef_mapentry(msgdef)) {
+    add_handlers_for_mapentry(msgdef, h, desc);
+    return;
   }
 
-  upb_handlerattr attr = UPB_HANDLERATTR_INITIALIZER;
   upb_handlers_setunknown(h, unknown_field_handler, &attr);
 
   for (upb_msg_field_begin(&i, desc->msgdef);
@@ -745,64 +713,51 @@ static void add_handlers_for_message(const void *closure, upb_handlers *h) {
       size_t oneof_case_offset =
           desc->layout->fields[upb_fielddef_index(f)].case_offset +
           sizeof(MessageHeader);
-      add_handlers_for_oneof_field(h, f, offset, oneof_case_offset);
+      add_handlers_for_oneof_field(h, f, offset, oneof_case_offset, desc);
     } else if (is_map_field(f)) {
       add_handlers_for_mapfield(h, f, offset, desc);
     } else if (upb_fielddef_isseq(f)) {
-      add_handlers_for_repeated_field(h, f, offset);
+      add_handlers_for_repeated_field(h, desc, f, offset);
     } else {
       add_handlers_for_singular_field(
-          h, f, offset, desc->layout->fields[upb_fielddef_index(f)].hasbit);
+          desc, h, f, offset,
+          desc->layout->fields[upb_fielddef_index(f)].hasbit);
     }
   }
 }
 
-// Creates upb handlers for populating a message.
-static const upb_handlers *new_fill_handlers(Descriptor* desc,
-                                             const void* owner) {
-  // TODO(cfallin, haberman): once upb gets a caching/memoization layer for
-  // handlers, reuse subdef handlers so that e.g. if we already parse
-  // B-with-field-of-type-C, we don't have to rebuild the whole hierarchy to
-  // parse A-with-field-of-type-B-with-field-of-type-C.
-  return upb_handlers_newfrozen(desc->msgdef, owner,
-                                add_handlers_for_message, NULL);
-}
-
 // Constructs the handlers for filling a message's data into an in-memory
 // object.
 const upb_handlers* get_fill_handlers(Descriptor* desc) {
-  if (!desc->fill_handlers) {
-    desc->fill_handlers =
-        new_fill_handlers(desc, &desc->fill_handlers);
-  }
-  return desc->fill_handlers;
+  DescriptorPool* pool = ruby_to_DescriptorPool(desc->descriptor_pool);
+  return upb_handlercache_get(pool->fill_handler_cache, desc->msgdef);
 }
 
-// Constructs the upb decoder method for parsing messages of this type.
-// This is called from the message class creation code.
-const upb_pbdecodermethod *new_fillmsg_decodermethod(Descriptor* desc,
-                                                     const void* owner) {
-  const upb_handlers* handlers = get_fill_handlers(desc);
-  upb_pbdecodermethodopts opts;
-  upb_pbdecodermethodopts_init(&opts, handlers);
+static const upb_pbdecodermethod *msgdef_decodermethod(Descriptor* desc) {
+  DescriptorPool* pool = ruby_to_DescriptorPool(desc->descriptor_pool);
+  return upb_pbcodecache_get(pool->fill_method_cache, desc->msgdef);
+}
 
-  return upb_pbdecodermethod_new(&opts, owner);
+static const upb_json_parsermethod *msgdef_jsonparsermethod(Descriptor* desc) {
+  DescriptorPool* pool = ruby_to_DescriptorPool(desc->descriptor_pool);
+  return upb_json_codecache_get(pool->json_fill_method_cache, desc->msgdef);
 }
 
-static const upb_pbdecodermethod *msgdef_decodermethod(Descriptor* desc) {
-  if (desc->fill_method == NULL) {
-    desc->fill_method = new_fillmsg_decodermethod(
-        desc, &desc->fill_method);
-  }
-  return desc->fill_method;
+static const upb_handlers* msgdef_pb_serialize_handlers(Descriptor* desc) {
+  DescriptorPool* pool = ruby_to_DescriptorPool(desc->descriptor_pool);
+  return upb_handlercache_get(pool->pb_serialize_handler_cache, desc->msgdef);
 }
 
-static const upb_json_parsermethod *msgdef_jsonparsermethod(Descriptor* desc) {
-  if (desc->json_fill_method == NULL) {
-    desc->json_fill_method =
-        upb_json_parsermethod_new(desc->msgdef, &desc->json_fill_method);
+static const upb_handlers* msgdef_json_serialize_handlers(
+    Descriptor* desc, bool preserve_proto_fieldnames) {
+  DescriptorPool* pool = ruby_to_DescriptorPool(desc->descriptor_pool);
+  if (preserve_proto_fieldnames) {
+    return upb_handlercache_get(pool->json_serialize_handler_preserve_cache,
+                                desc->msgdef);
+  } else {
+    return upb_handlercache_get(pool->json_serialize_handler_cache,
+                                desc->msgdef);
   }
-  return desc->json_fill_method;
 }
 
 
@@ -812,7 +767,8 @@ static const upb_json_parsermethod *msgdef_jsonparsermethod(Descriptor* desc) {
 // if any error occurs.
 #define STACK_ENV_STACKBYTES 4096
 typedef struct {
-  upb_env env;
+  upb_arena *arena;
+  upb_status status;
   const char* ruby_error_template;
   char allocbuf[STACK_ENV_STACKBYTES];
 } stackenv;
@@ -820,29 +776,22 @@ typedef struct {
 static void stackenv_init(stackenv* se, const char* errmsg);
 static void stackenv_uninit(stackenv* se);
 
-// Callback invoked by upb if any error occurs during parsing or serialization.
-static bool env_error_func(void* ud, const upb_status* status) {
-  stackenv* se = ud;
-  // Free the env -- rb_raise will longjmp up the stack past the encode/decode
-  // function so it would not otherwise have been freed.
-  stackenv_uninit(se);
-
-  // TODO(haberman): have a way to verify that this is actually a parse error,
-  // instead of just throwing "parse error" unconditionally.
-  rb_raise(cParseError, se->ruby_error_template, upb_status_errmsg(status));
-  // Never reached: rb_raise() always longjmp()s up the stack, past all of our
-  // code, back to Ruby.
-  return false;
-}
-
 static void stackenv_init(stackenv* se, const char* errmsg) {
   se->ruby_error_template = errmsg;
-  upb_env_init2(&se->env, se->allocbuf, sizeof(se->allocbuf), NULL);
-  upb_env_seterrorfunc(&se->env, env_error_func, se);
+  se->arena =
+      upb_arena_init(se->allocbuf, sizeof(se->allocbuf), &upb_alloc_global);
+  upb_status_clear(&se->status);
 }
 
 static void stackenv_uninit(stackenv* se) {
-  upb_env_uninit(&se->env);
+  upb_arena_free(se->arena);
+
+  if (!upb_ok(&se->status)) {
+    // TODO(haberman): have a way to verify that this is actually a parse error,
+    // instead of just throwing "parse error" unconditionally.
+    VALUE errmsg = rb_str_new2(upb_status_errmsg(&se->status));
+    rb_raise(cParseError, se->ruby_error_template, errmsg);
+  }
 }
 
 /*
@@ -873,10 +822,10 @@ VALUE Message_decode(VALUE klass, VALUE data) {
     stackenv se;
     upb_sink sink;
     upb_pbdecoder* decoder;
-    stackenv_init(&se, "Error occurred during parsing: %s");
+    stackenv_init(&se, "Error occurred during parsing: %" PRIsVALUE);
 
     upb_sink_reset(&sink, h, msg);
-    decoder = upb_pbdecoder_create(&se.env, method, &sink);
+    decoder = upb_pbdecoder_create(se.arena, method, sink, &se.status);
     upb_bufsrc_putbuf(RSTRING_PTR(data), RSTRING_LEN(data),
                       upb_pbdecoder_input(decoder));
 
@@ -894,8 +843,9 @@ VALUE Message_decode(VALUE klass, VALUE data) {
  * format) under the interpretration given by this message class's definition
  * and returns a message object with the corresponding field values.
  *
- * @param options [Hash] options for the decoder
- *   ignore_unknown_fields: set true to ignore unknown fields (default is to raise an error)
+ *  @param options [Hash] options for the decoder
+ *   ignore_unknown_fields: set true to ignore unknown fields (default is to
+ *   raise an error)
  */
 VALUE Message_decode_json(int argc, VALUE* argv, VALUE klass) {
   VALUE descriptor = rb_ivar_get(klass, descriptor_instancevar_interned);
@@ -923,6 +873,7 @@ VALUE Message_decode_json(int argc, VALUE* argv, VALUE klass) {
   if (TYPE(data) != T_STRING) {
     rb_raise(rb_eArgError, "Expected string for JSON data.");
   }
+
   // TODO(cfallin): Check and respect string encoding. If not UTF-8, we need to
   // convert, because string handlers pass data directly to message string
   // fields.
@@ -936,11 +887,11 @@ VALUE Message_decode_json(int argc, VALUE* argv, VALUE klass) {
     upb_sink sink;
     upb_json_parser* parser;
     DescriptorPool* pool = ruby_to_DescriptorPool(generated_pool);
-    stackenv_init(&se, "Error occurred during parsing: %s");
+    stackenv_init(&se, "Error occurred during parsing: %" PRIsVALUE);
 
     upb_sink_reset(&sink, get_fill_handlers(desc), msg);
-    parser = upb_json_parser_create(&se.env, method, pool->symtab,
-                                    &sink, ignore_unknown_fields);
+    parser = upb_json_parser_create(se.arena, method, pool->symtab, sink,
+                                    &se.status, RTEST(ignore_unknown_fields));
     upb_bufsrc_putbuf(RSTRING_PTR(data), RSTRING_LEN(data),
                       upb_json_parser_input(parser));
 
@@ -956,9 +907,8 @@ VALUE Message_decode_json(int argc, VALUE* argv, VALUE klass) {
 
 /* msgvisitor *****************************************************************/
 
-static void putmsg(VALUE msg, const Descriptor* desc,
-                   upb_sink *sink, int depth, bool emit_defaults,
-                   bool is_json, bool open_msg);
+static void putmsg(VALUE msg, const Descriptor* desc, upb_sink sink, int depth,
+                   bool emit_defaults, bool is_json, bool open_msg);
 
 static upb_selector_t getsel(const upb_fielddef *f, upb_handlertype_t type) {
   upb_selector_t ret;
@@ -967,7 +917,7 @@ static upb_selector_t getsel(const upb_fielddef *f, upb_handlertype_t type) {
   return ret;
 }
 
-static void putstr(VALUE str, const upb_fielddef *f, upb_sink *sink) {
+static void putstr(VALUE str, const upb_fielddef *f, upb_sink sink) {
   upb_sink subsink;
 
   if (str == Qnil) return;
@@ -984,12 +934,12 @@ static void putstr(VALUE str, const upb_fielddef *f, upb_sink *sink) {
 
   upb_sink_startstr(sink, getsel(f, UPB_HANDLER_STARTSTR), RSTRING_LEN(str),
                     &subsink);
-  upb_sink_putstring(&subsink, getsel(f, UPB_HANDLER_STRING), RSTRING_PTR(str),
+  upb_sink_putstring(subsink, getsel(f, UPB_HANDLER_STRING), RSTRING_PTR(str),
                      RSTRING_LEN(str), NULL);
   upb_sink_endstr(sink, getsel(f, UPB_HANDLER_ENDSTR));
 }
 
-static void putsubmsg(VALUE submsg, const upb_fielddef *f, upb_sink *sink,
+static void putsubmsg(VALUE submsg, const upb_fielddef *f, upb_sink sink,
                       int depth, bool emit_defaults, bool is_json) {
   upb_sink subsink;
   VALUE descriptor;
@@ -1001,16 +951,17 @@ static void putsubmsg(VALUE submsg, const upb_fielddef *f, upb_sink *sink,
   subdesc = ruby_to_Descriptor(descriptor);
 
   upb_sink_startsubmsg(sink, getsel(f, UPB_HANDLER_STARTSUBMSG), &subsink);
-  putmsg(submsg, subdesc, &subsink, depth + 1, emit_defaults, is_json, true);
+  putmsg(submsg, subdesc, subsink, depth + 1, emit_defaults, is_json, true);
   upb_sink_endsubmsg(sink, getsel(f, UPB_HANDLER_ENDSUBMSG));
 }
 
-static void putary(VALUE ary, const upb_fielddef *f, upb_sink *sink,
-                   int depth, bool emit_defaults, bool is_json) {
+static void putary(VALUE ary, const upb_fielddef* f, upb_sink sink, int depth,
+                   bool emit_defaults, bool is_json) {
   upb_sink subsink;
   upb_fieldtype_t type = upb_fielddef_type(f);
   upb_selector_t sel = 0;
   int size;
+  int i;
 
   if (ary == Qnil) return;
   if (!emit_defaults && NUM2INT(RepeatedField_length(ary)) == 0) return;
@@ -1024,12 +975,12 @@ static void putary(VALUE ary, const upb_fielddef *f, upb_sink *sink,
     sel = getsel(f, upb_handlers_getprimitivehandlertype(f));
   }
 
-  for (int i = 0; i < size; i++) {
+  for (i = 0; i < size; i++) {
     void* memory = RepeatedField_index_native(ary, i);
     switch (type) {
-#define T(upbtypeconst, upbtype, ctype)                         \
-  case upbtypeconst:                                            \
-    upb_sink_put##upbtype(&subsink, sel, *((ctype *)memory));   \
+#define T(upbtypeconst, upbtype, ctype)                     \
+  case upbtypeconst:                                        \
+    upb_sink_put##upbtype(subsink, sel, *((ctype*)memory)); \
     break;
 
       T(UPB_TYPE_FLOAT,  float,  float)
@@ -1043,11 +994,10 @@ static void putary(VALUE ary, const upb_fielddef *f, upb_sink *sink,
 
       case UPB_TYPE_STRING:
       case UPB_TYPE_BYTES:
-        putstr(*((VALUE *)memory), f, &subsink);
+        putstr(*((VALUE *)memory), f, subsink);
         break;
       case UPB_TYPE_MESSAGE:
-        putsubmsg(*((VALUE *)memory), f, &subsink, depth,
-                  emit_defaults, is_json);
+        putsubmsg(*((VALUE*)memory), f, subsink, depth, emit_defaults, is_json);
         break;
 
 #undef T
@@ -1057,19 +1007,16 @@ static void putary(VALUE ary, const upb_fielddef *f, upb_sink *sink,
   upb_sink_endseq(sink, getsel(f, UPB_HANDLER_ENDSEQ));
 }
 
-static void put_ruby_value(VALUE value,
-                           const upb_fielddef *f,
-                           VALUE type_class,
-                           int depth,
-                           upb_sink *sink,
-                           bool emit_defaults,
+static void put_ruby_value(VALUE value, const upb_fielddef* f, VALUE type_class,
+                           int depth, upb_sink sink, bool emit_defaults,
                            bool is_json) {
+  upb_selector_t sel = 0;
+
   if (depth > ENCODE_MAX_NESTING) {
     rb_raise(rb_eRuntimeError,
              "Maximum recursion depth exceeded during encoding.");
   }
 
-  upb_selector_t sel = 0;
   if (upb_fielddef_isprimitive(f)) {
     sel = getsel(f, upb_handlers_getprimitivehandlertype(f));
   }
@@ -1112,8 +1059,8 @@ static void put_ruby_value(VALUE value,
   }
 }
 
-static void putmap(VALUE map, const upb_fielddef *f, upb_sink *sink,
-                   int depth, bool emit_defaults, bool is_json) {
+static void putmap(VALUE map, const upb_fielddef* f, upb_sink sink, int depth,
+                   bool emit_defaults, bool is_json) {
   Map* self;
   upb_sink subsink;
   const upb_fielddef* key_field;
@@ -1137,17 +1084,17 @@ static void putmap(VALUE map, const upb_fielddef *f, upb_sink *sink,
     upb_status status;
 
     upb_sink entry_sink;
-    upb_sink_startsubmsg(&subsink, getsel(f, UPB_HANDLER_STARTSUBMSG),
+    upb_sink_startsubmsg(subsink, getsel(f, UPB_HANDLER_STARTSUBMSG),
                          &entry_sink);
-    upb_sink_startmsg(&entry_sink);
+    upb_sink_startmsg(entry_sink);
 
-    put_ruby_value(key, key_field, Qnil, depth + 1, &entry_sink,
-                   emit_defaults, is_json);
+    put_ruby_value(key, key_field, Qnil, depth + 1, entry_sink, emit_defaults,
+                   is_json);
     put_ruby_value(value, value_field, self->value_type_class, depth + 1,
-                   &entry_sink, emit_defaults, is_json);
+                   entry_sink, emit_defaults, is_json);
 
-    upb_sink_endmsg(&entry_sink, &status);
-    upb_sink_endsubmsg(&subsink, getsel(f, UPB_HANDLER_ENDSUBMSG));
+    upb_sink_endmsg(entry_sink, &status);
+    upb_sink_endsubmsg(subsink, getsel(f, UPB_HANDLER_ENDSUBMSG));
   }
 
   upb_sink_endseq(sink, getsel(f, UPB_HANDLER_ENDSEQ));
@@ -1156,8 +1103,8 @@ static void putmap(VALUE map, const upb_fielddef *f, upb_sink *sink,
 static const upb_handlers* msgdef_json_serialize_handlers(
     Descriptor* desc, bool preserve_proto_fieldnames);
 
-static void putjsonany(VALUE msg_rb, const Descriptor* desc,
-                       upb_sink* sink, int depth, bool emit_defaults) {
+static void putjsonany(VALUE msg_rb, const Descriptor* desc, upb_sink sink,
+                       int depth, bool emit_defaults) {
   upb_status status;
   MessageHeader* msg = NULL;
   const upb_fielddef* type_field = upb_msgdef_itof(desc->msgdef, UPB_ANY_TYPE);
@@ -1204,16 +1151,14 @@ static void putjsonany(VALUE msg_rb, const Descriptor* desc,
   {
     uint32_t value_offset;
     VALUE value_str_rb;
-    const char* value_str;
     size_t value_len;
 
     value_offset = desc->layout->fields[upb_fielddef_index(value_field)].offset;
     value_str_rb = DEREF(Message_data(msg), value_offset, VALUE);
-    value_str = RSTRING_PTR(value_str_rb);
     value_len = RSTRING_LEN(value_str_rb);
 
     if (value_len > 0) {
-      VALUE payload_desc_rb = get_def_obj(payload_type);
+      VALUE payload_desc_rb = get_msgdef_obj(generated_pool, payload_type);
       Descriptor* payload_desc = ruby_to_Descriptor(payload_desc_rb);
       VALUE payload_class = Descriptor_msgclass(payload_desc_rb);
       upb_sink subsink;
@@ -1231,8 +1176,8 @@ static void putjsonany(VALUE msg_rb, const Descriptor* desc,
 
       subsink.handlers =
           msgdef_json_serialize_handlers(payload_desc, true);
-      subsink.closure = sink->closure;
-      putmsg(payload_msg_rb, payload_desc, &subsink, depth, emit_defaults, true,
+      subsink.closure = sink.closure;
+      putmsg(payload_msg_rb, payload_desc, subsink, depth, emit_defaults, true,
              is_wellknown);
     }
   }
@@ -1242,7 +1187,7 @@ static void putjsonany(VALUE msg_rb, const Descriptor* desc,
 
 static void putjsonlistvalue(
     VALUE msg_rb, const Descriptor* desc,
-    upb_sink* sink, int depth, bool emit_defaults) {
+    upb_sink sink, int depth, bool emit_defaults) {
   upb_status status;
   upb_sink subsink;
   MessageHeader* msg = NULL;
@@ -1269,7 +1214,7 @@ static void putjsonlistvalue(
 }
 
 static void putmsg(VALUE msg_rb, const Descriptor* desc,
-                   upb_sink *sink, int depth, bool emit_defaults,
+                   upb_sink sink, int depth, bool emit_defaults,
                    bool is_json, bool open_msg) {
   MessageHeader* msg;
   upb_msg_field_iter i;
@@ -1360,20 +1305,19 @@ static void putmsg(VALUE msg_rb, const Descriptor* desc,
     } else {
       upb_selector_t sel = getsel(f, upb_handlers_getprimitivehandlertype(f));
 
-#define T(upbtypeconst, upbtype, ctype, default_value)                          \
-  case upbtypeconst: {                                                          \
-      ctype value = DEREF(msg, offset, ctype);                                  \
-      bool is_default = false;                                                  \
-      if (upb_fielddef_haspresence(f)) {                                        \
-        is_default = layout_has(desc->layout, Message_data(msg), f) == Qfalse;  \
-      } else if (upb_msgdef_syntax(desc->msgdef) == UPB_SYNTAX_PROTO3) {        \
-        is_default = default_value == value;                                    \
-      }                                                                         \
-      if (is_matching_oneof || emit_defaults || !is_default) {                  \
-        upb_sink_put##upbtype(sink, sel, value);                                \
-      }                                                                         \
-    }                                                                           \
-    break;
+#define T(upbtypeconst, upbtype, ctype, default_value)                       \
+  case upbtypeconst: {                                                       \
+    ctype value = DEREF(msg, offset, ctype);                                 \
+    bool is_default = false;                                                 \
+    if (upb_fielddef_haspresence(f)) {                                       \
+      is_default = layout_has(desc->layout, Message_data(msg), f) == Qfalse; \
+    } else if (upb_msgdef_syntax(desc->msgdef) == UPB_SYNTAX_PROTO3) {       \
+      is_default = default_value == value;                                   \
+    }                                                                        \
+    if (is_matching_oneof || emit_defaults || !is_default) {                 \
+      upb_sink_put##upbtype(sink, sel, value);                               \
+    }                                                                        \
+  } break;
 
       switch (upb_fielddef_type(f)) {
         T(UPB_TYPE_FLOAT,  float,  float, 0.0)
@@ -1395,9 +1339,11 @@ static void putmsg(VALUE msg_rb, const Descriptor* desc,
     }
   }
 
-  stringsink* unknown = msg->unknown_fields;
-  if (unknown != NULL) {
-    upb_sink_putunknown(sink, unknown->ptr, unknown->len);
+  {
+    stringsink* unknown = msg->unknown_fields;
+    if (unknown != NULL) {
+      upb_sink_putunknown(sink, unknown->ptr, unknown->len);
+    }
   }
 
   if (open_msg) {
@@ -1405,33 +1351,6 @@ static void putmsg(VALUE msg_rb, const Descriptor* desc,
   }
 }
 
-static const upb_handlers* msgdef_pb_serialize_handlers(Descriptor* desc) {
-  if (desc->pb_serialize_handlers == NULL) {
-    desc->pb_serialize_handlers =
-        upb_pb_encoder_newhandlers(desc->msgdef, &desc->pb_serialize_handlers);
-  }
-  return desc->pb_serialize_handlers;
-}
-
-static const upb_handlers* msgdef_json_serialize_handlers(
-    Descriptor* desc, bool preserve_proto_fieldnames) {
-  if (preserve_proto_fieldnames) {
-    if (desc->json_serialize_handlers == NULL) {
-      desc->json_serialize_handlers =
-          upb_json_printer_newhandlers(
-              desc->msgdef, true, &desc->json_serialize_handlers);
-    }
-    return desc->json_serialize_handlers;
-  } else {
-    if (desc->json_serialize_handlers_preserve == NULL) {
-      desc->json_serialize_handlers_preserve =
-          upb_json_printer_newhandlers(
-              desc->msgdef, false, &desc->json_serialize_handlers_preserve);
-    }
-    return desc->json_serialize_handlers_preserve;
-  }
-}
-
 /*
  * call-seq:
  *     MessageClass.encode(msg) => bytes
@@ -1454,8 +1373,8 @@ VALUE Message_encode(VALUE klass, VALUE msg_rb) {
     upb_pb_encoder* encoder;
     VALUE ret;
 
-    stackenv_init(&se, "Error occurred during encoding: %s");
-    encoder = upb_pb_encoder_create(&se.env, serialize_handlers, &sink.sink);
+    stackenv_init(&se, "Error occurred during encoding: %" PRIsVALUE);
+    encoder = upb_pb_encoder_create(se.arena, serialize_handlers, sink.sink);
 
     putmsg(msg_rb, desc, upb_pb_encoder_input(encoder), 0, false, false, true);
 
@@ -1512,8 +1431,8 @@ VALUE Message_encode_json(int argc, VALUE* argv, VALUE klass) {
     stackenv se;
     VALUE ret;
 
-    stackenv_init(&se, "Error occurred during encoding: %s");
-    printer = upb_json_printer_create(&se.env, serialize_handlers, &sink.sink);
+    stackenv_init(&se, "Error occurred during encoding: %" PRIsVALUE);
+    printer = upb_json_printer_create(se.arena, serialize_handlers, sink.sink);
 
     putmsg(msg_rb, desc, upb_json_printer_input(printer), 0,
            RTEST(emit_defaults), true, true);
@@ -1533,10 +1452,12 @@ static void discard_unknown(VALUE msg_rb, const Descriptor* desc) {
 
   TypedData_Get_Struct(msg_rb, MessageHeader, &Message_type, msg);
 
-  stringsink* unknown = msg->unknown_fields;
-  if (unknown != NULL) {
-    stringsink_uninit(unknown);
-    msg->unknown_fields = NULL;
+  {
+    stringsink* unknown = msg->unknown_fields;
+    if (unknown != NULL) {
+      stringsink_uninit(unknown);
+      msg->unknown_fields = NULL;
+    }
   }
 
   for (upb_msg_field_begin(&it, desc->msgdef);
@@ -1566,10 +1487,12 @@ static void discard_unknown(VALUE msg_rb, const Descriptor* desc) {
     }
 
     if (is_map_field(f)) {
+      VALUE map;
+      Map_iter map_it;
+
       if (!upb_fielddef_issubmsg(map_field_value(f))) continue;
-      VALUE map = DEREF(msg, offset, VALUE);
+      map = DEREF(msg, offset, VALUE);
       if (map == Qnil) continue;
-      Map_iter map_it;
       for (Map_begin(map, &map_it); !Map_done(&map_it); Map_next(&map_it)) {
         VALUE submsg = Map_iter_value(&map_it);
         VALUE descriptor = rb_ivar_get(submsg, descriptor_instancevar_interned);
@@ -1578,9 +1501,12 @@ static void discard_unknown(VALUE msg_rb, const Descriptor* desc) {
       }
     } else if (upb_fielddef_isseq(f)) {
       VALUE ary = DEREF(msg, offset, VALUE);
+      int size;
+      int i;
+
       if (ary == Qnil) continue;
-      int size = NUM2INT(RepeatedField_length(ary));
-      for (int i = 0; i < size; i++) {
+      size = NUM2INT(RepeatedField_length(ary));
+      for (i = 0; i < size; i++) {
         void* memory = RepeatedField_index_native(ary, i);
         VALUE submsg = *((VALUE *)memory);
         VALUE descriptor = rb_ivar_get(submsg, descriptor_instancevar_interned);
@@ -1589,9 +1515,12 @@ static void discard_unknown(VALUE msg_rb, const Descriptor* desc) {
       }
     } else {
       VALUE submsg = DEREF(msg, offset, VALUE);
+      VALUE descriptor;
+      const Descriptor* subdesc;
+
       if (submsg == Qnil) continue;
-      VALUE descriptor = rb_ivar_get(submsg, descriptor_instancevar_interned);
-      const Descriptor* subdesc = ruby_to_Descriptor(descriptor);
+      descriptor = rb_ivar_get(submsg, descriptor_instancevar_interned);
+      subdesc = ruby_to_Descriptor(descriptor);
       discard_unknown(submsg, subdesc);
     }
   }

ruby/ext/google/protobuf_c/extconf.rb

@@ -3,11 +3,9 @@
 require 'mkmf'
 
 if RUBY_PLATFORM =~ /darwin/ || RUBY_PLATFORM =~ /linux/
-  # XOPEN_SOURCE needed for strptime:
-  # https://stackoverflow.com/questions/35234152/strptime-giving-implicit-declaration-and-undefined-reference
-  $CFLAGS += " -std=c99 -O3 -DNDEBUG -D_XOPEN_SOURCE=700"
+  $CFLAGS += " -std=gnu90 -O3 -DNDEBUG -Wall -Wdeclaration-after-statement -Wsign-compare"
 else
-  $CFLAGS += " -std=c99 -O3 -DNDEBUG"
+  $CFLAGS += " -std=gnu90 -O3 -DNDEBUG"
 end
 
 

ruby/ext/google/protobuf_c/map.c

@@ -389,10 +389,7 @@ VALUE Map_index(VALUE _self, VALUE key) {
  * was just inserted.
  */
 VALUE Map_index_set(VALUE _self, VALUE key, VALUE value) {
-  rb_check_frozen(_self);
-
   Map* self = ruby_to_Map(_self);
-
   char keybuf[TABLE_KEY_BUF_LENGTH];
   const char* keyval = NULL;
   size_t length = 0;
@@ -400,6 +397,8 @@ VALUE Map_index_set(VALUE _self, VALUE key, VALUE value) {
   void* mem;
   key = table_key(self, key, keybuf, &keyval, &length);
 
+  rb_check_frozen(_self);
+
   if (TYPE(value) == T_HASH) {
     VALUE args[1] = { value };
     value = rb_class_new_instance(1, args, self->value_type_class);
@@ -448,16 +447,15 @@ VALUE Map_has_key(VALUE _self, VALUE key) {
  * nil if none was present. Throws an exception if the key is of the wrong type.
  */
 VALUE Map_delete(VALUE _self, VALUE key) {
-  rb_check_frozen(_self);
-
   Map* self = ruby_to_Map(_self);
-
   char keybuf[TABLE_KEY_BUF_LENGTH];
   const char* keyval = NULL;
   size_t length = 0;
   upb_value v;
   key = table_key(self, key, keybuf, &keyval, &length);
 
+  rb_check_frozen(_self);
+
   if (upb_strtable_remove2(&self->table, keyval, length, &v)) {
     void* mem = value_memory(&v);
     return native_slot_get(self->value_type, self->value_type_class, mem);
@@ -473,10 +471,10 @@ VALUE Map_delete(VALUE _self, VALUE key) {
  * Removes all entries from the map.
  */
 VALUE Map_clear(VALUE _self) {
-  rb_check_frozen(_self);
-
   Map* self = ruby_to_Map(_self);
 
+  rb_check_frozen(_self);
+
   // Uninit and reinit the table -- this is faster than iterating and doing a
   // delete-lookup on each key.
   upb_strtable_uninit(&self->table);

ruby/ext/google/protobuf_c/message.c

@@ -60,10 +60,16 @@ rb_data_type_t Message_type = {
 VALUE Message_alloc(VALUE klass) {
   VALUE descriptor = rb_ivar_get(klass, descriptor_instancevar_interned);
   Descriptor* desc = ruby_to_Descriptor(descriptor);
-  MessageHeader* msg = (MessageHeader*)ALLOC_N(
-      uint8_t, sizeof(MessageHeader) + desc->layout->size);
+  MessageHeader* msg;
   VALUE ret;
 
+  if (desc->layout == NULL) {
+    desc->layout = create_layout(desc);
+  }
+
+  msg = (MessageHeader*)ALLOC_N(uint8_t,
+                                sizeof(MessageHeader) + desc->layout->size);
+
   memset(Message_data(msg), 0, desc->layout->size);
 
   // We wrap first so that everything in the message object is GC-rooted in case
@@ -125,8 +131,9 @@ enum {
 };
 
 // Check if the field is a well known wrapper type
-static bool is_wrapper_type_field(const upb_fielddef* field) {
-  char* field_type_name = rb_class2name(field_type_class(field));
+static bool is_wrapper_type_field(const MessageLayout* layout,
+                                  const upb_fielddef* field) {
+  const char* field_type_name = rb_class2name(field_type_class(layout, field));
 
   return strcmp(field_type_name, "Google::Protobuf::DoubleValue") == 0 ||
          strcmp(field_type_name, "Google::Protobuf::FloatValue") == 0 ||
@@ -140,26 +147,34 @@ static bool is_wrapper_type_field(const upb_fielddef* field) {
 }
 
 // Get a new Ruby wrapper type and set the initial value
-static VALUE ruby_wrapper_type(const upb_fielddef* field, const VALUE* value) {
-  if (is_wrapper_type_field(field) && value != Qnil) {
+static VALUE ruby_wrapper_type(const MessageLayout* layout,
+                               const upb_fielddef* field, const VALUE value) {
+  if (is_wrapper_type_field(layout, field) && value != Qnil) {
     VALUE hash = rb_hash_new();
     rb_hash_aset(hash, rb_str_new2("value"), value);
-    VALUE args[1] = { hash };
-    return rb_class_new_instance(1, args, field_type_class(field));
+    {
+      VALUE args[1] = {hash};
+      return rb_class_new_instance(1, args, field_type_class(layout, field));
+    }
   }
   return Qnil;
 }
 
 static int extract_method_call(VALUE method_name, MessageHeader* self,
-			       const upb_fielddef **f, const upb_oneofdef **o) {
-  Check_Type(method_name, T_SYMBOL);
-
-  VALUE method_str = rb_id2str(SYM2ID(method_name));
-  char* name = RSTRING_PTR(method_str);
-  size_t name_len = RSTRING_LEN(method_str);
+                            const upb_fielddef **f, const upb_oneofdef **o) {
+  VALUE method_str;
+  char* name;
+  size_t name_len;
   int accessor_type;
   const upb_oneofdef* test_o;
   const upb_fielddef* test_f;
+  bool has_field;
+
+  Check_Type(method_name, T_SYMBOL);
+
+  method_str = rb_id2str(SYM2ID(method_name));
+  name = RSTRING_PTR(method_str);
+  name_len = RSTRING_LEN(method_str);
 
   if (name[name_len - 1] == '=') {
     accessor_type = METHOD_SETTER;
@@ -168,13 +183,13 @@ static int extract_method_call(VALUE method_name, MessageHeader* self,
     // we don't strip the prefix.
   } else if (strncmp("clear_", name, 6) == 0 &&
              !upb_msgdef_lookupname(self->descriptor->msgdef, name, name_len,
-				    &test_f, &test_o)) {
+                                    &test_f, &test_o)) {
     accessor_type = METHOD_CLEAR;
     name = name + 6;
     name_len = name_len - 6;
   } else if (strncmp("has_", name, 4) == 0 && name[name_len - 1] == '?' &&
              !upb_msgdef_lookupname(self->descriptor->msgdef, name, name_len,
-				    &test_f, &test_o)) {
+                                    &test_f, &test_o)) {
     accessor_type = METHOD_PRESENCE;
     name = name + 4;
     name_len = name_len - 5;
@@ -182,25 +197,26 @@ static int extract_method_call(VALUE method_name, MessageHeader* self,
     accessor_type = METHOD_GETTER;
   }
 
-  bool has_field = upb_msgdef_lookupname(self->descriptor->msgdef, name, name_len,
-			                                   &test_f, &test_o);
+  has_field = upb_msgdef_lookupname(self->descriptor->msgdef, name, name_len,
+                                    &test_f, &test_o);
 
   // Look for wrapper type accessor of the form <field_name>_as_value
   if (!has_field &&
       (accessor_type == METHOD_GETTER || accessor_type == METHOD_SETTER) &&
       name_len > 9 && strncmp(name + name_len - 9, "_as_value", 9) == 0) {
-    // Find the field name
+    const upb_oneofdef* test_o_wrapper;
+    const upb_fielddef* test_f_wrapper;
     char wrapper_field_name[name_len - 8];
+
+    // Find the field name
     strncpy(wrapper_field_name, name, name_len - 9);
     wrapper_field_name[name_len - 9] = '\0';
 
     // Check if field exists and is a wrapper type
-    const upb_oneofdef* test_o_wrapper;
-    const upb_fielddef* test_f_wrapper;
-    if (upb_msgdef_lookupname(self->descriptor->msgdef, wrapper_field_name, name_len - 9,
-			                        &test_f_wrapper, &test_o_wrapper) &&
+    if (upb_msgdef_lookupname(self->descriptor->msgdef, wrapper_field_name,
+                              name_len - 9, &test_f_wrapper, &test_o_wrapper) &&
         upb_fielddef_type(test_f_wrapper) == UPB_TYPE_MESSAGE &&
-        is_wrapper_type_field(test_f_wrapper)) {
+        is_wrapper_type_field(self->descriptor->layout, test_f_wrapper)) {
       // It does exist!
       has_field = true;
       if (accessor_type == METHOD_SETTER) {
@@ -216,17 +232,17 @@ static int extract_method_call(VALUE method_name, MessageHeader* self,
   // Look for enum accessor of the form <enum_name>_const
   if (!has_field && accessor_type == METHOD_GETTER &&
       name_len > 6 && strncmp(name + name_len - 6, "_const", 6) == 0) {
+    const upb_oneofdef* test_o_enum;
+    const upb_fielddef* test_f_enum;
+    char enum_name[name_len - 5];
 
     // Find enum field name
-    char enum_name[name_len - 5];
     strncpy(enum_name, name, name_len - 6);
     enum_name[name_len - 6] = '\0';
 
     // Check if enum field exists
-    const upb_oneofdef* test_o_enum;
-    const upb_fielddef* test_f_enum;
     if (upb_msgdef_lookupname(self->descriptor->msgdef, enum_name, name_len - 6,
-			                        &test_f_enum, &test_o_enum) &&
+                                             &test_f_enum, &test_o_enum) &&
         upb_fielddef_type(test_f_enum) == UPB_TYPE_ENUM) {
       // It does exist!
       has_field = true;
@@ -285,13 +301,14 @@ VALUE Message_method_missing(int argc, VALUE* argv, VALUE _self) {
   MessageHeader* self;
   const upb_oneofdef* o;
   const upb_fielddef* f;
+  int accessor_type;
 
   TypedData_Get_Struct(_self, MessageHeader, &Message_type, self);
   if (argc < 1) {
     rb_raise(rb_eArgError, "Expected method name as first argument.");
   }
 
-  int accessor_type = extract_method_call(argv[0], self, &f, &o);
+  accessor_type = extract_method_call(argv[0], self, &f, &o);
   if (accessor_type == METHOD_UNKNOWN || (o == NULL && f == NULL) ) {
     return rb_call_super(argc, argv);
   } else if (accessor_type == METHOD_SETTER || accessor_type == METHOD_WRAPPER_SETTER) {
@@ -305,11 +322,12 @@ VALUE Message_method_missing(int argc, VALUE* argv, VALUE _self) {
 
   // Return which of the oneof fields are set
   if (o != NULL) {
+    const upb_fielddef* oneof_field = which_oneof_field(self, o);
+
     if (accessor_type == METHOD_SETTER) {
       rb_raise(rb_eRuntimeError, "Oneof accessors are read-only.");
     }
 
-    const upb_fielddef* oneof_field = which_oneof_field(self, o);
     if (accessor_type == METHOD_PRESENCE) {
       return oneof_field == NULL ? Qfalse : Qtrue;
     } else if (accessor_type == METHOD_CLEAR) {
@@ -338,20 +356,21 @@ VALUE Message_method_missing(int argc, VALUE* argv, VALUE _self) {
     }
     return value;
   } else if (accessor_type == METHOD_WRAPPER_SETTER) {
-    VALUE wrapper = ruby_wrapper_type(f, argv[1]);
+    VALUE wrapper = ruby_wrapper_type(self->descriptor->layout, f, argv[1]);
     layout_set(self->descriptor->layout, Message_data(self), f, wrapper);
     return Qnil;
   } else if (accessor_type == METHOD_ENUM_GETTER) {
-    VALUE enum_type = field_type_class(f);
+    VALUE enum_type = field_type_class(self->descriptor->layout, f);
     VALUE method = rb_intern("const_get");
     VALUE raw_value = layout_get(self->descriptor->layout, Message_data(self), f);
 
     // Map repeated fields to a new type with ints
     if (upb_fielddef_label(f) == UPB_LABEL_REPEATED) {
       int array_size = FIX2INT(rb_funcall(raw_value, rb_intern("length"), 0));
+      int i;
       VALUE array_args[1] = { ID2SYM(rb_intern("int64")) };
       VALUE array = rb_class_new_instance(1, array_args, CLASS_OF(raw_value));
-      for (int i = 0; i < array_size; i++) {
+      for (i = 0; i < array_size; i++) {
         VALUE entry = rb_funcall(enum_type, method, 1, rb_funcall(raw_value,
                                  rb_intern("at"), 1, INT2NUM(i)));
         rb_funcall(array, rb_intern("push"), 1, entry);
@@ -370,13 +389,14 @@ VALUE Message_respond_to_missing(int argc, VALUE* argv, VALUE _self) {
   MessageHeader* self;
   const upb_oneofdef* o;
   const upb_fielddef* f;
+  int accessor_type;
 
   TypedData_Get_Struct(_self, MessageHeader, &Message_type, self);
   if (argc < 1) {
     rb_raise(rb_eArgError, "Expected method name as first argument.");
   }
 
-  int accessor_type = extract_method_call(argv[0], self, &f, &o);
+  accessor_type = extract_method_call(argv[0], self, &f, &o);
   if (accessor_type == METHOD_UNKNOWN) {
     return rb_call_super(argc, argv);
   } else if (o != NULL) {
@@ -386,15 +406,10 @@ VALUE Message_respond_to_missing(int argc, VALUE* argv, VALUE _self) {
   }
 }
 
-VALUE create_submsg_from_hash(const upb_fielddef *f, VALUE hash) {
-  const upb_def *d = upb_fielddef_subdef(f);
-  assert(d != NULL);
-
-  VALUE descriptor = get_def_obj(d);
-  VALUE msgclass = rb_funcall(descriptor, rb_intern("msgclass"), 0, NULL);
-
+VALUE create_submsg_from_hash(const MessageLayout* layout,
+                              const upb_fielddef* f, VALUE hash) {
   VALUE args[1] = { hash };
-  return rb_class_new_instance(1, args, msgclass);
+  return rb_class_new_instance(1, args, field_type_class(layout, f));
 }
 
 int Message_initialize_kwarg(VALUE key, VALUE val, VALUE _self) {
@@ -434,6 +449,7 @@ int Message_initialize_kwarg(VALUE key, VALUE val, VALUE _self) {
     Map_merge_into_self(map, val);
   } else if (upb_fielddef_label(f) == UPB_LABEL_REPEATED) {
     VALUE ary;
+    int i;
 
     if (TYPE(val) != T_ARRAY) {
       rb_raise(rb_eArgError,
@@ -441,17 +457,17 @@ int Message_initialize_kwarg(VALUE key, VALUE val, VALUE _self) {
                name, rb_class2name(CLASS_OF(val)));
     }
     ary = layout_get(self->descriptor->layout, Message_data(self), f);
-    for (int i = 0; i < RARRAY_LEN(val); i++) {
+    for (i = 0; i < RARRAY_LEN(val); i++) {
       VALUE entry = rb_ary_entry(val, i);
       if (TYPE(entry) == T_HASH && upb_fielddef_issubmsg(f)) {
-        entry = create_submsg_from_hash(f, entry);
+        entry = create_submsg_from_hash(self->descriptor->layout, f, entry);
       }
 
       RepeatedField_push(ary, entry);
     }
   } else {
     if (TYPE(val) == T_HASH && upb_fielddef_issubmsg(f)) {
-      val = create_submsg_from_hash(f, val);
+      val = create_submsg_from_hash(self->descriptor->layout, f, val);
     }
 
     layout_set(self->descriptor->layout, Message_data(self), f, val);
@@ -608,17 +624,18 @@ VALUE Message_to_h(VALUE _self) {
        !upb_msg_field_done(&it);
        upb_msg_field_next(&it)) {
     const upb_fielddef* field = upb_msg_iter_field(&it);
+    VALUE msg_value;
+    VALUE msg_key;
 
     // For proto2, do not include fields which are not set.
     if (upb_msgdef_syntax(self->descriptor->msgdef) == UPB_SYNTAX_PROTO2 &&
-	field_contains_hasbit(self->descriptor->layout, field) &&
-	!layout_has(self->descriptor->layout, Message_data(self), field)) {
+       field_contains_hasbit(self->descriptor->layout, field) &&
+       !layout_has(self->descriptor->layout, Message_data(self), field)) {
       continue;
     }
 
-    VALUE msg_value = layout_get(self->descriptor->layout, Message_data(self),
-                                 field);
-    VALUE msg_key   = ID2SYM(rb_intern(upb_fielddef_name(field)));
+    msg_value = layout_get(self->descriptor->layout, Message_data(self), field);
+    msg_key = ID2SYM(rb_intern(upb_fielddef_name(field)));
     if (is_map_field(field)) {
       msg_value = Map_to_h(msg_value);
     } else if (upb_fielddef_label(field) == UPB_LABEL_REPEATED) {
@@ -629,7 +646,8 @@ VALUE Message_to_h(VALUE _self) {
       }
 
       if (upb_fielddef_type(field) == UPB_TYPE_MESSAGE) {
-        for (int i = 0; i < RARRAY_LEN(msg_value); i++) {
+        int i;
+        for (i = 0; i < RARRAY_LEN(msg_value); i++) {
           VALUE elem = rb_ary_entry(msg_value, i);
           rb_ary_store(msg_value, i, Message_to_h(elem));
         }
@@ -696,17 +714,11 @@ VALUE Message_descriptor(VALUE klass) {
   return rb_ivar_get(klass, descriptor_instancevar_interned);
 }
 
-VALUE build_class_from_descriptor(Descriptor* desc) {
+VALUE build_class_from_descriptor(VALUE descriptor) {
+  Descriptor* desc = ruby_to_Descriptor(descriptor);
   const char *name;
   VALUE klass;
 
-  if (desc->layout == NULL) {
-    desc->layout = create_layout(desc->msgdef);
-  }
-  if (desc->fill_method == NULL) {
-    desc->fill_method = new_fillmsg_decodermethod(desc, &desc->fill_method);
-  }
-
   name = upb_msgdef_fullname(desc->msgdef);
   if (name == NULL) {
     rb_raise(rb_eRuntimeError, "Descriptor does not have assigned name.");
@@ -717,8 +729,7 @@ VALUE build_class_from_descriptor(Descriptor* desc) {
       // their own toplevel constant class name.
       rb_intern("Message"),
       rb_cObject);
-  rb_ivar_set(klass, descriptor_instancevar_interned,
-              get_def_obj(desc->msgdef));
+  rb_ivar_set(klass, descriptor_instancevar_interned, descriptor);
   rb_define_alloc_func(klass, Message_alloc);
   rb_require("google/protobuf/message_exts");
   rb_include_module(klass, rb_eval_string("::Google::Protobuf::MessageExts"));
@@ -802,7 +813,8 @@ VALUE enum_descriptor(VALUE self) {
   return rb_ivar_get(self, descriptor_instancevar_interned);
 }
 
-VALUE build_module_from_enumdesc(EnumDescriptor* enumdesc) {
+VALUE build_module_from_enumdesc(VALUE _enumdesc) {
+  EnumDescriptor* enumdesc = ruby_to_EnumDescriptor(_enumdesc);
   VALUE mod = rb_define_module_id(
       rb_intern(upb_enumdef_fullname(enumdesc->enumdef)));
 
@@ -823,8 +835,7 @@ VALUE build_module_from_enumdesc(EnumDescriptor* enumdesc) {
   rb_define_singleton_method(mod, "lookup", enum_lookup, 1);
   rb_define_singleton_method(mod, "resolve", enum_resolve, 1);
   rb_define_singleton_method(mod, "descriptor", enum_descriptor, 0);
-  rb_ivar_set(mod, descriptor_instancevar_interned,
-              get_def_obj(enumdesc->enumdef));
+  rb_ivar_set(mod, descriptor_instancevar_interned, _enumdesc);
 
   return mod;
 }

ruby/ext/google/protobuf_c/protobuf.c

@@ -30,26 +30,10 @@
 
 #include "protobuf.h"
 
-// -----------------------------------------------------------------------------
-// Global map from upb {msg,enum}defs to wrapper Descriptor/EnumDescriptor
-// instances.
-// -----------------------------------------------------------------------------
-
-// This is a hash table from def objects (encoded by converting pointers to
-// Ruby integers) to MessageDef/EnumDef instances (as Ruby values).
-VALUE upb_def_to_ruby_obj_map;
-
 VALUE cError;
 VALUE cParseError;
 VALUE cTypeError;
-
-void add_def_obj(const void* def, VALUE value) {
-  rb_hash_aset(upb_def_to_ruby_obj_map, ULL2NUM((intptr_t)def), value);
-}
-
-VALUE get_def_obj(const void* def) {
-  return rb_hash_aref(upb_def_to_ruby_obj_map, ULL2NUM((intptr_t)def));
-}
+VALUE c_only_cookie = Qnil;
 
 static VALUE cached_empty_string = Qnil;
 static VALUE cached_empty_bytes = Qnil;
@@ -142,8 +126,8 @@ void Init_protobuf_c() {
   kRubyStringASCIIEncoding = rb_usascii_encoding();
   kRubyString8bitEncoding = rb_ascii8bit_encoding();
 
-  rb_gc_register_address(&upb_def_to_ruby_obj_map);
-  upb_def_to_ruby_obj_map = rb_hash_new();
+  rb_gc_register_address(&c_only_cookie);
+  c_only_cookie = rb_class_new_instance(0, NULL, rb_cObject);
 
   rb_gc_register_address(&cached_empty_string);
   rb_gc_register_address(&cached_empty_bytes);

ruby/ext/google/protobuf_c/protobuf.h

@@ -107,62 +107,68 @@ typedef struct Builder Builder;
 // -----------------------------------------------------------------------------
 
 struct DescriptorPool {
+  VALUE def_to_descriptor;  // Hash table of def* -> Ruby descriptor.
   upb_symtab* symtab;
+  upb_handlercache* fill_handler_cache;
+  upb_handlercache* pb_serialize_handler_cache;
+  upb_handlercache* json_serialize_handler_cache;
+  upb_handlercache* json_serialize_handler_preserve_cache;
+  upb_pbcodecache* fill_method_cache;
+  upb_json_codecache* json_fill_method_cache;
 };
 
 struct Descriptor {
   const upb_msgdef* msgdef;
   MessageLayout* layout;
-  VALUE klass;  // begins as nil
-  const upb_handlers* fill_handlers;
-  const upb_pbdecodermethod* fill_method;
-  const upb_json_parsermethod* json_fill_method;
-  const upb_handlers* pb_serialize_handlers;
-  const upb_handlers* json_serialize_handlers;
-  const upb_handlers* json_serialize_handlers_preserve;
+  VALUE klass;
+  VALUE descriptor_pool;
 };
 
 struct FileDescriptor {
   const upb_filedef* filedef;
+  VALUE descriptor_pool;  // Owns the upb_filedef.
 };
 
 struct FieldDescriptor {
   const upb_fielddef* fielddef;
+  VALUE descriptor_pool;  // Owns the upb_fielddef.
 };
 
 struct OneofDescriptor {
   const upb_oneofdef* oneofdef;
+  VALUE descriptor_pool;  // Owns the upb_oneofdef.
 };
 
 struct EnumDescriptor {
   const upb_enumdef* enumdef;
   VALUE module;  // begins as nil
+  VALUE descriptor_pool;  // Owns the upb_enumdef.
 };
 
 struct MessageBuilderContext {
-  VALUE descriptor;
-  VALUE builder;
+  google_protobuf_DescriptorProto* msg_proto;
+  VALUE file_builder;
 };
 
 struct OneofBuilderContext {
-  VALUE descriptor;
-  VALUE builder;
+  int oneof_index;
+  VALUE message_builder;
 };
 
 struct EnumBuilderContext {
-  VALUE enumdesc;
+  google_protobuf_EnumDescriptorProto* enum_proto;
+  VALUE file_builder;
 };
 
 struct FileBuilderContext {
-  VALUE pending_list;
-  VALUE file_descriptor;
-  VALUE builder;
+  upb_arena *arena;
+  google_protobuf_FileDescriptorProto* file_proto;
+  VALUE descriptor_pool;
 };
 
 struct Builder {
-  VALUE pending_list;
-  VALUE default_file_descriptor;
-  upb_def** defs;  // used only while finalizing
+  VALUE descriptor_pool;
+  VALUE default_file_builder;
 };
 
 extern VALUE cDescriptorPool;
@@ -191,7 +197,6 @@ void DescriptorPool_free(void* _self);
 VALUE DescriptorPool_alloc(VALUE klass);
 void DescriptorPool_register(VALUE module);
 DescriptorPool* ruby_to_DescriptorPool(VALUE value);
-VALUE DescriptorPool_add(VALUE _self, VALUE def);
 VALUE DescriptorPool_build(int argc, VALUE* argv, VALUE _self);
 VALUE DescriptorPool_lookup(VALUE _self, VALUE name);
 VALUE DescriptorPool_generated_pool(VALUE _self);
@@ -203,13 +208,11 @@ void Descriptor_free(void* _self);
 VALUE Descriptor_alloc(VALUE klass);
 void Descriptor_register(VALUE module);
 Descriptor* ruby_to_Descriptor(VALUE value);
-VALUE Descriptor_initialize(VALUE _self, VALUE file_descriptor_rb);
+VALUE Descriptor_initialize(VALUE _self, VALUE cookie, VALUE descriptor_pool,
+                            VALUE ptr);
 VALUE Descriptor_name(VALUE _self);
-VALUE Descriptor_name_set(VALUE _self, VALUE str);
 VALUE Descriptor_each(VALUE _self);
 VALUE Descriptor_lookup(VALUE _self, VALUE name);
-VALUE Descriptor_add_field(VALUE _self, VALUE obj);
-VALUE Descriptor_add_oneof(VALUE _self, VALUE obj);
 VALUE Descriptor_each_oneof(VALUE _self);
 VALUE Descriptor_lookup_oneof(VALUE _self, VALUE name);
 VALUE Descriptor_msgclass(VALUE _self);
@@ -221,28 +224,24 @@ void FileDescriptor_free(void* _self);
 VALUE FileDescriptor_alloc(VALUE klass);
 void FileDescriptor_register(VALUE module);
 FileDescriptor* ruby_to_FileDescriptor(VALUE value);
-VALUE FileDescriptor_initialize(int argc, VALUE* argv, VALUE _self);
+VALUE FileDescriptor_initialize(VALUE _self, VALUE cookie,
+                                VALUE descriptor_pool, VALUE ptr);
 VALUE FileDescriptor_name(VALUE _self);
 VALUE FileDescriptor_syntax(VALUE _self);
-VALUE FileDescriptor_syntax_set(VALUE _self, VALUE syntax);
 
 void FieldDescriptor_mark(void* _self);
 void FieldDescriptor_free(void* _self);
 VALUE FieldDescriptor_alloc(VALUE klass);
 void FieldDescriptor_register(VALUE module);
 FieldDescriptor* ruby_to_FieldDescriptor(VALUE value);
+VALUE FieldDescriptor_initialize(VALUE _self, VALUE cookie,
+                                 VALUE descriptor_pool, VALUE ptr);
 VALUE FieldDescriptor_name(VALUE _self);
-VALUE FieldDescriptor_name_set(VALUE _self, VALUE str);
 VALUE FieldDescriptor_type(VALUE _self);
-VALUE FieldDescriptor_type_set(VALUE _self, VALUE type);
 VALUE FieldDescriptor_default(VALUE _self);
-VALUE FieldDescriptor_default_set(VALUE _self, VALUE default_value);
 VALUE FieldDescriptor_label(VALUE _self);
-VALUE FieldDescriptor_label_set(VALUE _self, VALUE label);
 VALUE FieldDescriptor_number(VALUE _self);
-VALUE FieldDescriptor_number_set(VALUE _self, VALUE number);
 VALUE FieldDescriptor_submsg_name(VALUE _self);
-VALUE FieldDescriptor_submsg_name_set(VALUE _self, VALUE value);
 VALUE FieldDescriptor_subtype(VALUE _self);
 VALUE FieldDescriptor_has(VALUE _self, VALUE msg_rb);
 VALUE FieldDescriptor_clear(VALUE _self, VALUE msg_rb);
@@ -256,21 +255,20 @@ void OneofDescriptor_free(void* _self);
 VALUE OneofDescriptor_alloc(VALUE klass);
 void OneofDescriptor_register(VALUE module);
 OneofDescriptor* ruby_to_OneofDescriptor(VALUE value);
+VALUE OneofDescriptor_initialize(VALUE _self, VALUE cookie,
+                                 VALUE descriptor_pool, VALUE ptr);
 VALUE OneofDescriptor_name(VALUE _self);
-VALUE OneofDescriptor_name_set(VALUE _self, VALUE value);
-VALUE OneofDescriptor_add_field(VALUE _self, VALUE field);
 VALUE OneofDescriptor_each(VALUE _self, VALUE field);
 
 void EnumDescriptor_mark(void* _self);
 void EnumDescriptor_free(void* _self);
 VALUE EnumDescriptor_alloc(VALUE klass);
+VALUE EnumDescriptor_initialize(VALUE _self, VALUE cookie,
+                                VALUE descriptor_pool, VALUE ptr);
 void EnumDescriptor_register(VALUE module);
 EnumDescriptor* ruby_to_EnumDescriptor(VALUE value);
-VALUE EnumDescriptor_initialize(VALUE _self, VALUE file_descriptor_rb);
 VALUE EnumDescriptor_file_descriptor(VALUE _self);
 VALUE EnumDescriptor_name(VALUE _self);
-VALUE EnumDescriptor_name_set(VALUE _self, VALUE str);
-VALUE EnumDescriptor_add_value(VALUE _self, VALUE name, VALUE number);
 VALUE EnumDescriptor_lookup_name(VALUE _self, VALUE name);
 VALUE EnumDescriptor_lookup_value(VALUE _self, VALUE number);
 VALUE EnumDescriptor_each(VALUE _self);
@@ -283,8 +281,8 @@ VALUE MessageBuilderContext_alloc(VALUE klass);
 void MessageBuilderContext_register(VALUE module);
 MessageBuilderContext* ruby_to_MessageBuilderContext(VALUE value);
 VALUE MessageBuilderContext_initialize(VALUE _self,
-                                       VALUE descriptor,
-                                       VALUE builder);
+                                       VALUE _file_builder,
+                                       VALUE name);
 VALUE MessageBuilderContext_optional(int argc, VALUE* argv, VALUE _self);
 VALUE MessageBuilderContext_required(int argc, VALUE* argv, VALUE _self);
 VALUE MessageBuilderContext_repeated(int argc, VALUE* argv, VALUE _self);
@@ -306,15 +304,20 @@ void EnumBuilderContext_free(void* _self);
 VALUE EnumBuilderContext_alloc(VALUE klass);
 void EnumBuilderContext_register(VALUE module);
 EnumBuilderContext* ruby_to_EnumBuilderContext(VALUE value);
-VALUE EnumBuilderContext_initialize(VALUE _self, VALUE enumdesc);
+VALUE EnumBuilderContext_initialize(VALUE _self, VALUE _file_builder,
+                                    VALUE name);
 VALUE EnumBuilderContext_value(VALUE _self, VALUE name, VALUE number);
 
 void FileBuilderContext_mark(void* _self);
 void FileBuilderContext_free(void* _self);
 VALUE FileBuilderContext_alloc(VALUE klass);
 void FileBuilderContext_register(VALUE module);
-VALUE FileBuilderContext_initialize(VALUE _self, VALUE file_descriptor,
-				    VALUE builder);
+FileBuilderContext* ruby_to_FileBuilderContext(VALUE _self);
+upb_strview FileBuilderContext_strdup(VALUE _self, VALUE rb_str);
+upb_strview FileBuilderContext_strdup_name(VALUE _self, VALUE rb_str);
+upb_strview FileBuilderContext_strdup_sym(VALUE _self, VALUE rb_sym);
+VALUE FileBuilderContext_initialize(VALUE _self, VALUE descriptor_pool,
+                                    VALUE name, VALUE options);
 VALUE FileBuilderContext_add_message(VALUE _self, VALUE name);
 VALUE FileBuilderContext_add_enum(VALUE _self, VALUE name);
 VALUE FileBuilderContext_pending_descriptors(VALUE _self);
@@ -324,7 +327,8 @@ void Builder_free(void* _self);
 VALUE Builder_alloc(VALUE klass);
 void Builder_register(VALUE module);
 Builder* ruby_to_Builder(VALUE value);
-VALUE Builder_initialize(VALUE _self);
+VALUE Builder_build(VALUE _self);
+VALUE Builder_initialize(VALUE _self, VALUE descriptor_pool);
 VALUE Builder_add_file(int argc, VALUE *argv, VALUE _self);
 VALUE Builder_add_message(VALUE _self, VALUE name);
 VALUE Builder_add_enum(VALUE _self, VALUE name);
@@ -368,7 +372,7 @@ extern rb_encoding* kRubyStringUtf8Encoding;
 extern rb_encoding* kRubyStringASCIIEncoding;
 extern rb_encoding* kRubyString8bitEncoding;
 
-VALUE field_type_class(const upb_fielddef* field);
+VALUE field_type_class(const MessageLayout* layout, const upb_fielddef* field);
 
 #define MAP_KEY_FIELD 1
 #define MAP_VALUE_FIELD 2
@@ -501,13 +505,15 @@ struct MessageField {
   size_t hasbit;
 };
 
+// MessageLayout is owned by the enclosing Descriptor, which must outlive us.
 struct MessageLayout {
+  const Descriptor* desc;
   const upb_msgdef* msgdef;
   MessageField* fields;
   size_t size;
 };
 
-MessageLayout* create_layout(const upb_msgdef* msgdef);
+MessageLayout* create_layout(const Descriptor* desc);
 void free_layout(MessageLayout* layout);
 bool field_contains_hasbit(MessageLayout* layout,
                  const upb_fielddef* field);
@@ -556,7 +562,7 @@ struct MessageHeader {
 
 extern rb_data_type_t Message_type;
 
-VALUE build_class_from_descriptor(Descriptor* descriptor);
+VALUE build_class_from_descriptor(VALUE descriptor);
 void* Message_data(void* msg);
 void Message_mark(void* self);
 void Message_free(void* self);
@@ -580,12 +586,14 @@ VALUE Message_encode_json(int argc, VALUE* argv, VALUE klass);
 VALUE Google_Protobuf_discard_unknown(VALUE self, VALUE msg_rb);
 VALUE Google_Protobuf_deep_copy(VALUE self, VALUE obj);
 
-VALUE build_module_from_enumdesc(EnumDescriptor* enumdef);
+VALUE build_module_from_enumdesc(VALUE _enumdesc);
 VALUE enum_lookup(VALUE self, VALUE number);
 VALUE enum_resolve(VALUE self, VALUE sym);
+VALUE enum_descriptor(VALUE self);
 
 const upb_pbdecodermethod *new_fillmsg_decodermethod(
     Descriptor* descriptor, const void *owner);
+void add_handlers_for_message(const void *closure, upb_handlers *h);
 
 // Maximum depth allowed during encoding, to avoid stack overflows due to
 // cycles.
@@ -600,8 +608,11 @@ VALUE get_frozen_string(const char* data, size_t size, bool binary);
 // Global map from upb {msg,enum}defs to wrapper Descriptor/EnumDescriptor
 // instances.
 // -----------------------------------------------------------------------------
-void add_def_obj(const void* def, VALUE value);
-VALUE get_def_obj(const void* def);
+VALUE get_msgdef_obj(VALUE descriptor_pool, const upb_msgdef* def);
+VALUE get_enumdef_obj(VALUE descriptor_pool, const upb_enumdef* def);
+VALUE get_fielddef_obj(VALUE descriptor_pool, const upb_fielddef* def);
+VALUE get_filedef_obj(VALUE descriptor_pool, const upb_filedef* def);
+VALUE get_oneofdef_obj(VALUE descriptor_pool, const upb_oneofdef* def);
 
 // -----------------------------------------------------------------------------
 // Utilities.
@@ -617,4 +628,17 @@ void check_upb_status(const upb_status* status, const char* msg);
 
 extern ID descriptor_instancevar_interned;
 
+// A distinct object that is not accessible from Ruby.  We use this as a
+// constructor argument to enforce that certain objects cannot be created from
+// Ruby.
+extern VALUE c_only_cookie;
+
+#ifdef NDEBUG
+#define UPB_ASSERT(expr) do {} while (false && (expr))
+#else
+#define UPB_ASSERT(expr) assert(expr)
+#endif
+
+#define UPB_UNUSED(var) (void)var
+
 #endif  // __GOOGLE_PROTOBUF_RUBY_PROTOBUF_H__

ruby/ext/google/protobuf_c/repeated_field.c

@@ -64,10 +64,11 @@ VALUE RepeatedField_subarray(VALUE _self, long beg, long len) {
   int element_size = native_slot_size(self->field_type);
   upb_fieldtype_t field_type = self->field_type;
   VALUE field_type_class = self->field_type_class;
-
   size_t off = beg * element_size;
   VALUE ary = rb_ary_new2(len);
-  for (int i = beg; i < beg + len; i++, off += element_size) {
+  int i;
+
+  for (i = beg; i < beg + len; i++, off += element_size) {
     void* mem = ((uint8_t *)self->elements) + off;
     VALUE elem = native_slot_get(field_type, field_type_class, mem);
     rb_ary_push(ary, elem);
@@ -88,9 +89,10 @@ VALUE RepeatedField_each(VALUE _self) {
   upb_fieldtype_t field_type = self->field_type;
   VALUE field_type_class = self->field_type_class;
   int element_size = native_slot_size(field_type);
-
   size_t off = 0;
-  for (int i = 0; i < self->size; i++, off += element_size) {
+  int i;
+
+  for (i = 0; i < self->size; i++, off += element_size) {
     void* memory = (void *) (((uint8_t *)self->elements) + off);
     VALUE val = native_slot_get(field_type, field_type_class, memory);
     rb_yield(val);
@@ -169,8 +171,10 @@ VALUE RepeatedField_index_set(VALUE _self, VALUE _index, VALUE val) {
   if (index >= self->size) {
     upb_fieldtype_t field_type = self->field_type;
     int element_size = native_slot_size(field_type);
+    int i;
+
     RepeatedField_reserve(self, index + 1);
-    for (int i = self->size; i <= index; i++) {
+    for (i = self->size; i <= index; i++) {
       void* elem = RepeatedField_memoryat(self, i, element_size);
       native_slot_init(field_type, elem);
     }
@@ -224,7 +228,8 @@ VALUE RepeatedField_push(VALUE _self, VALUE val) {
 }
 
 VALUE RepeatedField_push_vararg(VALUE _self, VALUE args) {
-  for (int i = 0; i < RARRAY_LEN(args); i++) {
+  int i;
+  for (i = 0; i < RARRAY_LEN(args); i++) {
     RepeatedField_push(_self, rb_ary_entry(args, i));
   }
   return _self;
@@ -285,9 +290,11 @@ VALUE RepeatedField_pop_one(VALUE _self) {
  */
 VALUE RepeatedField_replace(VALUE _self, VALUE list) {
   RepeatedField* self = ruby_to_RepeatedField(_self);
+  int i;
+
   Check_Type(list, T_ARRAY);
   self->size = 0;
-  for (int i = 0; i < RARRAY_LEN(list); i++) {
+  for (i = 0; i < RARRAY_LEN(list); i++) {
     RepeatedField_push(_self, rb_ary_entry(list, i));
   }
   return list;
@@ -344,8 +351,10 @@ VALUE RepeatedField_dup(VALUE _self) {
   upb_fieldtype_t field_type = self->field_type;
   size_t elem_size = native_slot_size(field_type);
   size_t off = 0;
+  int i;
+
   RepeatedField_reserve(new_rptfield_self, self->size);
-  for (int i = 0; i < self->size; i++, off += elem_size) {
+  for (i = 0; i < self->size; i++, off += elem_size) {
     void* to_mem = (uint8_t *)new_rptfield_self->elements + off;
     void* from_mem = (uint8_t *)self->elements + off;
     native_slot_dup(field_type, to_mem, from_mem);
@@ -363,8 +372,10 @@ VALUE RepeatedField_deep_copy(VALUE _self) {
   upb_fieldtype_t field_type = self->field_type;
   size_t elem_size = native_slot_size(field_type);
   size_t off = 0;
+  int i;
+
   RepeatedField_reserve(new_rptfield_self, self->size);
-  for (int i = 0; i < self->size; i++, off += elem_size) {
+  for (i = 0; i < self->size; i++, off += elem_size) {
     void* to_mem = (uint8_t *)new_rptfield_self->elements + off;
     void* from_mem = (uint8_t *)self->elements + off;
     native_slot_deep_copy(field_type, to_mem, from_mem);
@@ -384,11 +395,12 @@ VALUE RepeatedField_deep_copy(VALUE _self) {
 VALUE RepeatedField_to_ary(VALUE _self) {
   RepeatedField* self = ruby_to_RepeatedField(_self);
   upb_fieldtype_t field_type = self->field_type;
-
   size_t elem_size = native_slot_size(field_type);
   size_t off = 0;
   VALUE ary = rb_ary_new2(self->size);
-  for (int i = 0; i < self->size; i++, off += elem_size) {
+  int i;
+
+  for (i = 0; i < self->size; i++, off += elem_size) {
     void* mem = ((uint8_t *)self->elements) + off;
     VALUE elem = native_slot_get(field_type, self->field_type_class, mem);
     rb_ary_push(ary, elem);
@@ -434,7 +446,9 @@ VALUE RepeatedField_eq(VALUE _self, VALUE _other) {
     upb_fieldtype_t field_type = self->field_type;
     size_t elem_size = native_slot_size(field_type);
     size_t off = 0;
-    for (int i = 0; i < self->size; i++, off += elem_size) {
+    int i;
+
+    for (i = 0; i < self->size; i++, off += elem_size) {
       void* self_mem = ((uint8_t *)self->elements) + off;
       void* other_mem = ((uint8_t *)other->elements) + off;
       if (!native_slot_eq(field_type, self_mem, other_mem)) {
@@ -459,7 +473,9 @@ VALUE RepeatedField_hash(VALUE _self) {
   VALUE field_type_class = self->field_type_class;
   size_t elem_size = native_slot_size(field_type);
   size_t off = 0;
-  for (int i = 0; i < self->size; i++, off += elem_size) {
+  int i;
+
+  for (i = 0; i < self->size; i++, off += elem_size) {
     void* mem = ((uint8_t *)self->elements) + off;
     VALUE elem = native_slot_get(field_type, field_type_class, mem);
     h = rb_hash_uint(h, NUM2LONG(rb_funcall(elem, hash_sym, 0)));
@@ -481,7 +497,8 @@ VALUE RepeatedField_plus(VALUE _self, VALUE list) {
   VALUE dupped = RepeatedField_dup(_self);
 
   if (TYPE(list) == T_ARRAY) {
-    for (int i = 0; i < RARRAY_LEN(list); i++) {
+    int i;
+    for (i = 0; i < RARRAY_LEN(list); i++) {
       VALUE elem = rb_ary_entry(list, i);
       RepeatedField_push(dupped, elem);
     }
@@ -489,12 +506,14 @@ VALUE RepeatedField_plus(VALUE _self, VALUE list) {
              RTYPEDDATA_TYPE(list) == &RepeatedField_type) {
     RepeatedField* self = ruby_to_RepeatedField(_self);
     RepeatedField* list_rptfield = ruby_to_RepeatedField(list);
+    int i;
+
     if (self->field_type != list_rptfield->field_type ||
         self->field_type_class != list_rptfield->field_type_class) {
       rb_raise(rb_eArgError,
                "Attempt to append RepeatedField with different element type.");
     }
-    for (int i = 0; i < list_rptfield->size; i++) {
+    for (i = 0; i < list_rptfield->size; i++) {
       void* mem = RepeatedField_index_native(list, i);
       RepeatedField_push_native(dupped, mem);
     }
@@ -512,8 +531,10 @@ VALUE RepeatedField_plus(VALUE _self, VALUE list) {
  * concats the passed in array to self.  Returns a Ruby array.
  */
 VALUE RepeatedField_concat(VALUE _self, VALUE list) {
+  int i;
+
   Check_Type(list, T_ARRAY);
-  for (int i = 0; i < RARRAY_LEN(list); i++) {
+  for (i = 0; i < RARRAY_LEN(list); i++) {
     RepeatedField_push(_self, rb_ary_entry(list, i));
   }
   return _self;
@@ -574,10 +595,12 @@ void RepeatedField_init_args(int argc, VALUE* argv,
   }
 
   if (ary != Qnil) {
+    int i;
+
     if (!RB_TYPE_P(ary, T_ARRAY)) {
       rb_raise(rb_eArgError, "Expected array as initialize argument");
     }
-    for (int i = 0; i < RARRAY_LEN(ary); i++) {
+    for (i = 0; i < RARRAY_LEN(ary); i++) {
       RepeatedField_push(_self, rb_ary_entry(ary, i));
     }
   }
@@ -589,8 +612,10 @@ void RepeatedField_mark(void* _self) {
   RepeatedField* self = (RepeatedField*)_self;
   upb_fieldtype_t field_type = self->field_type;
   int element_size = native_slot_size(field_type);
+  int i;
+
   rb_gc_mark(self->field_type_class);
-  for (int i = 0; i < self->size; i++) {
+  for (i = 0; i < self->size; i++) {
     void* memory = (((uint8_t *)self->elements) + i * element_size);
     native_slot_mark(self->field_type, memory);
   }

ruby/ext/google/protobuf_c/storage.c

@@ -181,32 +181,43 @@ void native_slot_set_value_and_case(const char* name,
         value = Qnil;
       } else if (CLASS_OF(value) != type_class) {
         // check for possible implicit conversions
-        VALUE converted_value = NULL;
-        char* field_type_name = rb_class2name(type_class);
+        VALUE converted_value = Qnil;
+        const char* field_type_name = rb_class2name(type_class);
 
         if (strcmp(field_type_name, "Google::Protobuf::Timestamp") == 0 &&
             rb_obj_is_kind_of(value, rb_cTime)) {
           // Time -> Google::Protobuf::Timestamp
           VALUE hash = rb_hash_new();
-          rb_hash_aset(hash, rb_str_new2("seconds"), rb_funcall(value, rb_intern("to_i"), 0));
-          rb_hash_aset(hash, rb_str_new2("nanos"), rb_funcall(value, rb_intern("nsec"), 0));
-          VALUE args[1] = { hash };
-          converted_value = rb_class_new_instance(1, args, type_class);
+          rb_hash_aset(hash, rb_str_new2("seconds"),
+                       rb_funcall(value, rb_intern("to_i"), 0));
+          rb_hash_aset(hash, rb_str_new2("nanos"),
+                       rb_funcall(value, rb_intern("nsec"), 0));
+          {
+            VALUE args[1] = {hash};
+            converted_value = rb_class_new_instance(1, args, type_class);
+          }
         } else if (strcmp(field_type_name, "Google::Protobuf::Duration") == 0 &&
                    rb_obj_is_kind_of(value, rb_cNumeric)) {
           // Numeric -> Google::Protobuf::Duration
           VALUE hash = rb_hash_new();
-          rb_hash_aset(hash, rb_str_new2("seconds"), rb_funcall(value, rb_intern("to_i"), 0));
-          VALUE n_value = rb_funcall(value, rb_intern("remainder"), 1, INT2NUM(1));
-          n_value = rb_funcall(n_value, rb_intern("*"), 1, INT2NUM(1000000000));
-          n_value = rb_funcall(n_value, rb_intern("round"), 0);
-          rb_hash_aset(hash, rb_str_new2("nanos"), n_value);
-          VALUE args[1] = { hash };
-          converted_value = rb_class_new_instance(1, args, type_class);
+          rb_hash_aset(hash, rb_str_new2("seconds"),
+                       rb_funcall(value, rb_intern("to_i"), 0));
+          {
+            VALUE n_value =
+                rb_funcall(value, rb_intern("remainder"), 1, INT2NUM(1));
+            n_value =
+                rb_funcall(n_value, rb_intern("*"), 1, INT2NUM(1000000000));
+            n_value = rb_funcall(n_value, rb_intern("round"), 0);
+            rb_hash_aset(hash, rb_str_new2("nanos"), n_value);
+          }
+          {
+            VALUE args[1] = { hash };
+            converted_value = rb_class_new_instance(1, args, type_class);
+          }
         }
 
         // raise if no suitable conversaion could be found
-        if (converted_value == NULL) {
+        if (converted_value == Qnil) {
           rb_raise(cTypeError,
                    "Invalid type %s to assign to submessage field '%s'.",
                   rb_class2name(CLASS_OF(value)), name);
@@ -462,16 +473,18 @@ static size_t align_up_to(size_t offset, size_t granularity) {
   return (offset + granularity - 1) & ~(granularity - 1);
 }
 
-MessageLayout* create_layout(const upb_msgdef* msgdef) {
+MessageLayout* create_layout(const Descriptor* desc) {
+  const upb_msgdef *msgdef = desc->msgdef;
   MessageLayout* layout = ALLOC(MessageLayout);
   int nfields = upb_msgdef_numfields(msgdef);
   upb_msg_field_iter it;
   upb_msg_oneof_iter oit;
   size_t off = 0;
+  size_t hasbit = 0;
 
+  layout->desc = desc;
   layout->fields = ALLOC_N(MessageField, nfields);
 
-  size_t hasbit = 0;
   for (upb_msg_field_begin(&it, msgdef);
        !upb_msg_field_done(&it);
        upb_msg_field_next(&it)) {
@@ -480,7 +493,7 @@ MessageLayout* create_layout(const upb_msgdef* msgdef) {
       layout->fields[upb_fielddef_index(field)].hasbit = hasbit++;
     } else {
       layout->fields[upb_fielddef_index(field)].hasbit =
-	  MESSAGE_FIELD_NO_HASBIT;
+          MESSAGE_FIELD_NO_HASBIT;
     }
   }
 
@@ -569,28 +582,25 @@ MessageLayout* create_layout(const upb_msgdef* msgdef) {
   }
 
   layout->size = off;
-
   layout->msgdef = msgdef;
-  upb_msgdef_ref(layout->msgdef, &layout->msgdef);
 
   return layout;
 }
 
 void free_layout(MessageLayout* layout) {
   xfree(layout->fields);
-  upb_msgdef_unref(layout->msgdef, &layout->msgdef);
   xfree(layout);
 }
 
-VALUE field_type_class(const upb_fielddef* field) {
+VALUE field_type_class(const MessageLayout* layout, const upb_fielddef* field) {
   VALUE type_class = Qnil;
   if (upb_fielddef_type(field) == UPB_TYPE_MESSAGE) {
-    VALUE submsgdesc =
-        get_def_obj(upb_fielddef_subdef(field));
+    VALUE submsgdesc = get_msgdef_obj(layout->desc->descriptor_pool,
+                                      upb_fielddef_msgsubdef(field));
     type_class = Descriptor_msgclass(submsgdesc);
   } else if (upb_fielddef_type(field) == UPB_TYPE_ENUM) {
-    VALUE subenumdesc =
-        get_def_obj(upb_fielddef_subdef(field));
+    VALUE subenumdesc = get_enumdef_obj(layout->desc->descriptor_pool,
+                                        upb_fielddef_enumsubdef(field));
     type_class = EnumDescriptor_enummodule(subenumdesc);
   }
   return type_class;
@@ -664,7 +674,7 @@ void layout_clear(MessageLayout* layout,
 
     const upb_fielddef* key_field = map_field_key(field);
     const upb_fielddef* value_field = map_field_value(field);
-    VALUE type_class = field_type_class(value_field);
+    VALUE type_class = field_type_class(layout, value_field);
 
     if (type_class != Qnil) {
       VALUE args[3] = {
@@ -685,7 +695,7 @@ void layout_clear(MessageLayout* layout,
   } else if (upb_fielddef_label(field) == UPB_LABEL_REPEATED) {
     VALUE ary = Qnil;
 
-    VALUE type_class = field_type_class(field);
+    VALUE type_class = field_type_class(layout, field);
 
     if (type_class != Qnil) {
       VALUE args[2] = {
@@ -700,9 +710,9 @@ void layout_clear(MessageLayout* layout,
 
     DEREF(memory, VALUE) = ary;
   } else {
-    native_slot_set(upb_fielddef_name(field),
-                    upb_fielddef_type(field), field_type_class(field),
-                    memory, layout_get_default(field));
+    native_slot_set(upb_fielddef_name(field), upb_fielddef_type(field),
+                    field_type_class(layout, field), memory,
+                    layout_get_default(field));
   }
 }
 
@@ -756,20 +766,19 @@ VALUE layout_get(MessageLayout* layout,
       return layout_get_default(field);
     }
     return native_slot_get(upb_fielddef_type(field),
-                           field_type_class(field),
-                           memory);
+                           field_type_class(layout, field), memory);
   } else if (upb_fielddef_label(field) == UPB_LABEL_REPEATED) {
     return *((VALUE *)memory);
   } else if (!field_set) {
     return layout_get_default(field);
   } else {
     return native_slot_get(upb_fielddef_type(field),
-                           field_type_class(field),
-                           memory);
+                           field_type_class(layout, field), memory);
   }
 }
 
-static void check_repeated_field_type(VALUE val, const upb_fielddef* field) {
+static void check_repeated_field_type(const MessageLayout* layout, VALUE val,
+                                      const upb_fielddef* field) {
   RepeatedField* self;
   assert(upb_fielddef_label(field) == UPB_LABEL_REPEATED);
 
@@ -783,25 +792,13 @@ static void check_repeated_field_type(VALUE val, const upb_fielddef* field) {
     rb_raise(cTypeError, "Repeated field array has wrong element type");
   }
 
-  if (self->field_type == UPB_TYPE_MESSAGE) {
-    if (self->field_type_class !=
-        Descriptor_msgclass(get_def_obj(upb_fielddef_subdef(field)))) {
-      rb_raise(cTypeError,
-               "Repeated field array has wrong message class");
-    }
-  }
-
-
-  if (self->field_type == UPB_TYPE_ENUM) {
-    if (self->field_type_class !=
-        EnumDescriptor_enummodule(get_def_obj(upb_fielddef_subdef(field)))) {
-      rb_raise(cTypeError,
-               "Repeated field array has wrong enum class");
-    }
+  if (self->field_type_class != field_type_class(layout, field)) {
+    rb_raise(cTypeError, "Repeated field array has wrong message/enum class");
   }
 }
 
-static void check_map_field_type(VALUE val, const upb_fielddef* field) {
+static void check_map_field_type(const MessageLayout* layout, VALUE val,
+                                 const upb_fielddef* field) {
   const upb_fielddef* key_field = map_field_key(field);
   const upb_fielddef* value_field = map_field_value(field);
   Map* self;
@@ -818,17 +815,11 @@ static void check_map_field_type(VALUE val, const upb_fielddef* field) {
   if (self->value_type != upb_fielddef_type(value_field)) {
     rb_raise(cTypeError, "Map value type does not match field's value type");
   }
-  if (upb_fielddef_type(value_field) == UPB_TYPE_MESSAGE ||
-      upb_fielddef_type(value_field) == UPB_TYPE_ENUM) {
-    if (self->value_type_class !=
-        get_def_obj(upb_fielddef_subdef(value_field))) {
-      rb_raise(cTypeError,
-               "Map value type has wrong message/enum class");
-    }
+  if (self->value_type_class != field_type_class(layout, value_field)) {
+    rb_raise(cTypeError, "Map value type has wrong message/enum class");
   }
 }
 
-
 void layout_set(MessageLayout* layout,
                 void* storage,
                 const upb_fielddef* field,
@@ -856,20 +847,19 @@ void layout_set(MessageLayout* layout,
       // and case number are altered atomically (w.r.t. the Ruby VM).
       native_slot_set_value_and_case(
           upb_fielddef_name(field),
-          upb_fielddef_type(field), field_type_class(field),
+          upb_fielddef_type(field), field_type_class(layout, field),
           memory, val,
           oneof_case, upb_fielddef_number(field));
     }
   } else if (is_map_field(field)) {
-    check_map_field_type(val, field);
+    check_map_field_type(layout, val, field);
     DEREF(memory, VALUE) = val;
   } else if (upb_fielddef_label(field) == UPB_LABEL_REPEATED) {
-    check_repeated_field_type(val, field);
+    check_repeated_field_type(layout, val, field);
     DEREF(memory, VALUE) = val;
   } else {
-    native_slot_set(upb_fielddef_name(field),
-                    upb_fielddef_type(field), field_type_class(field),
-                    memory, val);
+    native_slot_set(upb_fielddef_name(field), upb_fielddef_type(field),
+                    field_type_class(layout, field), memory, val);
   }
 
   if (layout->fields[upb_fielddef_index(field)].hasbit !=

ruby/lib/google/protobuf.rb

@@ -50,6 +50,72 @@ else
   rescue LoadError
     require 'google/protobuf_c'
   end
+
+  module Google
+    module Protobuf
+      module Internal
+        def self.infer_package(names)
+          # Package is longest common prefix ending in '.', if any.
+          min, max = names.minmax
+          last_common_dot = nil
+          min.size.times { |i|
+            if min[i] != max[i] then break end
+            if min[i] == ?. then last_common_dot = i end
+          }
+          if last_common_dot
+            return min.slice(0, last_common_dot)
+          end
+        end
+
+        class NestingBuilder
+          def initialize(msg_names, enum_names)
+            @to_pos = {nil=>nil}
+            @msg_children = Hash.new { |hash, key| hash[key] = [] }
+            @enum_children = Hash.new { |hash, key| hash[key] = [] }
+
+            msg_names.each_with_index { |name, idx| @to_pos[name] = idx }
+            enum_names.each_with_index { |name, idx| @to_pos[name] = idx }
+
+            msg_names.each { |name| @msg_children[parent(name)] << name }
+            enum_names.each { |name| @enum_children[parent(name)] << name }
+          end
+
+          def build(package)
+            return build_msg(package)
+          end
+
+          private
+          def build_msg(msg)
+            return {
+              :pos => @to_pos[msg],
+              :msgs => @msg_children[msg].map { |child| build_msg(child) },
+              :enums => @enum_children[msg].map { |child| @to_pos[child] },
+            }
+          end
+
+          private
+          def parent(name)
+            idx = name.rindex(?.)
+            if idx
+              return name.slice(0, idx)
+            else
+              return nil
+            end
+          end
+        end
+
+        def self.fixup_descriptor(package, msg_names, enum_names)
+          if package.nil?
+            package = self.infer_package(msg_names + enum_names)
+          end
+
+          nesting = NestingBuilder.new(msg_names, enum_names).build(package)
+
+          return package, nesting
+        end
+      end
+    end
+  end
 end
 
 require 'google/protobuf/repeated_field'

ruby/tests/basic.rb

@@ -17,7 +17,6 @@ module BasicTest
     add_message "BadFieldNames" do
       optional :dup, :int32, 1
       optional :class, :int32, 2
-      optional :"a.b", :int32, 3
     end
   end
 
@@ -285,6 +284,14 @@ module BasicTest
       assert_match(/No such field: not_in_message/, e.message)
     end
 
+    #def test_json_quoted_string
+    #  m = TestMessage.decode_json(%q(
+    #    "optionalInt64": "1",,
+    #  }))
+    #  puts(m)
+    #  assert_equal 1, m.optional_int32
+    #end
+
     def test_to_h
       m = TestMessage.new(:optional_bool => true, :optional_double => -10.100001, :optional_string => 'foo', :repeated_string => ['bar1', 'bar2'], :repeated_msg => [TestMessage2.new(:foo => 100)])
       expected_result = {
@@ -371,11 +378,6 @@ module BasicTest
       assert nil != file_descriptor
       assert_equal "tests/basic_test.proto", file_descriptor.name
       assert_equal :proto3, file_descriptor.syntax
-
-      file_descriptor = BadFieldNames.descriptor.file_descriptor
-      assert nil != file_descriptor
-      assert_equal nil, file_descriptor.name
-      assert_equal :proto3, file_descriptor.syntax
     end
 
     # Ruby 2.5 changed to raise FrozenError instead of RuntimeError

ruby/tests/basic_proto2.rb

@@ -18,7 +18,6 @@ module BasicTestProto2
       add_message "BadFieldNames" do
         optional :dup, :int32, 1
         optional :class, :int32, 2
-        optional :"a.b", :int32, 3
       end
     end
   end

ruby/tests/common_tests.rb

@@ -807,7 +807,7 @@ module CommonTests
                                                         proto_module::TestMessage2.new(:foo => 2)])
     data = proto_module::TestMessage.encode m
     m2 = proto_module::TestMessage.decode data
-    assert m == m2
+    assert_equal m, m2
 
     data = Google::Protobuf.encode m
     m2 = Google::Protobuf.decode(proto_module::TestMessage, data)
@@ -902,8 +902,6 @@ module CommonTests
     assert m['class'] == 2
     m['dup'] = 3
     assert m['dup'] == 3
-    m['a.b'] = 4
-    assert m['a.b'] == 4
   end
 
   def test_int_ranges
@@ -1084,9 +1082,7 @@ module CommonTests
 
     json_text = proto_module::TestMessage.encode_json(m)
     m2 = proto_module::TestMessage.decode_json(json_text)
-    puts m.inspect
-    puts m2.inspect
-    assert m == m2
+    assert_equal m, m2
 
     # Crash case from GitHub issue 283.
     bar = proto_module::Bar.new(msg: "bar")
@@ -1132,7 +1128,7 @@ module CommonTests
 
     actual = proto_module::TestMessage.encode_json(m, :emit_defaults => true)
 
-    assert JSON.parse(actual, :symbolize_names => true) == expected
+    assert_equal expected, JSON.parse(actual, :symbolize_names => true)
   end
 
   def test_json_emit_defaults_submsg
@@ -1167,7 +1163,7 @@ module CommonTests
 
     actual = proto_module::TestMessage.encode_json(m, :emit_defaults => true)
 
-    assert JSON.parse(actual, :symbolize_names => true) == expected
+    assert_equal expected, JSON.parse(actual, :symbolize_names => true)
   end
 
   def test_json_emit_defaults_repeated_submsg
@@ -1201,7 +1197,7 @@ module CommonTests
 
     actual = proto_module::TestMessage.encode_json(m, :emit_defaults => true)
 
-    assert JSON.parse(actual, :symbolize_names => true) == expected
+    assert_equal expected, JSON.parse(actual, :symbolize_names => true)
   end
 
   def value_from_ruby(value)