Merge remote-tracking branch 'upstream/master' into py2_py3_straddle

cmake/CMakeLists.txt

@@ -1,13 +1,42 @@
+# Minimum CMake required
 cmake_minimum_required(VERSION 2.8)
 
+# Project
 project(protobuf C CXX)
 
+# Options
 option(BUILD_TESTING "Build tests" ON)
 option(BUILD_SHARED_LIBS "Build Shared Libraries" OFF)
 if (MSVC)
   option(ZLIB "Build with zlib support" OFF)
 endif (MSVC)
 
+# Path to main configure script
+set(protobuf_CONFIGURE_SCRIPT "../configure.ac")
+
+# Parse version from configure script
+file(STRINGS "${protobuf_CONFIGURE_SCRIPT}" protobuf_VERSION_LINE
+  LIMIT_COUNT 1
+  REGEX "^AC_INIT")
+# Replace special characters
+string(REPLACE "(" "_" protobuf_VERSION_LINE ${protobuf_VERSION_LINE})
+string(REPLACE ")" "_" protobuf_VERSION_LINE ${protobuf_VERSION_LINE})
+string(REPLACE "[" "_" protobuf_VERSION_LINE ${protobuf_VERSION_LINE})
+string(REPLACE "]" "_" protobuf_VERSION_LINE ${protobuf_VERSION_LINE})
+# Parse version string
+string(REGEX REPLACE "^AC_INIT__Protocol Buffers_,_([^_]+).*$" "\\1"
+    protobuf_VERSION_STRING "${protobuf_VERSION_LINE}")
+# Parse version tweaks
+string(REGEX REPLACE "^([0-9]+)\\.([0-9]+)\\.([0-9]+).*$" "\\1"
+  protobuf_VERSION_MAJOR "${protobuf_VERSION_STRING}")
+string(REGEX REPLACE "^([0-9]+)\\.([0-9]+)\\.([0-9]+).*$" "\\2"
+  protobuf_VERSION_MINOR "${protobuf_VERSION_STRING}")
+string(REGEX REPLACE "^([0-9]+)\\.([0-9]+)\\.([0-9]+).*$" "\\3"
+  protobuf_VERSION_PATCH "${protobuf_VERSION_STRING}")
+# Package version
+set(protobuf_VERSION
+  "${protobuf_VERSION_MAJOR}.${protobuf_VERSION_MINOR}.${protobuf_VERSION_PATCH}")
+
 add_definitions(-DGOOGLE_PROTOBUF_CMAKE_BUILD)
 
 find_package(Threads REQUIRED)
@@ -86,6 +115,9 @@ include(libprotobuf-lite.cmake)
 include(libprotobuf.cmake)
 include(libprotoc.cmake)
 include(protoc.cmake)
+
 if (BUILD_TESTING)
   include(tests.cmake)
 endif (BUILD_TESTING)
+
+include(install.cmake)

cmake/install.cmake

@@ -0,0 +1,103 @@
+include(GNUInstallDirs)
+
+foreach(_library
+  libprotobuf-lite
+  libprotobuf
+  libprotoc)
+  set_property(TARGET ${_library}
+    PROPERTY INTERFACE_INCLUDE_DIRECTORIES
+    $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>)
+  install(TARGETS ${_library} EXPORT protobuf-targets
+    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT ${_library}
+    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT ${_library}
+    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT ${_library})
+endforeach()
+
+install(TARGETS protoc EXPORT protobuf-targets
+  RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT protoc)
+
+if(TRUE)
+  file(STRINGS extract_includes.bat.in _extract_strings
+    REGEX "^copy")
+  foreach(_extract_string ${_extract_strings})
+    string(REPLACE "copy \${PROTOBUF_SOURCE_WIN32_PATH}\\" ""
+      _extract_string ${_extract_string})
+    string(REPLACE "\\" "/" _extract_string ${_extract_string})
+    string(REGEX MATCH "^[^ ]+"
+      _extract_from ${_extract_string})
+    string(REGEX REPLACE "^${_extract_from} ([^$]+)" "\\1"
+      _extract_to ${_extract_string})
+    get_filename_component(_extract_from "${protobuf_SOURCE_DIR}/${_extract_from}" ABSOLUTE)
+    get_filename_component(_extract_name ${_extract_to} NAME)
+    get_filename_component(_extract_to ${_extract_to} PATH)
+    string(REPLACE "include/" "${CMAKE_INSTALL_INCLUDEDIR}/"
+      _extract_to "${_extract_to}")
+    if(EXISTS "${_extract_from}")
+      install(FILES "${_extract_from}"
+        DESTINATION "${_extract_to}"
+        COMPONENT protobuf-headers
+        RENAME "${_extract_name}")
+    else()
+      message(AUTHOR_WARNING "The file \"${_extract_from}\" is listed in "
+        "\"${protobuf_SOURCE_DIR}/cmake/extract_includes.bat.in\" "
+        "but there not exists. The file will not be installed.")
+    endif()
+  endforeach()
+endif()
+
+# Internal function for parsing auto tools scripts
+function(_protobuf_auto_list FILE_NAME VARIABLE)
+  file(STRINGS ${FILE_NAME} _strings)
+  set(_list)
+  foreach(_string ${_strings})
+    set(_found)
+    string(REGEX MATCH "^[ \t]*${VARIABLE}[ \t]*=[ \t]*" _found "${_string}")
+    if(_found)
+      string(LENGTH "${_found}" _length)
+      string(SUBSTRING "${_string}" ${_length} -1 _draft_list)
+      foreach(_item ${_draft_list})
+        string(STRIP "${_item}" _item)
+        list(APPEND _list "${_item}")
+      endforeach()
+    endif()
+  endforeach()
+  set(${VARIABLE} ${_list} PARENT_SCOPE)
+endfunction()
+
+# Install well-known type proto files
+_protobuf_auto_list("../src/Makefile.am" nobase_dist_proto_DATA)
+foreach(_file ${nobase_dist_proto_DATA})
+  get_filename_component(_file_from "../src/${_file}" ABSOLUTE)
+  get_filename_component(_file_name ${_file} NAME)
+  get_filename_component(_file_path ${_file} PATH)
+  if(EXISTS "${_file_from}")
+    install(FILES "${_file_from}"
+      DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/${_file_path}"
+      COMPONENT protobuf-protos
+      RENAME "${_file_name}")
+  else()
+    message(AUTHOR_WARNING "The file \"${_file_from}\" is listed in "
+      "\"${protobuf_SOURCE_DIR}/../src/Makefile.am\" as nobase_dist_proto_DATA "
+      "but there not exists. The file will not be installed.")
+  endif()
+endforeach()
+
+# Export configuration
+
+install(EXPORT protobuf-targets
+  DESTINATION "lib/cmake/protobuf"
+  COMPONENT protobuf-export)
+
+configure_file(protobuf-config.cmake.in
+  protobuf-config.cmake @ONLY)
+configure_file(protobuf-config-version.cmake.in
+  protobuf-config-version.cmake @ONLY)
+configure_file(protobuf-module.cmake.in
+  protobuf-module.cmake @ONLY)
+
+install(FILES
+  "${protobuf_BINARY_DIR}/protobuf-config.cmake"
+  "${protobuf_BINARY_DIR}/protobuf-config-version.cmake"
+  "${protobuf_BINARY_DIR}/protobuf-module.cmake"
+  DESTINATION "lib/cmake/protobuf"
+  COMPONENT protobuf-export)

cmake/protobuf-config-version.cmake.in

@@ -0,0 +1 @@
+set(PACKAGE_VERSION @protobuf_VERSION@)

cmake/protobuf-config.cmake.in

@@ -0,0 +1,27 @@
+# Version info variables
+set(PROTOBUF_VERSION        "@protobuf_VERSION@")
+set(PROTOBUF_VERSION_STRING "@protobuf_VERSION_STRING@")
+
+# Current dir
+get_filename_component(_PROTOBUF_PACKAGE_PREFIX
+  "${CMAKE_CURRENT_LIST_FILE}" PATH)
+
+# Imported targets
+include("${_PROTOBUF_PACKAGE_PREFIX}/protobuf-targets.cmake")
+
+# Compute the installation prefix relative to this file.
+get_filename_component(_PROTOBUF_IMPORT_PREFIX
+  "${_PROTOBUF_PACKAGE_PREFIX}" PATH)
+get_filename_component(_PROTOBUF_IMPORT_PREFIX
+  "${_PROTOBUF_IMPORT_PREFIX}" PATH)
+get_filename_component(_PROTOBUF_IMPORT_PREFIX
+  "${_PROTOBUF_IMPORT_PREFIX}" PATH)
+
+# CMake FindProtobuf module compatible file
+if(NOT DEFINED PROTOBUF_MODULE_COMPATIBLE OR "${PROTOBUF_MODULE_COMPATIBLE}")
+  include("${_PROTOBUF_PACKAGE_PREFIX}/protobuf-module.cmake")
+endif()
+
+# Cleanup temporary variables.
+set(_PROTOBUF_PACKAGE_PREFIX)
+set(_PROTOBUF_IMPORT_PREFIX)

cmake/protobuf-module.cmake.in

@@ -0,0 +1,139 @@
+if(PROTOBUF_SRC_ROOT_FOLDER)
+  message(AUTHOR_WARNING "Variable PROTOBUF_SRC_ROOT_FOLDER defined, but not"
+    " used in CONFIG mode")
+endif()
+
+function(PROTOBUF_GENERATE_CPP SRCS HDRS)
+  if(NOT ARGN)
+    message(SEND_ERROR "Error: PROTOBUF_GENERATE_CPP() called without any proto files")
+    return()
+  endif()
+
+  if(PROTOBUF_GENERATE_CPP_APPEND_PATH)
+    # Create an include path for each file specified
+    foreach(FIL ${ARGN})
+      get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+      get_filename_component(ABS_PATH ${ABS_FIL} PATH)
+      list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
+      if(${_contains_already} EQUAL -1)
+          list(APPEND _protobuf_include_path -I ${ABS_PATH})
+      endif()
+    endforeach()
+  else()
+    set(_protobuf_include_path -I ${CMAKE_CURRENT_SOURCE_DIR})
+  endif()
+
+  # Add well-known type protos include path
+  list(APPEND _protobuf_include_path
+    -I "${_PROTOBUF_IMPORT_PREFIX}/@CMAKE_INSTALL_INCLUDEDIR@")
+
+  if(DEFINED PROTOBUF_IMPORT_DIRS)
+    foreach(DIR ${PROTOBUF_IMPORT_DIRS})
+      get_filename_component(ABS_PATH ${DIR} ABSOLUTE)
+      list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
+      if(${_contains_already} EQUAL -1)
+          list(APPEND _protobuf_include_path -I ${ABS_PATH})
+      endif()
+    endforeach()
+  endif()
+
+  set(${SRCS})
+  set(${HDRS})
+  foreach(FIL ${ARGN})
+    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+    get_filename_component(FIL_WE ${FIL} NAME_WE)
+
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")
+
+    add_custom_command(
+      OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
+             "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
+      COMMAND ${PROTOBUF_PROTOC_EXECUTABLE}
+      ARGS --cpp_out  ${CMAKE_CURRENT_BINARY_DIR} ${_protobuf_include_path} ${ABS_FIL}
+      DEPENDS ${ABS_FIL} ${PROTOBUF_PROTOC_EXECUTABLE}
+      COMMENT "Running C++ protocol buffer compiler on ${FIL}"
+      VERBATIM)
+  endforeach()
+
+  set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)
+  set(${SRCS} ${${SRCS}} PARENT_SCOPE)
+  set(${HDRS} ${${HDRS}} PARENT_SCOPE)
+endfunction()
+
+# Internal function: search for normal library as well as a debug one
+#    if the debug one is specified also include debug/optimized keywords
+#    in *_LIBRARIES variable
+function(_protobuf_find_libraries name filename)
+   get_target_property(${name}_LIBRARY lib${filename}
+     IMPORTED_LOCATION_RELEASE)
+   set(${name}_LIBRARY "${${name}_LIBRARY}" PARENT_SCOPE)
+   get_target_property(${name}_LIBRARY_DEBUG lib${filename}
+     IMPORTED_LOCATION_DEBUG)
+   set(${name}_LIBRARY_DEBUG "${${name}_LIBRARY_DEBUG}" PARENT_SCOPE)
+
+   if(NOT ${name}_LIBRARY_DEBUG)
+      # There is no debug library
+      set(${name}_LIBRARY_DEBUG ${${name}_LIBRARY} PARENT_SCOPE)
+      set(${name}_LIBRARIES     ${${name}_LIBRARY} PARENT_SCOPE)
+   else()
+      # There IS a debug library
+      set(${name}_LIBRARIES
+          optimized ${${name}_LIBRARY}
+          debug     ${${name}_LIBRARY_DEBUG}
+          PARENT_SCOPE
+      )
+   endif()
+endfunction()
+
+# Internal function: find threads library
+function(_protobuf_find_threads)
+    set(CMAKE_THREAD_PREFER_PTHREAD TRUE)
+    find_package(Threads)
+    if(Threads_FOUND)
+        list(APPEND PROTOBUF_LIBRARIES ${CMAKE_THREAD_LIBS_INIT})
+        set(PROTOBUF_LIBRARIES "${PROTOBUF_LIBRARIES}" PARENT_SCOPE)
+    endif()
+endfunction()
+
+#
+# Main.
+#
+
+# By default have PROTOBUF_GENERATE_CPP macro pass -I to protoc
+# for each directory where a proto file is referenced.
+if(NOT DEFINED PROTOBUF_GENERATE_CPP_APPEND_PATH)
+  set(PROTOBUF_GENERATE_CPP_APPEND_PATH TRUE)
+endif()
+
+# The Protobuf library
+_protobuf_find_libraries(PROTOBUF protobuf)
+
+# The Protobuf Lite library
+_protobuf_find_libraries(PROTOBUF_LITE protobuf-lite)
+
+# The Protobuf Protoc Library
+_protobuf_find_libraries(PROTOBUF_PROTOC protoc)
+
+if(UNIX)
+  _protobuf_find_threads()
+endif()
+
+# Set the include directory
+set(PROTOBUF_INCLUDE_DIR "${_PROTOBUF_IMPORT_PREFIX}/@CMAKE_INSTALL_INCLUDEDIR@")
+
+# Set the protoc Executable
+get_target_property(PROTOBUF_PROTOC_EXECUTABLE protoc
+  IMPORTED_LOCATION_RELEASE)
+if(NOT PROTOBUF_PROTOC_EXECUTABLE)
+  get_target_property(PROTOBUF_PROTOC_EXECUTABLE protoc
+    IMPORTED_LOCATION_DEBUG)
+endif()
+
+include(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(PROTOBUF DEFAULT_MSG
+    PROTOBUF_LIBRARY PROTOBUF_INCLUDE_DIR)
+
+if(PROTOBUF_FOUND)
+    set(PROTOBUF_INCLUDE_DIRS ${PROTOBUF_INCLUDE_DIR})
+endif()

conformance/failure_list_ruby.txt

@@ -1,17 +1,2 @@
 JsonInput.HelloWorld.JsonOutput
 JsonInput.HelloWorld.ProtobufOutput
-ProtobufInput.PrematureEofBeforeUnknownValue.DOUBLE
-ProtobufInput.PrematureEofBeforeUnknownValue.FIXED32
-ProtobufInput.PrematureEofBeforeUnknownValue.FIXED64
-ProtobufInput.PrematureEofBeforeUnknownValue.FLOAT
-ProtobufInput.PrematureEofBeforeUnknownValue.SFIXED32
-ProtobufInput.PrematureEofBeforeUnknownValue.SFIXED64
-ProtobufInput.PrematureEofInDelimitedDataForUnknownValue.BYTES
-ProtobufInput.PrematureEofInDelimitedDataForUnknownValue.MESSAGE
-ProtobufInput.PrematureEofInDelimitedDataForUnknownValue.STRING
-ProtobufInput.PrematureEofInsideUnknownValue.DOUBLE
-ProtobufInput.PrematureEofInsideUnknownValue.FIXED32
-ProtobufInput.PrematureEofInsideUnknownValue.FIXED64
-ProtobufInput.PrematureEofInsideUnknownValue.FLOAT
-ProtobufInput.PrematureEofInsideUnknownValue.SFIXED32
-ProtobufInput.PrematureEofInsideUnknownValue.SFIXED64

csharp/src/Google.Protobuf/Reflection/FileDescriptor.cs

@@ -368,5 +368,20 @@ namespace Google.Protobuf.Reflection
         {
             return "FileDescriptor for " + proto.Name;
         }
+
+        /// <summary>
+        /// Returns the file descriptor for descriptor.proto.
+        /// </summary>
+        /// <remarks>
+        /// This is used for protos which take a direct dependency on <c>descriptor.proto</c>, typically for
+        /// annotations. While <c>descriptor.proto</c> is a proto2 file, it is built into the Google.Protobuf
+        /// runtime for reflection purposes. The messages are internal to the runtime as they would require
+        /// proto2 semantics for full support, but the file descriptor is available via this property. The
+        /// C# codegen in protoc automatically uses this property when it detects a dependency on <c>descriptor.proto</c>.
+        /// </remarks>
+        /// <value>
+        /// The file descriptor for <c>descriptor.proto</c>.
+        /// </value>
+        public static FileDescriptor DescriptorProtoFileDescriptor { get { return DescriptorProtoFile.Descriptor; } }
     }
 }

ruby/ext/google/protobuf_c/upb.c

@@ -7478,6 +7478,8 @@ void upb_pbdecodermethodopts_setlazy(upb_pbdecodermethodopts *opts, bool lazy) {
 
 /* Error messages that are shared between the bytecode and JIT decoders. */
 const char *kPbDecoderStackOverflow = "Nesting too deep.";
+const char *kPbDecoderSubmessageTooLong =
+    "Submessage end extends past enclosing submessage.";
 
 /* Error messages shared within this file. */
 static const char *kUnterminatedVarint = "Unterminated varint.";
@@ -7512,6 +7514,28 @@ static bool consumes_input(opcode op) {
   }
 }
 
+static size_t stacksize(upb_pbdecoder *d, size_t entries) {
+  UPB_UNUSED(d);
+  return entries * sizeof(upb_pbdecoder_frame);
+}
+
+static size_t callstacksize(upb_pbdecoder *d, size_t entries) {
+  UPB_UNUSED(d);
+
+#ifdef UPB_USE_JIT_X64
+  if (d->method_->is_native_) {
+    /* Each native stack frame needs two pointers, plus we need a few frames for
+     * the enter/exit trampolines. */
+    size_t ret = entries * sizeof(void*) * 2;
+    ret += sizeof(void*) * 10;
+    return ret;
+  }
+#endif
+
+  return entries * sizeof(uint32_t*);
+}
+
+
 static bool in_residual_buf(const upb_pbdecoder *d, const char *p);
 
 /* It's unfortunate that we have to micro-manage the compiler with
@@ -7544,11 +7568,21 @@ static size_t curbufleft(const upb_pbdecoder *d) {
   return d->data_end - d->ptr;
 }
 
+/* How many bytes are available before end-of-buffer. */
+static size_t bufleft(const upb_pbdecoder *d) {
+  return d->end - d->ptr;
+}
+
 /* Overall stream offset of d->ptr. */
 uint64_t offset(const upb_pbdecoder *d) {
   return d->bufstart_ofs + (d->ptr - d->buf);
 }
 
+/* How many bytes are available before the end of this delimited region. */
+size_t delim_remaining(const upb_pbdecoder *d) {
+  return d->top->end_ofs - offset(d);
+}
+
 /* Advances d->ptr. */
 static void advance(upb_pbdecoder *d, size_t len) {
   assert(curbufleft(d) >= len);
@@ -7597,24 +7631,72 @@ static void checkpoint(upb_pbdecoder *d) {
   d->checkpoint = d->ptr;
 }
 
+/* Skips "bytes" bytes in the stream, which may be more than available.  If we
+ * skip more bytes than are available, we return a long read count to the caller
+ * indicating how many bytes can be skipped over before passing actual data
+ * again.  Skipped bytes can pass a NULL buffer and the decoder guarantees they
+ * won't actually be read.
+ */
+static int32_t skip(upb_pbdecoder *d, size_t bytes) {
+  assert(!in_residual_buf(d, d->ptr) || d->size_param == 0);
+  assert(d->skip == 0);
+  if (bytes > delim_remaining(d)) {
+    seterr(d, "Skipped value extended beyond enclosing submessage.");
+    return upb_pbdecoder_suspend(d);
+  } else if (bufleft(d) > bytes) {
+    /* Skipped data is all in current buffer, and more is still available. */
+    advance(d, bytes);
+    d->skip = 0;
+    return DECODE_OK;
+  } else {
+    /* Skipped data extends beyond currently available buffers. */
+    d->pc = d->last;
+    d->skip = bytes - curbufleft(d);
+    d->bufstart_ofs += (d->end - d->buf);
+    d->residual_end = d->residual;
+    switchtobuf(d, d->residual, d->residual_end);
+    return d->size_param + d->skip;
+  }
+}
+
+
 /* Resumes the decoder from an initial state or from a previous suspend. */
 int32_t upb_pbdecoder_resume(upb_pbdecoder *d, void *p, const char *buf,
                              size_t size, const upb_bufhandle *handle) {
   UPB_UNUSED(p);  /* Useless; just for the benefit of the JIT. */
+
   d->buf_param = buf;
   d->size_param = size;
   d->handle = handle;
+
   if (d->residual_end > d->residual) {
     /* We have residual bytes from the last buffer. */
     assert(d->ptr == d->residual);
   } else {
     switchtobuf(d, buf, buf + size);
   }
+
   d->checkpoint = d->ptr;
+
+  if (d->skip) {
+    size_t skip_bytes = d->skip;
+    d->skip = 0;
+    CHECK_RETURN(skip(d, skip_bytes));
+    d->checkpoint = d->ptr;
+  }
+
+  if (!buf) {
+    /* NULL buf is ok if its entire span is covered by the "skip" above, but
+     * by this point we know that "skip" doesn't cover the buffer. */
+    seterr(d, "Passed NULL buffer over non-skippable region.");
+    return upb_pbdecoder_suspend(d);
+  }
+
   if (d->top->groupnum < 0) {
     CHECK_RETURN(upb_pbdecoder_skipunknown(d, -1, 0));
     d->checkpoint = d->ptr;
   }
+
   return DECODE_OK;
 }
 
@@ -7674,28 +7756,6 @@ static size_t suspend_save(upb_pbdecoder *d) {
   return d->size_param;
 }
 
-/* Skips "bytes" bytes in the stream, which may be more than available.  If we
- * skip more bytes than are available, we return a long read count to the caller
- * indicating how many bytes the caller should skip before passing a new buffer.
- */
-static int32_t skip(upb_pbdecoder *d, size_t bytes) {
-  assert(!in_residual_buf(d, d->ptr) || d->size_param == 0);
-  if (curbufleft(d) >= bytes) {
-    /* Skipped data is all in current buffer. */
-    advance(d, bytes);
-    return DECODE_OK;
-  } else {
-    /* Skipped data extends beyond currently available buffers. */
-    size_t skip;
-    d->pc = d->last;
-    skip = bytes - curbufleft(d);
-    d->bufstart_ofs += (d->end - d->buf) + skip;
-    d->residual_end = d->residual;
-    switchtobuf(d, d->residual, d->residual_end);
-    return d->size_param + skip;
-  }
-}
-
 /* Copies the next "bytes" bytes into "buf" and advances the stream.
  * Requires that this many bytes are available in the current buffer. */
 UPB_FORCEINLINE static void consumebytes(upb_pbdecoder *d, void *buf,
@@ -7860,7 +7920,7 @@ static bool decoder_push(upb_pbdecoder *d, uint64_t end) {
   upb_pbdecoder_frame *fr = d->top;
 
   if (end > fr->end_ofs) {
-    seterr(d, "Submessage end extends past enclosing submessage.");
+    seterr(d, kPbDecoderSubmessageTooLong);
     return false;
   } else if (fr == d->limit) {
     seterr(d, kPbDecoderStackOverflow);
@@ -7964,34 +8024,7 @@ have_tag:
       return DECODE_OK;
     }
 
-    if (d->ptr == d->delim_end) {
-      seterr(d, "Enclosing submessage ended in the middle of value or group");
-      /* Unlike most errors we notice during parsing, right now we have consumed
-       * all of the user's input.
-       *
-       * There are three different options for how to handle this case:
-       *
-       *   1. decode() = short count, error = set
-       *   2. decode() = full count, error = set
-       *   3. decode() = full count, error NOT set, short count and error will
-       *      be reported on next call to decode() (or end())
-       *
-       * (1) and (3) have the advantage that they preserve the invariant that an
-       * error occurs iff decode() returns a short count.
-       *
-       * (2) and (3) have the advantage of reflecting the fact that all of the
-       * bytes were in fact parsed (and possibly delivered to the unknown field
-       * handler, in the future when that is supported).
-       *
-       * (3) requires extra state in the decode (a place to store the "permanent
-       * error" that we should return for all subsequent attempts to decode).
-       * But we likely want this anyway.
-       *
-       * Right now we do (1), thanks to the fact that we checkpoint *after* this
-       * check.  (3) may be a better choice long term; unclear at the moment. */
-      return upb_pbdecoder_suspend(d);
-    }
-
+    /* Unknown group -- continue looping over unknown fields. */
     checkpoint(d);
   }
 }
@@ -8015,7 +8048,7 @@ static int32_t dispatch(upb_pbdecoder *d) {
   uint8_t wire_type;
   uint32_t fieldnum;
   upb_value val;
-  int32_t ret;
+  int32_t retval;
 
   /* Decode tag. */
   CHECK_RETURN(decode_v32(d, &tag));
@@ -8039,23 +8072,25 @@ static int32_t dispatch(upb_pbdecoder *d) {
     }
   }
 
+  /* We have some unknown fields (or ENDGROUP) to parse.  The DISPATCH or TAG
+   * bytecode that triggered this is preceded by a CHECKDELIM bytecode which
+   * we need to back up to, so that when we're done skipping unknown data we
+   * can re-check the delimited end. */
+  d->last--;  /* Necessary if we get suspended */
+  d->pc = d->last;
+  assert(getop(*d->last) == OP_CHECKDELIM);
+
   /* Unknown field or ENDGROUP. */
-  ret = upb_pbdecoder_skipunknown(d, fieldnum, wire_type);
+  retval = upb_pbdecoder_skipunknown(d, fieldnum, wire_type);
+
+  CHECK_RETURN(retval);
 
-  if (ret == DECODE_ENDGROUP) {
+  if (retval == DECODE_ENDGROUP) {
     goto_endmsg(d);
     return DECODE_OK;
-  } else if (ret == DECODE_OK) {
-    /* We just consumed some input, so we might now have consumed all the data
-     * in the delmited region.  Since every opcode that can trigger dispatch is
-     * directly preceded by OP_CHECKDELIM, rewind to it now to re-check the
-     * delimited end. */
-    d->pc = d->last - 1;
-    assert(getop(*d->pc) == OP_CHECKDELIM);
-    return DECODE_OK;
   }
 
-  return ret;
+  return DECODE_OK;
 }
 
 /* Callers know that the stack is more than one deep because the opcodes that
@@ -8070,18 +8105,8 @@ upb_pbdecoder_frame *outer_frame(upb_pbdecoder *d) {
 
 /* The main decoder VM function.  Uses traditional bytecode dispatch loop with a
  * switch() statement. */
-size_t upb_pbdecoder_decode(void *closure, const void *hd, const char *buf,
-                            size_t size, const upb_bufhandle *handle) {
-  upb_pbdecoder *d = closure;
-  const mgroup *group = hd;
-  int32_t result;
-  assert(buf);
-  result = upb_pbdecoder_resume(d, NULL, buf, size, handle);
-  if (result == DECODE_ENDGROUP) {
-    goto_endmsg(d);
-  }
-  CHECK_RETURN(result);
-  UPB_UNUSED(group);
+size_t run_decoder_vm(upb_pbdecoder *d, const mgroup *group,
+                      const upb_bufhandle* handle) {
 
 #define VMCASE(op, code) \
   case op: { code; if (consumes_input(op)) checkpoint(d); break; }
@@ -8104,6 +8129,7 @@ size_t upb_pbdecoder_decode(void *closure, const void *hd, const char *buf,
     arg = instruction >> 8;
     longofs = arg;
     assert(d->ptr != d->residual_end);
+    UPB_UNUSED(group);
 #ifdef UPB_DUMP_BYTECODE
     fprintf(stderr, "s_ofs=%d buf_ofs=%d data_rem=%d buf_rem=%d delim_rem=%d "
                     "%x %s (%d)\n",
@@ -8160,7 +8186,7 @@ size_t upb_pbdecoder_decode(void *closure, const void *hd, const char *buf,
         CHECK_SUSPEND(upb_sink_endsubmsg(&d->top->sink, arg));
       )
       VMCASE(OP_STARTSTR,
-        uint32_t len = d->top->end_ofs - offset(d);
+        uint32_t len = delim_remaining(d);
         upb_pbdecoder_frame *outer = outer_frame(d);
         CHECK_SUSPEND(upb_sink_startstr(&outer->sink, arg, len, &d->top->sink));
         if (len == 0) {
@@ -8171,7 +8197,7 @@ size_t upb_pbdecoder_decode(void *closure, const void *hd, const char *buf,
         uint32_t len = curbufleft(d);
         size_t n = upb_sink_putstring(&d->top->sink, arg, d->ptr, len, handle);
         if (n > len) {
-          if (n > d->top->end_ofs - offset(d)) {
+          if (n > delim_remaining(d)) {
             seterr(d, "Tried to skip past end of string.");
             return upb_pbdecoder_suspend(d);
           } else {
@@ -8279,12 +8305,15 @@ size_t upb_pbdecoder_decode(void *closure, const void *hd, const char *buf,
         CHECK_RETURN(dispatch(d));
       })
       VMCASE(OP_HALT, {
-        return size;
+        return d->size_param;
       })
     }
   }
 }
 
+
+/* BytesHandler handlers ******************************************************/
+
 void *upb_pbdecoder_startbc(void *closure, const void *pc, size_t size_hint) {
   upb_pbdecoder *d = closure;
   UPB_UNUSED(size_hint);
@@ -8293,6 +8322,7 @@ void *upb_pbdecoder_startbc(void *closure, const void *pc, size_t size_hint) {
   d->call_len = 1;
   d->callstack[0] = &halt;
   d->pc = pc;
+  d->skip = 0;
   return d;
 }
 
@@ -8303,6 +8333,7 @@ void *upb_pbdecoder_startjit(void *closure, const void *hd, size_t size_hint) {
   d->top->end_ofs = UINT64_MAX;
   d->bufstart_ofs = 0;
   d->call_len = 0;
+  d->skip = 0;
   return d;
 }
 
@@ -8311,12 +8342,14 @@ bool upb_pbdecoder_end(void *closure, const void *handler_data) {
   const upb_pbdecodermethod *method = handler_data;
   uint64_t end;
   char dummy;
-#ifdef UPB_USE_JIT_X64
-  const mgroup *group = (const mgroup*)method->group;
-#endif
 
   if (d->residual_end > d->residual) {
-    seterr(d, "Unexpected EOF");
+    seterr(d, "Unexpected EOF: decoder still has buffered unparsed data");
+    return false;
+  }
+
+  if (d->skip) {
+    seterr(d, "Unexpected EOF inside skipped data");
     return false;
   }
 
@@ -8325,12 +8358,13 @@ bool upb_pbdecoder_end(void *closure, const void *handler_data) {
     return false;
   }
 
-  /* Message ends here. */
+  /* The user's end() call indicates that the message ends here. */
   end = offset(d);
   d->top->end_ofs = end;
 
 #ifdef UPB_USE_JIT_X64
-  if (group->jit_code) {
+  if (method->is_native_) {
+    const mgroup *group = (const mgroup*)method->group;
     if (d->top != d->stack)
       d->stack->end_ofs = 0;
     group->jit_code(closure, method->code_base.ptr, &dummy, 0, NULL);
@@ -8353,13 +8387,26 @@ bool upb_pbdecoder_end(void *closure, const void *handler_data) {
   }
 
   if (d->call_len != 0) {
-    seterr(d, "Unexpected EOF");
+    seterr(d, "Unexpected EOF inside submessage or group");
     return false;
   }
 
   return true;
 }
 
+size_t upb_pbdecoder_decode(void *decoder, const void *group, const char *buf,
+                            size_t size, const upb_bufhandle *handle) {
+  int32_t result = upb_pbdecoder_resume(decoder, NULL, buf, size, handle);
+
+  if (result == DECODE_ENDGROUP) goto_endmsg(decoder);
+  CHECK_RETURN(result);
+
+  return run_decoder_vm(decoder, group, handle);
+}
+
+
+/* Public API *****************************************************************/
+
 void upb_pbdecoder_reset(upb_pbdecoder *d) {
   d->top = d->stack;
   d->top->groupnum = 0;
@@ -8369,27 +8416,6 @@ void upb_pbdecoder_reset(upb_pbdecoder *d) {
   d->residual_end = d->residual;
 }
 
-static size_t stacksize(upb_pbdecoder *d, size_t entries) {
-  UPB_UNUSED(d);
-  return entries * sizeof(upb_pbdecoder_frame);
-}
-
-static size_t callstacksize(upb_pbdecoder *d, size_t entries) {
-  UPB_UNUSED(d);
-
-#ifdef UPB_USE_JIT_X64
-  if (d->method_->is_native_) {
-    /* Each native stack frame needs two pointers, plus we need a few frames for
-     * the enter/exit trampolines. */
-    size_t ret = entries * sizeof(void*) * 2;
-    ret += sizeof(void*) * 10;
-    return ret;
-  }
-#endif
-
-  return entries * sizeof(uint32_t*);
-}
-
 upb_pbdecoder *upb_pbdecoder_create(upb_env *e, const upb_pbdecodermethod *m,
                                     upb_sink *sink) {
   const size_t default_max_nesting = 64;

ruby/ext/google/protobuf_c/upb.h

@@ -5635,7 +5635,7 @@ UPB_INLINE bool upb_bufsrc_putbuf(const char *buf, size_t len,
   upb_bufhandle_setbuf(&handle, buf, 0);
   ret = upb_bytessink_start(sink, len, &subc);
   if (ret && len != 0) {
-    ret = (upb_bytessink_putbuf(sink, subc, buf, len, &handle) == len);
+    ret = (upb_bytessink_putbuf(sink, subc, buf, len, &handle) >= len);
   }
   if (ret) {
     ret = upb_bytessink_end(sink);
@@ -7123,7 +7123,7 @@ class upb::pb::DecoderMethod {
  * constructed.  This hint may be an overestimate for some build configurations.
  * But if the decoder library is upgraded without recompiling the application,
  * it may be an underestimate. */
-#define UPB_PB_DECODER_SIZE 4400
+#define UPB_PB_DECODER_SIZE 4408
 
 #ifdef __cplusplus
 
@@ -7548,6 +7548,12 @@ struct upb_pbdecoder {
   char residual[12];
   char *residual_end;
 
+  /* Bytes of data that should be discarded from the input beore we start
+   * parsing again.  We set this when we internally determine that we can
+   * safely skip the next N bytes, but this region extends past the current
+   * user buffer. */
+  size_t skip;
+
   /* Stores the user buffer passed to our decode function. */
   const char *buf_param;
   size_t size_param;
@@ -7590,6 +7596,7 @@ void upb_pbdecoder_seterr(upb_pbdecoder *d, const char *msg);
 
 /* Error messages that are shared between the bytecode and JIT decoders. */
 extern const char *kPbDecoderStackOverflow;
+extern const char *kPbDecoderSubmessageTooLong;
 
 /* Access to decoderplan members needed by the decoder. */
 const char *upb_pbdecoder_getopname(unsigned int op);

src/google/protobuf/compiler/csharp/csharp_helpers.h

@@ -117,7 +117,10 @@ inline bool IsMapEntryMessage(const Descriptor* descriptor) {
 inline bool IsDescriptorProto(const FileDescriptor* descriptor) {
   // TODO: Do this better! (Currently this depends on a hack in generate_protos.sh to rename
   // the file...)
-  return descriptor->name() == "google/protobuf/descriptor_proto_file.proto";
+  // We need to be able to detect the "normal" name as well, for times that we're just
+  // depending on descriptor.proto instead of generating it.
+  return descriptor->name() == "google/protobuf/descriptor_proto_file.proto"
+      || descriptor->name() == "google/protobuf/descriptor.proto";
 }
 
 inline bool IsWrapperType(const FieldDescriptor* descriptor) {

src/google/protobuf/compiler/csharp/csharp_umbrella_class.cc

@@ -180,10 +180,17 @@ void UmbrellaClassGenerator::WriteDescriptor(io::Printer* printer) {
       "descriptor = pbr::FileDescriptor.InternalBuildGeneratedFileFrom(descriptorData,\n");
   printer->Print("    new pbr::FileDescriptor[] { ");
   for (int i = 0; i < file_->dependency_count(); i++) {
-    printer->Print(
+    // descriptor.proto is special: we don't allow access to the generated code, but there's
+    // a separately-exposed property to get at the file descriptor, specifically to allow this
+    // kind of dependency.
+    if (IsDescriptorProto(file_->dependency(i))) {
+      printer->Print("pbr::FileDescriptor.DescriptorProtoFileDescriptor, ");
+    } else {
+      printer->Print(
       "$full_umbrella_class_name$.Descriptor, ",
       "full_umbrella_class_name",
       GetUmbrellaClassName(file_->dependency(i)));
+    }
   }
   printer->Print("},\n"
       "    new pbr::GeneratedCodeInfo(");