Merge pull request #6151 from haon4/201905171200

Down Integrate to GitHub

benchmarks/python/py_benchmark.py

@@ -41,7 +41,7 @@ import benchmarks_pb2 as benchmarks_pb2
 
 
 def run_one_test(filename):
-  data = open(filename).read()
+  data = open(filename, "rb").read()
   benchmark_dataset = benchmarks_pb2.BenchmarkDataset()
   benchmark_dataset.ParseFromString(data)
   total_bytes = 0
@@ -69,7 +69,7 @@ def init(filename):
   message_list=[]
   counter = 0
   total_bytes = 0
-  data = open(filename).read()
+  data = open(filename, "rb").read()
   benchmark_dataset = benchmarks_pb2.BenchmarkDataset()
   benchmark_dataset.ParseFromString(data)
 

java/core/src/test/proto/com/google/protobuf/multiple_files_test.proto

@@ -41,7 +41,7 @@ package protobuf_unittest;
 import "google/protobuf/unittest.proto";
 import "google/protobuf/descriptor.proto";
 
-option java_generic_services = true; // auto-added
+option java_generic_services = true;  // auto-added
 option java_multiple_files = true;
 option java_outer_classname = "MultipleFilesTestProto";
 

java/core/src/test/proto/com/google/protobuf/test_bad_identifiers.proto

@@ -39,7 +39,7 @@ syntax = "proto2";
 // See:  http://go/proto2-generic-services-default
 package io_protocol_tests;
 
-option java_generic_services = true; // auto-added
+option java_generic_services = true;  // auto-added
 option java_package = "com.google.protobuf";
 option java_outer_classname = "TestBadIdentifiersProto";
 

protobuf.pc.in

@@ -8,5 +8,6 @@ Description: Google's Data Interchange Format
 Version: @VERSION@
 Libs: -L${libdir} -lprotobuf @PTHREAD_LIBS@
 Libs.private: @LIBS@
+
 Cflags: -I${includedir} @PTHREAD_CFLAGS@
 Conflicts: protobuf-lite

python/google/protobuf/descriptor_pool.py

@@ -99,6 +99,8 @@ def _IsMessageSetExtension(field):
           field.label == descriptor.FieldDescriptor.LABEL_OPTIONAL)
 
 
+
+
 class DescriptorPool(object):
   """A collection of protobufs dynamically constructed by descriptor protos."""
 
@@ -161,18 +163,18 @@ class DescriptorPool(object):
 
         if not isinstance(desc, descriptor_type) or (
             old_file != file_name):
-          warn_msg = ('Conflict register for file "' + file_name +
-                      '": ' + desc_name +
-                      ' is already defined in file "' +
-                      old_file + '". Please fix the conflict by adding '
-                      'package name on the proto file, or use different '
-                      'name for the duplication. This warning will '
-                      'turn into error soon.')
+          error_msg = ('Conflict register for file "' + file_name +
+                       '": ' + desc_name +
+                       ' is already defined in file "' +
+                       old_file + '". Please fix the conflict by adding '
+                       'package name on the proto file, or use different '
+                       'name for the duplication.')
           if isinstance(desc, descriptor.EnumValueDescriptor):
-            warn_msg += ('\nNote: enum values appear as '
-                         'siblings of the enum type instead of '
-                         'children of it.')
-          warnings.warn(warn_msg, RuntimeWarning)
+            error_msg += ('\nNote: enum values appear as '
+                          'siblings of the enum type instead of '
+                          'children of it.')
+
+          raise TypeError(error_msg)
 
         return
 

python/google/protobuf/internal/descriptor_database_test.py

@@ -115,19 +115,18 @@ class DescriptorDatabaseTest(unittest.TestCase):
     db.Add(unittest_fd)
     conflict_fd = descriptor_pb2.FileDescriptorProto.FromString(
         unittest_pb2.DESCRIPTOR.serialized_pb)
-    conflict_fd.name = 'other_file'
+    conflict_fd.name = 'other_file2'
     with warnings.catch_warnings(record=True) as w:
       # Cause all warnings to always be triggered.
       warnings.simplefilter('always')
       db.Add(conflict_fd)
       self.assertTrue(len(w))
       self.assertIs(w[0].category, RuntimeWarning)
-      self.assertIn('Conflict register for file "other_file": ',
+      self.assertIn('Conflict register for file "other_file2": ',
                     str(w[0].message))
       self.assertIn('already defined in file '
                     '"google/protobuf/unittest.proto"',
                     str(w[0].message))
 
-
 if __name__ == '__main__':
   unittest.main()

python/google/protobuf/internal/descriptor_pool_test.py

@@ -36,7 +36,6 @@ __author__ = 'matthewtoia@google.com (Matt Toia)'
 
 import copy
 import os
-import warnings
 
 try:
   import unittest2 as unittest  #PY26
@@ -532,33 +531,21 @@ class DescriptorPoolTestBase(object):
     if api_implementation.Type() == 'cpp':
         pass
     else:
-      with warnings.catch_warnings(record=True) as w:
-        # Cause all warnings to always be triggered.
-        warnings.simplefilter('always')
-        pool = copy.deepcopy(self.pool)
-        # No warnings to add the same descriptors.
-        file_descriptor = unittest_pb2.DESCRIPTOR
-        pool.AddDescriptor(
-            file_descriptor.message_types_by_name['TestAllTypes'])
-        pool.AddEnumDescriptor(
-            file_descriptor.enum_types_by_name['ForeignEnum'])
-        pool.AddServiceDescriptor(
-            file_descriptor.services_by_name['TestService'])
-        pool.AddExtensionDescriptor(
-            file_descriptor.extensions_by_name['optional_int32_extension'])
-        self.assertEqual(len(w), 0)
-        # Check warnings for conflict descriptors with the same name.
-        pool.Add(unittest_fd)
-        pool.Add(conflict_fd)
-        pool.FindFileByName(unittest_fd.name)
+      pool = copy.deepcopy(self.pool)
+      file_descriptor = unittest_pb2.DESCRIPTOR
+      pool.AddDescriptor(
+          file_descriptor.message_types_by_name['TestAllTypes'])
+      pool.AddEnumDescriptor(
+          file_descriptor.enum_types_by_name['ForeignEnum'])
+      pool.AddServiceDescriptor(
+          file_descriptor.services_by_name['TestService'])
+      pool.AddExtensionDescriptor(
+          file_descriptor.extensions_by_name['optional_int32_extension'])
+      pool.Add(unittest_fd)
+      pool.Add(conflict_fd)
+      pool.FindFileByName(unittest_fd.name)
+      with self.assertRaises(TypeError):
         pool.FindFileByName(conflict_fd.name)
-        self.assertTrue(len(w))
-        self.assertIs(w[0].category, RuntimeWarning)
-        self.assertIn('Conflict register for file "other_file": ',
-                      str(w[0].message))
-        self.assertIn('already defined in file '
-                      '"google/protobuf/unittest.proto"',
-                      str(w[0].message))
 
 
 @testing_refleaks.TestCase

python/google/protobuf/internal/python_message.py

@@ -692,7 +692,11 @@ def _AddPropertiesForNonRepeatedScalarField(field, cls):
     # pylint: disable=protected-access
     # Testing the value for truthiness captures all of the proto3 defaults
     # (0, 0.0, enum 0, and False).
-    new_value = type_checker.CheckValue(new_value)
+    try:
+      new_value = type_checker.CheckValue(new_value)
+    except TypeError as e:
+      raise TypeError(
+          'Cannot set %s to %.1024r: %s' % (field.full_name, new_value, e))
     if clear_when_set_to_default and not new_value:
       self._fields.pop(field, None)
     else:

src/google/protobuf/compiler/js/js_generator.cc

@@ -2426,18 +2426,22 @@ void Generator::GenerateObjectTypedef(const GeneratorOptions& options,
       "method.\n"
       " * @record\n"
       " */\n"
-      "$typeName$ = function() {};\n\n",
+      "$typeName$ = function() {\n",
       "messageName", desc->name(), "typeName", type_name);
 
   for (int i = 0; i < desc->field_count(); i++) {
+    if (i > 0) {
+      printer->Print("\n");
+    }
     printer->Print(
-        "/** @type {$fieldType$|undefined} */\n"
-        "$typeName$.prototype.$fieldName$;\n\n",
-        "typeName", type_name, "fieldName",
-        JSObjectFieldName(options, desc->field(i)),
+        "  /** @type {$fieldType$|undefined} */\n"
+        "  this.$fieldName$;\n",
+        "fieldName", JSObjectFieldName(options, desc->field(i)),
         // TODO(b/121097361): Add type checking for field values.
         "fieldType", "?");
   }
+
+  printer->Print("};\n\n");
 }
 
 void Generator::GenerateClassFromObject(const GeneratorOptions& options,

src/google/protobuf/descriptor.cc

@@ -1487,6 +1487,7 @@ const MethodDescriptor* DescriptorPool::FindMethodByName(
 
 const FieldDescriptor* DescriptorPool::FindExtensionByNumber(
     const Descriptor* extendee, int number) const {
+  if (extendee->extension_range_count() == 0) return nullptr;
   // A faster path to reduce lock contention in finding extensions, assuming
   // most extensions will be cache hit.
   if (mutex_ != nullptr) {

src/google/protobuf/descriptor_database.cc

@@ -127,14 +127,15 @@ bool SimpleDescriptorDatabase::DescriptorIndex<Value>::AddFile(
 
   for (int i = 0; i < file.message_type_size(); i++) {
     if (!AddSymbol(path + file.message_type(i).name(), value)) return false;
-    if (!AddNestedExtensions(file.message_type(i), value)) return false;
+    if (!AddNestedExtensions(file.name(), file.message_type(i), value))
+      return false;
   }
   for (int i = 0; i < file.enum_type_size(); i++) {
     if (!AddSymbol(path + file.enum_type(i).name(), value)) return false;
   }
   for (int i = 0; i < file.extension_size(); i++) {
     if (!AddSymbol(path + file.extension(i).name(), value)) return false;
-    if (!AddExtension(file.extension(i), value)) return false;
+    if (!AddExtension(file.name(), file.extension(i), value)) return false;
   }
   for (int i = 0; i < file.service_size(); i++) {
     if (!AddSymbol(path + file.service(i).name(), value)) return false;
@@ -203,19 +204,22 @@ bool SimpleDescriptorDatabase::DescriptorIndex<Value>::AddSymbol(
 
 template <typename Value>
 bool SimpleDescriptorDatabase::DescriptorIndex<Value>::AddNestedExtensions(
-    const DescriptorProto& message_type, Value value) {
+    const std::string& filename, const DescriptorProto& message_type,
+    Value value) {
   for (int i = 0; i < message_type.nested_type_size(); i++) {
-    if (!AddNestedExtensions(message_type.nested_type(i), value)) return false;
+    if (!AddNestedExtensions(filename, message_type.nested_type(i), value))
+      return false;
   }
   for (int i = 0; i < message_type.extension_size(); i++) {
-    if (!AddExtension(message_type.extension(i), value)) return false;
+    if (!AddExtension(filename, message_type.extension(i), value)) return false;
   }
   return true;
 }
 
 template <typename Value>
 bool SimpleDescriptorDatabase::DescriptorIndex<Value>::AddExtension(
-    const FieldDescriptorProto& field, Value value) {
+    const std::string& filename, const FieldDescriptorProto& field,
+    Value value) {
   if (!field.extendee().empty() && field.extendee()[0] == '.') {
     // The extension is fully-qualified.  We can use it as a lookup key in
     // the by_symbol_ table.
@@ -226,7 +230,7 @@ bool SimpleDescriptorDatabase::DescriptorIndex<Value>::AddExtension(
       GOOGLE_LOG(ERROR) << "Extension conflicts with extension already in database: "
                     "extend "
                  << field.extendee() << " { " << field.name() << " = "
-                 << field.number() << " }";
+                 << field.number() << " } from:" << filename;
       return false;
     }
   } else {

src/google/protobuf/descriptor_database.h

@@ -199,8 +199,10 @@ class PROTOBUF_EXPORT SimpleDescriptorDatabase : public DescriptorDatabase {
     // to the index.
     bool AddFile(const FileDescriptorProto& file, Value value);
     bool AddSymbol(const std::string& name, Value value);
-    bool AddNestedExtensions(const DescriptorProto& message_type, Value value);
-    bool AddExtension(const FieldDescriptorProto& field, Value value);
+    bool AddNestedExtensions(const std::string& filename,
+                             const DescriptorProto& message_type, Value value);
+    bool AddExtension(const std::string& filename,
+                      const FieldDescriptorProto& field, Value value);
 
     Value FindFile(const std::string& filename);
     Value FindSymbol(const std::string& name);

src/google/protobuf/generated_message_reflection.cc

@@ -1015,8 +1015,7 @@ struct FieldNumberSorter {
   }
 };
 
-bool IsIndexInHasBitSet(const uint32* has_bit_set,
-                               uint32 has_bit_index) {
+bool IsIndexInHasBitSet(const uint32* has_bit_set, uint32 has_bit_index) {
   GOOGLE_DCHECK_NE(has_bit_index, ~0u);
   return ((has_bit_set[has_bit_index / 32] >> (has_bit_index % 32)) &
           static_cast<uint32>(1)) != 0;
@@ -1874,8 +1873,8 @@ uint32* Reflection::MutableHasBits(Message* message) const {
   return GetPointerAtOffset<uint32>(message, schema_.HasBitsOffset());
 }
 
-uint32 Reflection::GetOneofCase(
-    const Message& message, const OneofDescriptor* oneof_descriptor) const {
+uint32 Reflection::GetOneofCase(const Message& message,
+                                const OneofDescriptor* oneof_descriptor) const {
   return GetConstRefAtOffset<uint32>(
       message, schema_.GetOneofCaseOffset(oneof_descriptor));
 }
@@ -1886,8 +1885,7 @@ uint32* Reflection::MutableOneofCase(
       message, schema_.GetOneofCaseOffset(oneof_descriptor));
 }
 
-const ExtensionSet& Reflection::GetExtensionSet(
-    const Message& message) const {
+const ExtensionSet& Reflection::GetExtensionSet(const Message& message) const {
   return GetConstRefAtOffset<ExtensionSet>(message,
                                            schema_.GetExtensionSetOffset());
 }
@@ -1901,8 +1899,8 @@ Arena* Reflection::GetArena(Message* message) const {
   return GetInternalMetadataWithArena(*message).arena();
 }
 
-const InternalMetadataWithArena&
-Reflection::GetInternalMetadataWithArena(const Message& message) const {
+const InternalMetadataWithArena& Reflection::GetInternalMetadataWithArena(
+    const Message& message) const {
   return GetConstRefAtOffset<InternalMetadataWithArena>(
       message, schema_.GetMetadataOffset());
 }
@@ -1920,7 +1918,7 @@ const Type& Reflection::DefaultRaw(const FieldDescriptor* field) const {
 
 // Simple accessors for manipulating has_bits_.
 bool Reflection::HasBit(const Message& message,
-                               const FieldDescriptor* field) const {
+                        const FieldDescriptor* field) const {
   GOOGLE_DCHECK(!field->options().weak());
   if (schema_.HasHasbits()) {
     return IsIndexInHasBitSet(GetHasBits(message), schema_.HasBitIndex(field));
@@ -1980,8 +1978,7 @@ bool Reflection::HasBit(const Message& message,
   }
 }
 
-void Reflection::SetBit(Message* message,
-                               const FieldDescriptor* field) const {
+void Reflection::SetBit(Message* message, const FieldDescriptor* field) const {
   GOOGLE_DCHECK(!field->options().weak());
   if (!schema_.HasHasbits()) {
     return;
@@ -1992,7 +1989,7 @@ void Reflection::SetBit(Message* message,
 }
 
 void Reflection::ClearBit(Message* message,
-                                 const FieldDescriptor* field) const {
+                          const FieldDescriptor* field) const {
   GOOGLE_DCHECK(!field->options().weak());
   if (!schema_.HasHasbits()) {
     return;
@@ -2003,7 +2000,7 @@ void Reflection::ClearBit(Message* message,
 }
 
 void Reflection::SwapBit(Message* message1, Message* message2,
-                                const FieldDescriptor* field) const {
+                         const FieldDescriptor* field) const {
   GOOGLE_DCHECK(!field->options().weak());
   if (!schema_.HasHasbits()) {
     return;
@@ -2027,24 +2024,24 @@ bool Reflection::HasOneof(const Message& message,
 }
 
 bool Reflection::HasOneofField(const Message& message,
-                                      const FieldDescriptor* field) const {
+                               const FieldDescriptor* field) const {
   return (GetOneofCase(message, field->containing_oneof()) == field->number());
 }
 
 void Reflection::SetOneofCase(Message* message,
-                                     const FieldDescriptor* field) const {
+                              const FieldDescriptor* field) const {
   *MutableOneofCase(message, field->containing_oneof()) = field->number();
 }
 
 void Reflection::ClearOneofField(Message* message,
-                                        const FieldDescriptor* field) const {
+                                 const FieldDescriptor* field) const {
   if (HasOneofField(*message, field)) {
     ClearOneof(message, field->containing_oneof());
   }
 }
 
-void Reflection::ClearOneof(
-    Message* message, const OneofDescriptor* oneof_descriptor) const {
+void Reflection::ClearOneof(Message* message,
+                            const OneofDescriptor* oneof_descriptor) const {
   // TODO(jieluo): Consider to cache the unused object instead of deleting
   // it. It will be much faster if an application switches a lot from
   // a few oneof fields.  Time/space tradeoff
@@ -2118,13 +2115,13 @@ void* Reflection::MutableRawRepeatedString(Message* message,
 // used for all types except messages.
 template <typename Type>
 const Type& Reflection::GetField(const Message& message,
-                                        const FieldDescriptor* field) const {
+                                 const FieldDescriptor* field) const {
   return GetRaw<Type>(message, field);
 }
 
 template <typename Type>
 void Reflection::SetField(Message* message, const FieldDescriptor* field,
-                                 const Type& value) const {
+                          const Type& value) const {
   if (field->containing_oneof() && !HasOneofField(*message, field)) {
     ClearOneof(message, field->containing_oneof());
   }
@@ -2135,7 +2132,7 @@ void Reflection::SetField(Message* message, const FieldDescriptor* field,
 
 template <typename Type>
 Type* Reflection::MutableField(Message* message,
-                                      const FieldDescriptor* field) const {
+                               const FieldDescriptor* field) const {
   field->containing_oneof() ? SetOneofCase(message, field)
                             : SetBit(message, field);
   return MutableRaw<Type>(message, field);
@@ -2143,29 +2140,29 @@ Type* Reflection::MutableField(Message* message,
 
 template <typename Type>
 const Type& Reflection::GetRepeatedField(const Message& message,
-                                                const FieldDescriptor* field,
-                                                int index) const {
+                                         const FieldDescriptor* field,
+                                         int index) const {
   return GetRaw<RepeatedField<Type> >(message, field).Get(index);
 }
 
 template <typename Type>
 const Type& Reflection::GetRepeatedPtrField(const Message& message,
-                                                   const FieldDescriptor* field,
-                                                   int index) const {
+                                            const FieldDescriptor* field,
+                                            int index) const {
   return GetRaw<RepeatedPtrField<Type> >(message, field).Get(index);
 }
 
 template <typename Type>
 void Reflection::SetRepeatedField(Message* message,
-                                         const FieldDescriptor* field,
-                                         int index, Type value) const {
+                                  const FieldDescriptor* field, int index,
+                                  Type value) const {
   MutableRaw<RepeatedField<Type> >(message, field)->Set(index, value);
 }
 
 template <typename Type>
 Type* Reflection::MutableRepeatedField(Message* message,
-                                              const FieldDescriptor* field,
-                                              int index) const {
+                                       const FieldDescriptor* field,
+                                       int index) const {
   RepeatedPtrField<Type>* repeated =
       MutableRaw<RepeatedPtrField<Type> >(message, field);
   return repeated->Mutable(index);
@@ -2173,13 +2170,13 @@ Type* Reflection::MutableRepeatedField(Message* message,
 
 template <typename Type>
 void Reflection::AddField(Message* message, const FieldDescriptor* field,
-                                 const Type& value) const {
+                          const Type& value) const {
   MutableRaw<RepeatedField<Type> >(message, field)->Add(value);
 }
 
 template <typename Type>
 Type* Reflection::AddField(Message* message,
-                                  const FieldDescriptor* field) const {
+                           const FieldDescriptor* field) const {
   RepeatedPtrField<Type>* repeated =
       MutableRaw<RepeatedPtrField<Type> >(message, field);
   return repeated->Add();

src/google/protobuf/util/time_util.h

@@ -39,12 +39,12 @@
 #ifdef _MSC_VER
 #ifdef _XBOX_ONE
 struct timeval {
-	long    tv_sec;         /* seconds */
-	long    tv_usec;        /* and microseconds */
+  int64 tv_sec;  /* seconds */
+  int64 tv_usec; /* and microseconds */
 };
 #else
 #include <winsock2.h>
-#endif // _XBOX_ONE
+#endif  // _XBOX_ONE
 #else
 #include <sys/time.h>
 #endif