Switch on "new" buffer API (#8339)

"Old" buffer API will removed in Python 3.10. This is also fix #7930.

python/google/protobuf/internal/message_test.py

@@ -841,8 +841,7 @@ class MessageTest(unittest.TestCase):
     m1.MergeFromString(m2.SerializeToString())
     self.assertEqual(1, m1.optional_nested_message.bb)
 
-  @unittest.skipIf(six.PY2, 'memoryview objects are not supported on py2')
-  def testMergeFromStringUsingMemoryViewWorksInPy3(self, message_module):
+  def testMergeFromStringUsingMemoryView(self, message_module):
     m2 = message_module.TestAllTypes()
     m2.optional_string = 'scalar string'
     m2.repeated_string.append('repeated string')
@@ -864,12 +863,6 @@ class MessageTest(unittest.TestCase):
     self.assertIsInstance(m1.optional_string, six.text_type)
     self.assertIsInstance(m1.repeated_string[0], six.text_type)
 
-  @unittest.skipIf(six.PY3, 'memoryview is supported by py3')
-  def testMergeFromStringUsingMemoryViewIsPy2Error(self, message_module):
-    memview = memoryview(b'')
-    with self.assertRaises(TypeError):
-      message_module.TestAllTypes.FromString(memview)
-
   def testMergeFromEmpty(self, message_module):
     m1 = message_module.TestAllTypes()
     # Cpp extension will lazily create a sub message which is immutable.

python/google/protobuf/internal/python_message.py

@@ -1133,12 +1133,6 @@ def _AddSerializePartialToStringMethod(message_descriptor, cls):
 def _AddMergeFromStringMethod(message_descriptor, cls):
   """Helper for _AddMessageMethods()."""
   def MergeFromString(self, serialized):
-    if isinstance(serialized, memoryview) and six.PY2:
-      raise TypeError(
-          'memoryview not supported in Python 2 with the pure Python proto '
-          'implementation: this is to maintain compatibility with the C++ '
-          'implementation')
-
     serialized = memoryview(serialized)
     length = len(serialized)
     try:

python/google/protobuf/pyext/descriptor.cc

@@ -248,7 +248,7 @@ static PyObject* GetOrBuildOptions(const DescriptorClass *descriptor) {
     return NULL;
   }
   ScopedPyObjectPtr value(
-      PyEval_CallObject(message_class->AsPyObject(), NULL));
+      PyObject_Call(message_class->AsPyObject(), NULL, NULL));
   Py_DECREF(message_class);
   if (value == NULL) {
     return NULL;

python/google/protobuf/pyext/message.cc

@@ -1927,9 +1927,8 @@ PyObject* SetAllowOversizeProtos(PyObject* m, PyObject* arg) {
 }
 
 static PyObject* MergeFromString(CMessage* self, PyObject* arg) {
-  const void* data;
-  Py_ssize_t data_length;
-  if (PyObject_AsReadBuffer(arg, &data, &data_length) < 0) {
+  Py_buffer data;
+  if (PyObject_GetBuffer(arg, &data, PyBUF_SIMPLE) < 0) {
     return NULL;
   }
 
@@ -1942,7 +1941,8 @@ static PyObject* MergeFromString(CMessage* self, PyObject* arg) {
   const char* ptr;
   internal::ParseContext ctx(
       depth, false, &ptr,
-      StringPiece(static_cast<const char*>(data), data_length));
+      StringPiece(static_cast<const char*>(data.buf), data.len));
+  PyBuffer_Release(&data);
   ctx.data().pool = factory->pool->pool;
   ctx.data().factory = factory->message_factory;
 
@@ -1968,9 +1968,9 @@ static PyObject* MergeFromString(CMessage* self, PyObject* arg) {
     // TODO(jieluo): Raise error and return NULL instead.
     // b/27494216
     PyErr_Warn(nullptr, "Unexpected end-group tag: Not all data was converted");
-    return PyInt_FromLong(data_length - ctx.BytesUntilLimit(ptr));
+    return PyInt_FromLong(data.len - ctx.BytesUntilLimit(ptr));
   }
-  return PyInt_FromLong(data_length);
+  return PyInt_FromLong(data.len);
 }
 
 static PyObject* ParseFromString(CMessage* self, PyObject* arg) {

python/tox.ini

@@ -14,10 +14,9 @@ setenv =
 commands =
     python setup.py -q build_py
     python: python setup.py -q build
-    # --warnings_as_errors disabled until we update the Python C extension. See:
-    # https://github.com/protocolbuffers/protobuf/issues/7930
-    # cpp: python setup.py -q build --cpp_implementation --warnings_as_errors --compile_static_extension
-    cpp: python setup.py -q build --cpp_implementation --compile_static_extension
+    # --warnings_as_errors disabled for Python 2.7 because _POSIX_C_SOURCE and _XOPEN_SOURCE are redefined
+    py27-cpp: python setup.py -q build --cpp_implementation --compile_static_extension
+    py{33,34,35,36,37,38,39}-cpp: python setup.py -q build --cpp_implementation --warnings_as_errors --compile_static_extension
     python: python setup.py -q test -q
     cpp: python setup.py -q test -q --cpp_implementation
     python: python setup.py -q test_conformance