Fix Python 3.4 cpp implementation

Fixes the ScalarMapContainer/MessageMapContainer implementations on
Python 3.4, by dynamically allocating their PyTypeObjects using
PyType_FromSpecWithBases, instead of statically allocating them. This is
necessary because Python 3.4+ disallows statically allocating a class
with a dynamically allocated parent.

Signed-off-by: Dan O'Reilly <oreilldf@gmail.com>

python/google/protobuf/pyext/message.cc

@@ -2863,6 +2863,14 @@ bool InitProto2MessageModule(PyObject *m) {
     }
 
     Py_INCREF(mutable_mapping);
+#if PY_MAJOR_VERSION >= 3
+    PyObject* bases = PyTuple_New(1);
+    PyTuple_SET_ITEM(bases, 0, mutable_mapping.get());
+
+    ScalarMapContainer_Type = 
+        PyType_FromSpecWithBases(&ScalarMapContainer_Type_spec, bases);
+    PyModule_AddObject(m, "ScalarMapContainer", ScalarMapContainer_Type);
+#else
     ScalarMapContainer_Type.tp_base =
         reinterpret_cast<PyTypeObject*>(mutable_mapping.get());
 
@@ -2872,6 +2880,7 @@ bool InitProto2MessageModule(PyObject *m) {
 
     PyModule_AddObject(m, "ScalarMapContainer",
                        reinterpret_cast<PyObject*>(&ScalarMapContainer_Type));
+#endif
 
     if (PyType_Ready(&ScalarMapIterator_Type) < 0) {
       return false;
@@ -2880,6 +2889,12 @@ bool InitProto2MessageModule(PyObject *m) {
     PyModule_AddObject(m, "ScalarMapIterator",
                        reinterpret_cast<PyObject*>(&ScalarMapIterator_Type));
 
+
+#if PY_MAJOR_VERSION >= 3
+    MessageMapContainer_Type = 
+        PyType_FromSpecWithBases(&MessageMapContainer_Type_spec, bases);
+    PyModule_AddObject(m, "MessageMapContainer", MessageMapContainer_Type);
+#else
     Py_INCREF(mutable_mapping);
     MessageMapContainer_Type.tp_base =
         reinterpret_cast<PyTypeObject*>(mutable_mapping.get());
@@ -2890,6 +2905,7 @@ bool InitProto2MessageModule(PyObject *m) {
 
     PyModule_AddObject(m, "MessageMapContainer",
                        reinterpret_cast<PyObject*>(&MessageMapContainer_Type));
+#endif
 
     if (PyType_Ready(&MessageMapIterator_Type) < 0) {
       return false;

python/google/protobuf/pyext/message_map_container.cc

@@ -84,7 +84,12 @@ PyObject* NewContainer(CMessage* parent,
     return NULL;
   }
 
+#if PY_MAJOR_VERSION >= 3
+  PyObject* obj = PyType_GenericAlloc(
+        reinterpret_cast<PyTypeObject *>(MessageMapContainer_Type), 0);
+#else
   PyObject* obj = PyType_GenericAlloc(&MessageMapContainer_Type, 0);
+#endif
   if (obj == NULL) {
     return PyErr_Format(PyExc_RuntimeError,
                         "Could not allocate new container.");
@@ -458,44 +463,67 @@ PyObject* IterNext(PyObject* _self) {
 
 }  // namespace message_map_iterator
 
-PyTypeObject MessageMapContainer_Type = {
-  PyVarObject_HEAD_INIT(&PyType_Type, 0)
-  FULL_MODULE_NAME ".MessageMapContainer",  //  tp_name
-  sizeof(MessageMapContainer),         //  tp_basicsize
-  0,                                   //  tp_itemsize
-  message_map_container::Dealloc,      //  tp_dealloc
-  0,                                   //  tp_print
-  0,                                   //  tp_getattr
-  0,                                   //  tp_setattr
-  0,                                   //  tp_compare
-  0,                                   //  tp_repr
-  0,                                   //  tp_as_number
-  0,                                   //  tp_as_sequence
-  &message_map_container::MpMethods,   //  tp_as_mapping
-  0,                                   //  tp_hash
-  0,                                   //  tp_call
-  0,                                   //  tp_str
-  0,                                   //  tp_getattro
-  0,                                   //  tp_setattro
-  0,                                   //  tp_as_buffer
-  Py_TPFLAGS_DEFAULT,                  //  tp_flags
-  "A map container for message",       //  tp_doc
-  0,                                   //  tp_traverse
-  0,                                   //  tp_clear
-  0,                                   //  tp_richcompare
-  0,                                   //  tp_weaklistoffset
-  message_map_container::GetIterator,  //  tp_iter
-  0,                                   //  tp_iternext
-  message_map_container::Methods,      //  tp_methods
-  0,                                   //  tp_members
-  0,                                   //  tp_getset
-  0,                                   //  tp_base
-  0,                                   //  tp_dict
-  0,                                   //  tp_descr_get
-  0,                                   //  tp_descr_set
-  0,                                   //  tp_dictoffset
-  0,                                   //  tp_init
-};
+#if PY_MAJOR_VERSION >= 3
+  static PyType_Slot MessageMapContainer_Type_slots[] = {
+      {Py_tp_dealloc, (void *)message_map_container::Dealloc},
+      {Py_mp_length, (void *)message_map_container::Length},
+      {Py_mp_subscript, (void *)message_map_container::GetItem},
+      {Py_mp_ass_subscript, (void *)message_map_container::SetItem},
+      {Py_tp_methods, (void *)message_map_container::Methods},
+      {Py_tp_iter, (void *)message_map_container::GetIterator},
+      {0, 0}
+  };
+
+  PyType_Spec MessageMapContainer_Type_spec = {
+      FULL_MODULE_NAME ".MessageMapContainer",
+      sizeof(MessageMapContainer),
+      0,
+      Py_TPFLAGS_DEFAULT,
+      MessageMapContainer_Type_slots
+  };
+
+  PyObject *MessageMapContainer_Type;
+
+#else
+  PyTypeObject MessageMapContainer_Type = {
+    PyVarObject_HEAD_INIT(&PyType_Type, 0)
+    FULL_MODULE_NAME ".MessageMapContainer",  //  tp_name
+    sizeof(MessageMapContainer),         //  tp_basicsize
+    0,                                   //  tp_itemsize
+    message_map_container::Dealloc,      //  tp_dealloc
+    0,                                   //  tp_print
+    0,                                   //  tp_getattr
+    0,                                   //  tp_setattr
+    0,                                   //  tp_compare
+    0,                                   //  tp_repr
+    0,                                   //  tp_as_number
+    0,                                   //  tp_as_sequence
+    &message_map_container::MpMethods,   //  tp_as_mapping
+    0,                                   //  tp_hash
+    0,                                   //  tp_call
+    0,                                   //  tp_str
+    0,                                   //  tp_getattro
+    0,                                   //  tp_setattro
+    0,                                   //  tp_as_buffer
+    Py_TPFLAGS_DEFAULT,                  //  tp_flags
+    "A map container for message",       //  tp_doc
+    0,                                   //  tp_traverse
+    0,                                   //  tp_clear
+    0,                                   //  tp_richcompare
+    0,                                   //  tp_weaklistoffset
+    message_map_container::GetIterator,  //  tp_iter
+    0,                                   //  tp_iternext
+    message_map_container::Methods,      //  tp_methods
+    0,                                   //  tp_members
+    0,                                   //  tp_getset
+    0,                                   //  tp_base
+    0,                                   //  tp_dict
+    0,                                   //  tp_descr_get
+    0,                                   //  tp_descr_set
+    0,                                   //  tp_dictoffset
+    0,                                   //  tp_init
+  };
+#endif
 
 PyTypeObject MessageMapIterator_Type = {
   PyVarObject_HEAD_INIT(&PyType_Type, 0)

python/google/protobuf/pyext/message_map_container.h

@@ -89,7 +89,12 @@ struct MessageMapContainer {
   uint64 version;
 };
 
-extern PyTypeObject MessageMapContainer_Type;
+#if PY_MAJOR_VERSION >= 3
+  extern PyObject *MessageMapContainer_Type;
+  extern PyType_Spec MessageMapContainer_Type_spec;
+#else
+  extern PyTypeObject MessageMapContainer_Type;
+#endif
 extern PyTypeObject MessageMapIterator_Type;
 
 namespace message_map_container {

python/google/protobuf/pyext/scalar_map_container.cc

@@ -83,7 +83,12 @@ PyObject *NewContainer(
     return NULL;
   }
 
+#if PY_MAJOR_VERSION >= 3
+  ScopedPyObjectPtr obj(PyType_GenericAlloc(
+        reinterpret_cast<PyTypeObject *>(ScalarMapContainer_Type), 0));
+#else
   ScopedPyObjectPtr obj(PyType_GenericAlloc(&ScalarMapContainer_Type, 0));
+#endif
   if (obj.get() == NULL) {
     return PyErr_Format(PyExc_RuntimeError,
                         "Could not allocate new container.");
@@ -432,44 +437,66 @@ PyObject* IterNext(PyObject* _self) {
 
 }  // namespace scalar_map_iterator
 
-PyTypeObject ScalarMapContainer_Type = {
-  PyVarObject_HEAD_INIT(&PyType_Type, 0)
-  FULL_MODULE_NAME ".ScalarMapContainer",  //  tp_name
-  sizeof(ScalarMapContainer),          //  tp_basicsize
-  0,                                   //  tp_itemsize
-  scalar_map_container::Dealloc,       //  tp_dealloc
-  0,                                   //  tp_print
-  0,                                   //  tp_getattr
-  0,                                   //  tp_setattr
-  0,                                   //  tp_compare
-  0,                                   //  tp_repr
-  0,                                   //  tp_as_number
-  0,                                   //  tp_as_sequence
-  &scalar_map_container::MpMethods,    //  tp_as_mapping
-  0,                                   //  tp_hash
-  0,                                   //  tp_call
-  0,                                   //  tp_str
-  0,                                   //  tp_getattro
-  0,                                   //  tp_setattro
-  0,                                   //  tp_as_buffer
-  Py_TPFLAGS_DEFAULT,                  //  tp_flags
-  "A scalar map container",            //  tp_doc
-  0,                                   //  tp_traverse
-  0,                                   //  tp_clear
-  0,                                   //  tp_richcompare
-  0,                                   //  tp_weaklistoffset
-  scalar_map_container::GetIterator,   //  tp_iter
-  0,                                   //  tp_iternext
-  scalar_map_container::Methods,       //  tp_methods
-  0,                                   //  tp_members
-  0,                                   //  tp_getset
-  0,                                   //  tp_base
-  0,                                   //  tp_dict
-  0,                                   //  tp_descr_get
-  0,                                   //  tp_descr_set
-  0,                                   //  tp_dictoffset
-  0,                                   //  tp_init
-};
+ 
+#if PY_MAJOR_VERSION >= 3
+  static PyType_Slot ScalarMapContainer_Type_slots[] = {
+      {Py_tp_dealloc, (void *)scalar_map_container::Dealloc},
+      {Py_mp_length, (void *)scalar_map_container::Length},
+      {Py_mp_subscript, (void *)scalar_map_container::GetItem},
+      {Py_mp_ass_subscript, (void *)scalar_map_container::SetItem},
+      {Py_tp_methods, (void *)scalar_map_container::Methods},
+      {Py_tp_iter, (void *)scalar_map_container::GetIterator},
+      {0, 0},
+  };
+
+  PyType_Spec ScalarMapContainer_Type_spec = {
+      FULL_MODULE_NAME ".ScalarMapContainer",
+      sizeof(ScalarMapContainer),
+      0,
+      Py_TPFLAGS_DEFAULT,
+      ScalarMapContainer_Type_slots
+  };
+  PyObject *ScalarMapContainer_Type;
+#else
+  PyTypeObject ScalarMapContainer_Type = {
+    PyVarObject_HEAD_INIT(&PyType_Type, 0)
+    FULL_MODULE_NAME ".ScalarMapContainer",  //  tp_name
+    sizeof(ScalarMapContainer),          //  tp_basicsize
+    0,                                   //  tp_itemsize
+    scalar_map_container::Dealloc,       //  tp_dealloc
+    0,                                   //  tp_print
+    0,                                   //  tp_getattr
+    0,                                   //  tp_setattr
+    0,                                   //  tp_compare
+    0,                                   //  tp_repr
+    0,                                   //  tp_as_number
+    0,                                   //  tp_as_sequence
+    &scalar_map_container::MpMethods,    //  tp_as_mapping
+    0,                                   //  tp_hash
+    0,                                   //  tp_call
+    0,                                   //  tp_str
+    0,                                   //  tp_getattro
+    0,                                   //  tp_setattro
+    0,                                   //  tp_as_buffer
+    Py_TPFLAGS_DEFAULT,                  //  tp_flags
+    "A scalar map container",            //  tp_doc
+    0,                                   //  tp_traverse
+    0,                                   //  tp_clear
+    0,                                   //  tp_richcompare
+    0,                                   //  tp_weaklistoffset
+    scalar_map_container::GetIterator,   //  tp_iter
+    0,                                   //  tp_iternext
+    scalar_map_container::Methods,       //  tp_methods
+    0,                                   //  tp_members
+    0,                                   //  tp_getset
+    0,                                   //  tp_base
+    0,                                   //  tp_dict
+    0,                                   //  tp_descr_get
+    0,                                   //  tp_descr_set
+    0,                                   //  tp_dictoffset
+    0,                                   //  tp_init
+  };
+#endif
 
 PyTypeObject ScalarMapIterator_Type = {
   PyVarObject_HEAD_INIT(&PyType_Type, 0)

python/google/protobuf/pyext/scalar_map_container.h

@@ -83,7 +83,12 @@ struct ScalarMapContainer {
   uint64 version;
 };
 
-extern PyTypeObject ScalarMapContainer_Type;
+#if PY_MAJOR_VERSION >= 3
+  extern PyObject *ScalarMapContainer_Type;
+  extern PyType_Spec ScalarMapContainer_Type_spec;
+#else
+  extern PyTypeObject ScalarMapContainer_Type;
+#endif
 extern PyTypeObject ScalarMapIterator_Type;
 
 namespace scalar_map_container {

python/tox.ini

@@ -2,8 +2,7 @@
 envlist =
     # cpp implementation on py34 is currently broken due to
     # changes introduced by http://bugs.python.org/issue22079.
-    #py{26,27,33,34}-{cpp,python}
-    py{26,27,33}-{cpp,python}, py34-{python}
+    py{26,27,33,34}-{cpp,python}
 
 [testenv]
 usedevelop=true

travis.sh

@@ -113,12 +113,14 @@ build_javanano_oracle7() {
 
 internal_install_python_deps() {
   sudo pip install tox
-  # Only install Python2.6 on Linux.
+  # Only install Python2.6/3.x on Linux.
   if [ $(uname -s) == "Linux" ]; then
     sudo apt-get install -y python-software-properties # for apt-add-repository
     sudo apt-add-repository -y ppa:fkrull/deadsnakes
     sudo apt-get update -qq
     sudo apt-get install -y python2.6 python2.6-dev
+    sudo apt-get install -y python3.3 python3.3-dev
+    sudo apt-get install -y python3.4 python3.4-dev
   fi
 }
 
@@ -127,9 +129,9 @@ build_python() {
   internal_build_cpp
   internal_install_python_deps
   cd python
-  # Only test Python 2.6 on Linux
+  # Only test Python 2.6/3.x on Linux
   if [ $(uname -s) == "Linux" ]; then
-    envlist=py26-python,py27-python
+    envlist=py\{26,27,33,34\}-python
   else
     envlist=py27-python
   fi
@@ -143,9 +145,9 @@ build_python_cpp() {
   export LD_LIBRARY_PATH=../src/.libs # for Linux
   export DYLD_LIBRARY_PATH=../src/.libs # for OS X
   cd python
-  # Only test Python 2.6 on Linux
+  # Only test Python 2.6/3.x on Linux
   if [ $(uname -s) == "Linux" ]; then
-    envlist=py26-cpp,py27-cpp
+    envlist=py\{26,27,33,34\}-cpp
   else
     envlist=py27-cpp
   fi