Pickle segfaults with custom type

[Marco Sulla]

Found. I simply forgot:


    if (PyType_Ready(&PyFrozenDictIterKey_Type) < 0) {
        goto fail;
    }

in the frozendict_exec function for the module.

On Fri, 7 Jan 2022 at 20:27, Marco Sulla <Marco.Sulla.Python at gmail.com>
wrote:

> I have a custom implementation of dict using a C extension. All works but
> the pickling of views and iter types. Python segfaults if I try to pickle
> them.
>
> For example, I have:
>
>
> static PyTypeObject PyFrozenDictIterKey_Type = {
>     PyVarObject_HEAD_INIT(NULL, 0)
>     "frozendict.keyiterator",                   /* tp_name */
>     sizeof(dictiterobject),                     /* tp_basicsize */
>     0,                                          /* tp_itemsize */
>     /* methods */
>     (destructor)dictiter_dealloc,               /* tp_dealloc */
>     0,                                          /* tp_vectorcall_offset */
>     0,                                          /* tp_getattr */
>     0,                                          /* tp_setattr */
>     0,                                          /* tp_as_async */
>     0,                                          /* tp_repr */
>     0,                                          /* tp_as_number */
>     0,                                          /* tp_as_sequence */
>     0,                                          /* tp_as_mapping */
>     PyObject_HashNotImplemented,                /* tp_hash */
>     0,                                          /* tp_call */
>     0,                                          /* tp_str */
>     PyObject_GenericGetAttr,                    /* tp_getattro */
>     0,                                          /* tp_setattro */
>     0,                                          /* tp_as_buffer */
>     Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,    /* tp_flags */
>     0,                                          /* tp_doc */
>     (traverseproc)dictiter_traverse,            /* tp_traverse */
>     0,                                          /* tp_clear */
>     0,                                          /* tp_richcompare */
>     0,                                          /* tp_weaklistoffset */
>     PyObject_SelfIter,                          /* tp_iter */
>     (iternextfunc)frozendictiter_iternextkey,   /* tp_iternext */
>     dictiter_methods,                           /* tp_methods */
>     0,
> };
>
> This is the backtrace I get with gdb:
>
> #0  PyObject_Hash (v=0x7f043ce15540 <PyFrozenDictIterKey_Type>) at
> ../cpython_3_10/Objects/object.c:788
> #1  0x000000000048611c in PyDict_GetItemWithError (op=0x7f043e1f4900,
> key=key at entry=0x7f043ce15540 <PyFrozenDictIterKey_Type>)
>     at ../cpython_3_10/Objects/dictobject.c:1520
> #2  0x00007f043ce227f6 in save (self=self at entry=0x7f043d8507d0,
> obj=obj at entry=0x7f043e1fb0b0, pers_save=pers_save at entry=0)
>     at /home/marco/sources/cpython_3_10/Modules/_pickle.c:4381
> #3  0x00007f043ce2534d in dump (self=self at entry=0x7f043d8507d0,
> obj=obj at entry=0x7f043e1fb0b0) at
> /home/marco/sources/cpython_3_10/Modules/_pickle.c:4515
> #4  0x00007f043ce2567f in _pickle_dumps_impl (module=<optimized out>,
> buffer_callback=<optimized out>, fix_imports=<optimized out>,
> protocol=<optimized out>,
>     obj=0x7f043e1fb0b0) at
> /home/marco/sources/cpython_3_10/Modules/_pickle.c:1203
> #5  _pickle_dumps (module=<optimized out>, args=<optimized out>,
> nargs=<optimized out>, kwnames=<optimized out>)
>     at /home/marco/sources/cpython_3_10/Modules/clinic/_pickle.c.h:619
>
> and so on. The problematic part is in the second frame. Indeed the code of
> _pickle.c here is:
>
>
>         reduce_func = PyDict_GetItemWithError(st->dispatch_table,
>                                               (PyObject *)type);
>
> The problem is that type is NULL. It tries to get the attribute tp_hash
> and it segfaults.
>
> I tried to change the header of the type to:
>
> PyVarObject_HEAD_INIT(&PyType_Type, 0)
>
> This way it works but, as known, it does not compile on Windows.
>
> The strange fact is that pickling the main type works, even if the type is
> NULL, as suggested for a custom type. This is the main type:
>
> PyTypeObject PyFrozenDict_Type = {
>     PyVarObject_HEAD_INIT(NULL, 0)
>     "frozendict." FROZENDICT_CLASS_NAME,        /* tp_name */
>     sizeof(PyFrozenDictObject),                 /* tp_basicsize */
>     0,                                          /* tp_itemsize */
>     (destructor)dict_dealloc,                   /* tp_dealloc */
>     0,                                          /* tp_vectorcall_offset */
>     0,                                          /* tp_getattr */
>     0,                                          /* tp_setattr */
>     0,                                          /* tp_as_async */
>     (reprfunc)frozendict_repr,                  /* tp_repr */
>     &frozendict_as_number,                      /* tp_as_number */
>     &dict_as_sequence,                          /* tp_as_sequence */
>     &frozendict_as_mapping,                     /* tp_as_mapping */
>     (hashfunc)frozendict_hash,                  /* tp_hash */
>     0,                                          /* tp_call */
>     0,                                          /* tp_str */
>     PyObject_GenericGetAttr,                    /* tp_getattro */
>     0,                                          /* tp_setattro */
>     0,                                          /* tp_as_buffer */
>     Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC
>         | Py_TPFLAGS_BASETYPE
>         | _Py_TPFLAGS_MATCH_SELF
>         | Py_TPFLAGS_MAPPING,                   /* tp_flags */
>     frozendict_doc,                             /* tp_doc */
>     dict_traverse,                              /* tp_traverse */
>     0,                                          /* tp_clear */
>     dict_richcompare,                           /* tp_richcompare */
>     0,                                          /* tp_weaklistoffset */
>     (getiterfunc)frozendict_iter,               /* tp_iter */
>     0,                                          /* tp_iternext */
>     frozendict_mapp_methods,                    /* tp_methods */
>     0,                                          /* tp_members */
>     0,                                          /* tp_getset */
>     0,                                          /* tp_base */
>     0,                                          /* tp_dict */
>     0,                                          /* tp_descr_get */
>     0,                                          /* tp_descr_set */
>     0,                                          /* tp_dictoffset */
>     0,                                          /* tp_init */
>     PyType_GenericAlloc,                        /* tp_alloc */
>     frozendict_new,                             /* tp_new */
>     PyObject_GC_Del,                            /* tp_free */
>     .tp_vectorcall = frozendict_vectorcall,
> };
>