[pypy-commit] pypy py3.5: Implement main part of PEP 489 (Multi-phase extension module initialization).
mjacob
pypy.commits at gmail.com
Sun Mar 26 14:22:26 EDT 2017
Author: Manuel Jacob <me at manueljacob.de>
Branch: py3.5
Changeset: r90812:76f032290171
Date: 2017-03-26 20:22 +0200
http://bitbucket.org/pypy/pypy/changeset/76f032290171/
Log: Implement main part of PEP 489 (Multi-phase extension module
initialization).
This also removes the cpyext.load_module() app-level function, which
is more difficult to support with the new multi-phase initialization
and equivalent in functionality to imp.load_dynamic(). I've started
a thread on the mailing list whether this function is worth re-
adding.
diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py
--- a/pypy/module/cpyext/__init__.py
+++ b/pypy/module/cpyext/__init__.py
@@ -4,7 +4,6 @@
class Module(MixedModule):
interpleveldefs = {
- 'load_module': 'api.load_extension_module',
}
appleveldefs = {
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -565,7 +565,7 @@
'PyUnicode_FromFormat', 'PyUnicode_FromFormatV', 'PyUnicode_AsWideCharString',
'PyUnicode_GetSize', 'PyUnicode_GetLength',
'PyModule_AddObject', 'PyModule_AddIntConstant', 'PyModule_AddStringConstant',
- 'PyModule_GetDef',
+ 'PyModule_GetDef', 'PyModuleDef_Init',
'Py_BuildValue', 'Py_VaBuildValue', 'PyTuple_Pack',
'_PyArg_Parse_SizeT', '_PyArg_ParseTuple_SizeT',
'_PyArg_ParseTupleAndKeywords_SizeT', '_PyArg_VaParse_SizeT',
@@ -1489,8 +1489,7 @@
copy_header_files(cts, trunk_include, use_micronumpy)
- at unwrap_spec(path='fsencode', name='text')
-def load_extension_module(space, path, name):
+def create_extension_module(space, w_spec):
# note: this is used both to load CPython-API-style C extension
# modules (cpyext) and to load CFFI-style extension modules
# (_cffi_backend). Any of the two can be disabled at translation
@@ -1498,6 +1497,9 @@
# order of things here.
from rpython.rlib import rdynload
+ name = space.text_w(space.getattr(w_spec, space.newtext("name")))
+ path = space.text_w(space.getattr(w_spec, space.newtext("origin")))
+
if os.sep not in path:
path = os.curdir + os.sep + path # force a '/' in the path
basename = name.split('.')[-1]
@@ -1535,7 +1537,7 @@
except KeyError:
pass
else:
- return load_cpyext_module(space, name, path, dll, initptr)
+ return create_cpyext_module(space, w_spec, name, path, dll, initptr)
if look_for is not None:
look_for += ' or ' + also_look_for
else:
@@ -1549,8 +1551,9 @@
initfunctype = lltype.Ptr(lltype.FuncType([], PyObject))
-def load_cpyext_module(space, name, path, dll, initptr):
+def create_cpyext_module(space, w_spec, name, path, dll, initptr):
from rpython.rlib import rdynload
+ from pypy.module.cpyext.pyobject import get_w_obj_and_decref
space.getbuiltinmodule("cpyext") # mandatory to init cpyext
state = space.fromcache(State)
@@ -1562,25 +1565,54 @@
state.package_context = name, path
try:
initfunc = rffi.cast(initfunctype, initptr)
- w_mod = generic_cpy_call(space, initfunc)
+ initret = generic_cpy_call_dont_convert_result(space, initfunc)
state.check_and_raise_exception()
+ if not initret.c_ob_type:
+ raise oefmt(space.w_SystemError,
+ "init function of %s returned uninitialized object",
+ name)
+ # This should probably compare by identity with PyModuleDef_Type from
+ # modsupport.c, but I didn't find a way to do that.
+ tp_name_nonconst = rffi.cast(rffi.CCHARP, initret.c_ob_type.c_tp_name)
+ if rffi.charp2str(tp_name_nonconst) == "moduledef":
+ from pypy.module.cpyext.modsupport import \
+ create_module_from_def_and_spec
+ return create_module_from_def_and_spec(space, initret, w_spec,
+ name)
finally:
state.package_context = old_context
+ w_mod = get_w_obj_and_decref(space, initret)
state.fixup_extension(w_mod, name, path)
return w_mod
+def exec_extension_module(space, w_mod):
+ from pypy.module.cpyext.modsupport import exec_def
+ if not space.config.objspace.usemodules.cpyext:
+ return
+ if not isinstance(w_mod, Module):
+ return
+ space.getbuiltinmodule("cpyext")
+ mod_as_pyobj = rawrefcount.from_obj(PyObject, w_mod)
+ if mod_as_pyobj:
+ return exec_def(space, w_mod, mod_as_pyobj)
+
@specialize.ll()
def generic_cpy_call(space, func, *args):
FT = lltype.typeOf(func).TO
- return make_generic_cpy_call(FT, False)(space, func, *args)
+ return make_generic_cpy_call(FT, False, True)(space, func, *args)
@specialize.ll()
def generic_cpy_call_expect_null(space, func, *args):
FT = lltype.typeOf(func).TO
- return make_generic_cpy_call(FT, True)(space, func, *args)
+ return make_generic_cpy_call(FT, True, True)(space, func, *args)
+
+ at specialize.ll()
+def generic_cpy_call_dont_convert_result(space, func, *args):
+ FT = lltype.typeOf(func).TO
+ return make_generic_cpy_call(FT, False, False)(space, func, *args)
@specialize.memo()
-def make_generic_cpy_call(FT, expect_null):
+def make_generic_cpy_call(FT, expect_null, convert_result):
from pypy.module.cpyext.pyobject import make_ref, from_ref
from pypy.module.cpyext.pyobject import is_pyobj, as_pyobj
from pypy.module.cpyext.pyobject import get_w_obj_and_decref
@@ -1634,8 +1666,9 @@
keepalive_until_here(*keepalives)
if is_PyObject(RESULT_TYPE):
- if not is_pyobj(result):
+ if not convert_result or not is_pyobj(result):
ret = result
+ has_result = bool(ret)
else:
# The object reference returned from a C function
# that is called from Python must be an owned reference
@@ -1644,10 +1677,10 @@
ret = get_w_obj_and_decref(space, result)
else:
ret = None
+ has_result = ret is not None
# Check for exception consistency
has_error = PyErr_Occurred(space) is not None
- has_result = ret is not None
if has_error and has_result:
raise oefmt(space.w_SystemError,
"An exception was set, but function returned a "
diff --git a/pypy/module/cpyext/include/moduleobject.h b/pypy/module/cpyext/include/moduleobject.h
--- a/pypy/module/cpyext/include/moduleobject.h
+++ b/pypy/module/cpyext/include/moduleobject.h
@@ -8,6 +8,8 @@
#include "cpyext_moduleobject.h"
+PyAPI_FUNC(PyObject *) PyModuleDef_Init(struct PyModuleDef*);
+
#ifdef __cplusplus
}
#endif
diff --git a/pypy/module/cpyext/modsupport.py b/pypy/module/cpyext/modsupport.py
--- a/pypy/module/cpyext/modsupport.py
+++ b/pypy/module/cpyext/modsupport.py
@@ -1,7 +1,7 @@
from rpython.rtyper.lltypesystem import rffi, lltype
from pypy.module.cpyext.api import (
cpython_api, METH_STATIC, METH_CLASS, METH_COEXIST, CANNOT_FAIL, cts,
- parse_dir, bootstrap_function)
+ parse_dir, bootstrap_function, generic_cpy_call)
from pypy.module.cpyext.pyobject import PyObject, as_pyobj, make_typedescr
from pypy.interpreter.module import Module
from pypy.module.cpyext.methodobject import (
@@ -14,6 +14,7 @@
cts.parse_header(parse_dir / 'cpyext_moduleobject.h')
PyModuleDef = cts.gettype('PyModuleDef *')
PyModuleObject = cts.gettype('PyModuleObject *')
+PyModuleDef_Slot = cts.gettype('PyModuleDef_Slot')
@bootstrap_function
def init_moduleobject(space):
@@ -64,6 +65,89 @@
return w_mod
+createfunctype = lltype.Ptr(lltype.FuncType([PyObject, PyModuleDef], PyObject))
+execfunctype = lltype.Ptr(lltype.FuncType([PyObject], rffi.INT_real))
+
+
+def create_module_from_def_and_spec(space, moddef, w_spec, name):
+ moddef = rffi.cast(PyModuleDef, moddef)
+ if moddef.c_m_size < 0:
+ raise oefmt(space.w_SystemError,
+ "module %s: m_size may not be negative for multi-phase "
+ "initialization", name)
+ createf = lltype.nullptr(rffi.VOIDP.TO)
+ has_execution_slots = False
+ cur_slot = rffi.cast(rffi.CArrayPtr(PyModuleDef_Slot), moddef.c_m_slots)
+ if cur_slot:
+ while True:
+ slot = rffi.cast(lltype.Signed, cur_slot[0].c_slot)
+ if slot == 0:
+ break
+ elif slot == 1:
+ if createf:
+ raise oefmt(space.w_SystemError,
+ "module %s has multiple create slots", name)
+ createf = cur_slot[0].c_value
+ elif slot < 0 or slot > 2:
+ raise oefmt(space.w_SystemError,
+ "module %s uses unknown slot ID %d", name, slot)
+ else:
+ has_execution_slots = True
+ cur_slot = rffi.ptradd(cur_slot, 1)
+ if createf:
+ createf = rffi.cast(createfunctype, createf)
+ w_mod = generic_cpy_call(space, createf, w_spec, moddef)
+ else:
+ w_mod = Module(space, space.newtext(name))
+ if isinstance(w_mod, Module):
+ mod = rffi.cast(PyModuleObject, as_pyobj(space, w_mod))
+ #mod.c_md_state = None
+ mod.c_md_def = moddef
+ else:
+ if moddef.c_m_size > 0 or moddef.c_m_traverse or moddef.c_m_clear or \
+ moddef.c_m_free:
+ raise oefmt(space.w_SystemError,
+ "module %s is not a module object, but requests "
+ "module state", name)
+ if has_execution_slots:
+ raise oefmt(space.w_SystemError,
+ "module %s specifies execution slots, but did not "
+ "create a ModuleType instance", name)
+ dict_w = {}
+ convert_method_defs(space, dict_w, moddef.c_m_methods, None, w_mod, name)
+ for key, w_value in dict_w.items():
+ space.setattr(w_mod, space.newtext(key), w_value)
+ if moddef.c_m_doc:
+ doc = rffi.charp2str(rffi.cast(rffi.CCHARP, moddef.c_m_doc))
+ space.setattr(w_mod, space.newtext('__doc__'), space.newtext(doc))
+ return w_mod
+
+
+def exec_def(space, w_mod, mod_as_pyobj):
+ from pypy.module.cpyext.pyerrors import PyErr_Occurred
+ mod = rffi.cast(PyModuleObject, mod_as_pyobj)
+ moddef = mod.c_md_def
+ cur_slot = rffi.cast(rffi.CArrayPtr(PyModuleDef_Slot), moddef.c_m_slots)
+ while cur_slot and rffi.cast(lltype.Signed, cur_slot[0].c_slot):
+ if rffi.cast(lltype.Signed, cur_slot[0].c_slot) == 2:
+ execf = rffi.cast(execfunctype, cur_slot[0].c_value)
+ res = generic_cpy_call(space, execf, w_mod)
+ has_error = PyErr_Occurred(space) is not None
+ if rffi.cast(lltype.Signed, res):
+ if has_error:
+ state = space.fromcache(State)
+ state.check_and_raise_exception()
+ else:
+ raise oefmt(space.w_SystemError,
+ "execution of module %S failed without "
+ "setting an exception", w_mod.w_name)
+ if has_error:
+ raise oefmt(space.w_SystemError,
+ "execution of module %S raised unreported "
+ "exception", w_mod.w_name)
+ cur_slot = rffi.ptradd(cur_slot, 1)
+
+
def convert_method_defs(space, dict_w, methods, w_type, w_self=None, name=None):
w_name = space.newtext_or_none(name)
methods = rffi.cast(rffi.CArrayPtr(PyMethodDef), methods)
diff --git a/pypy/module/cpyext/src/modsupport.c b/pypy/module/cpyext/src/modsupport.c
--- a/pypy/module/cpyext/src/modsupport.c
+++ b/pypy/module/cpyext/src/modsupport.c
@@ -602,3 +602,26 @@
}
return ((PyModuleObject *)m)->md_def;
}
+
+PyTypeObject PyModuleDef_Type = {
+ PyVarObject_HEAD_INIT(&PyType_Type, 0)
+ "moduledef", /* tp_name */
+ sizeof(struct PyModuleDef), /* tp_size */
+ 0, /* tp_itemsize */
+};
+
+static Py_ssize_t max_module_number;
+
+PyObject*
+PyModuleDef_Init(struct PyModuleDef* def)
+{
+ if (PyType_Ready(&PyModuleDef_Type) < 0)
+ return NULL;
+ if (def->m_base.m_index == 0) {
+ max_module_number++;
+ Py_REFCNT(def) = 1;
+ Py_TYPE(def) = &PyModuleDef_Type;
+ def->m_base.m_index = max_module_number;
+ }
+ return (PyObject*)def;
+}
diff --git a/pypy/module/cpyext/test/multiphase.c b/pypy/module/cpyext/test/multiphase.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/multiphase.c
@@ -0,0 +1,28 @@
+#include "Python.h"
+
+static struct PyModuleDef multiphase_def;
+
+static PyObject *check_getdef_same(PyObject *self, PyObject *args) {
+ return PyBool_FromLong(PyModule_GetDef(self) == &multiphase_def);
+}
+
+static PyMethodDef methods[] = {
+ {"check_getdef_same", check_getdef_same, METH_NOARGS},
+ {NULL}
+};
+
+static PyModuleDef multiphase_def = {
+ PyModuleDef_HEAD_INIT, /* m_base */
+ "multiphase", /* m_name */
+ "example docstring", /* m_doc */
+ 0, /* m_size */
+ methods, /* m_methods */
+ NULL, /* m_slots */
+ NULL, /* m_traverse */
+ NULL, /* m_clear */
+ NULL, /* m_free */
+};
+
+PyMODINIT_FUNC PyInit_multiphase(void) {
+ return PyModuleDef_Init(&multiphase_def);
+}
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -40,9 +40,11 @@
def load_module(self, mod, name):
space = self.space
- api.load_extension_module(space, mod, name)
- return space.getitem(
- space.sys.get('modules'), space.wrap(name))
+ w_path = space.newtext(mod)
+ w_name = space.newtext(name)
+ return space.appexec([w_name, w_path], '''(name, path):
+ import imp
+ return imp.load_dynamic(name, path)''')
def get_cpyext_info(space):
diff --git a/pypy/module/cpyext/test/test_module.py b/pypy/module/cpyext/test/test_module.py
--- a/pypy/module/cpyext/test/test_module.py
+++ b/pypy/module/cpyext/test/test_module.py
@@ -25,12 +25,85 @@
module = self.import_extension('foo', [
("check_getdef_same", "METH_NOARGS",
"""
- return PyBool_FromLong(PyModule_GetDef(mod_global) == &moduledef);
+ return PyBool_FromLong(PyModule_GetDef(self) == &moduledef);
"""
)], prologue="""
static struct PyModuleDef moduledef;
- static PyObject *mod_global;
- """, more_init="""
- mod_global = mod;
""")
assert module.check_getdef_same()
+
+
+class AppTestMultiPhase(AppTestCpythonExtensionBase):
+ def test_basic(self):
+ from types import ModuleType
+ module = self.import_module(name='multiphase')
+ assert isinstance(module, ModuleType)
+ assert module.__name__ == 'multiphase'
+ assert module.__doc__ == "example docstring"
+
+ def test_getdef(self):
+ from types import ModuleType
+ module = self.import_module(name='multiphase')
+ assert module.check_getdef_same()
+
+ def test_slots(self):
+ from types import ModuleType
+ body = """
+ static PyModuleDef multiphase_def;
+
+ static PyObject* multiphase_create(PyObject *spec, PyModuleDef *def) {
+ PyObject *module = PyModule_New("altname");
+ PyObject_SetAttrString(module, "create_spec", spec);
+ PyObject_SetAttrString(module, "create_def_eq",
+ PyBool_FromLong(def == &multiphase_def));
+ return module;
+ }
+
+ static int multiphase_exec(PyObject* module) {
+ Py_INCREF(Py_True);
+ PyObject_SetAttrString(module, "exec_called", Py_True);
+ return 0;
+ }
+
+ static PyModuleDef_Slot multiphase_slots[] = {
+ {Py_mod_create, multiphase_create},
+ {Py_mod_exec, multiphase_exec},
+ {0, NULL}
+ };
+
+ static PyModuleDef multiphase_def = {
+ PyModuleDef_HEAD_INIT, /* m_base */
+ "multiphase", /* m_name */
+ "example docstring", /* m_doc */
+ 0, /* m_size */
+ NULL, /* m_methods */
+ multiphase_slots, /* m_slots */
+ NULL, /* m_traverse */
+ NULL, /* m_clear */
+ NULL, /* m_free */
+ };
+ """
+ init = """
+ return PyModuleDef_Init(&multiphase_def);
+ """
+ module = self.import_module(name='multiphase', body=body, init=init)
+ assert module.create_spec
+ assert module.create_spec is module.__spec__
+ assert module.create_def_eq
+ assert module.exec_called
+
+ def test_forget_init(self):
+ from types import ModuleType
+ body = """
+ static PyModuleDef multiphase_def = {
+ PyModuleDef_HEAD_INIT, /* m_base */
+ "multiphase", /* m_name */
+ "example docstring", /* m_doc */
+ 0, /* m_size */
+ };
+ """
+ init = """
+ return (PyObject *) &multiphase_def;
+ """
+ raises(SystemError, self.import_module, name='multiphase', body=body,
+ init=init)
diff --git a/pypy/module/imp/__init__.py b/pypy/module/imp/__init__.py
--- a/pypy/module/imp/__init__.py
+++ b/pypy/module/imp/__init__.py
@@ -16,8 +16,8 @@
'init_frozen': 'interp_imp.init_frozen',
'is_builtin': 'interp_imp.is_builtin',
'is_frozen': 'interp_imp.is_frozen',
+ 'exec_dynamic': 'interp_imp.exec_dynamic',
'exec_builtin': 'interp_imp.exec_builtin',
- 'exec_dynamic': 'interp_imp.exec_builtin',
'get_frozen_object': 'interp_imp.get_frozen_object',
'is_frozen_package': 'interp_imp.is_frozen_package',
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -126,11 +126,6 @@
space.sys.setmodule(w_mod)
return w_mod
-def load_c_extension(space, filename, modulename):
- from pypy.module.cpyext.api import load_extension_module
- return load_extension_module(space, filename, modulename)
- # NB. cpyext.api.load_extension_module() can also delegate to _cffi_backend
-
# __________________________________________________________________
#
# import lock, to prevent two threads from running module-level code in
diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py
--- a/pypy/module/imp/interp_imp.py
+++ b/pypy/module/imp/interp_imp.py
@@ -49,11 +49,9 @@
def create_dynamic(space, w_spec, w_file=None):
if not importing.has_so_extension(space):
raise oefmt(space.w_ImportError, "Not implemented")
- w_modulename = space.getattr(w_spec, space.newtext("name"))
- w_path = space.getattr(w_spec, space.newtext("origin"))
- filename = space.fsencode_w(w_path)
- return importing.load_c_extension(space, filename,
- space.text_w(w_modulename))
+ from pypy.module.cpyext.api import create_extension_module
+ # NB. cpyext.api.create_extension_module() can also delegate to _cffi_backend
+ return create_extension_module(space, w_spec)
def create_builtin(space, w_spec):
w_name = space.getattr(w_spec, space.newtext("name"))
@@ -66,8 +64,12 @@
reuse = space.finditem(space.sys.get('modules'), w_name) is not None
return space.getbuiltinmodule(name, force_init=True, reuse=reuse)
+def exec_dynamic(space, w_mod):
+ from pypy.module.cpyext.api import exec_extension_module
+ exec_extension_module(space, w_mod)
+
def exec_builtin(space, w_mod):
- return # Until we really support ModuleDef
+ return
def init_frozen(space, w_name):
return None
More information about the pypy-commit
mailing list