[pypy-commit] pypy py3.6: merge the cpyext-speedup-tests-py36 branch, which greatly spees up running cpyext tests, especially if you run them one at a time, up to 4x faster than before :)
antocuni
pypy.commits at gmail.com
Tue Dec 24 08:14:12 EST 2019
Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3.6
Changeset: r98384:98474e851cf8
Date: 2019-12-24 14:11 +0100
http://bitbucket.org/pypy/pypy/changeset/98474e851cf8/
Log: merge the cpyext-speedup-tests-py36 branch, which greatly spees up
running cpyext tests, especially if you run them one at a time, up
to 4x faster than before :)
diff --git a/lib-python/3/pickle.py b/lib-python/3/pickle.py
--- a/lib-python/3/pickle.py
+++ b/lib-python/3/pickle.py
@@ -1599,6 +1599,15 @@
import doctest
return doctest.testmod()
+# ===== PyPy modification to support pickling cpyext methods =====
+try:
+ import cpyext
+except ImportError:
+ pass
+else:
+ Pickler.dispatch[cpyext.FunctionType] = Pickler.save_global
+# ================= end of PyPy modification ====================
+
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser(
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -7,7 +7,7 @@
from pypy.interpreter.error import OperationError
from rpython.rtyper.lltypesystem import lltype
from pypy.module.cpyext import api
-from pypy.module.cpyext.api import cts
+from pypy.module.cpyext.api import cts, create_extension_module
from pypy.module.cpyext.pyobject import from_ref
from pypy.module.cpyext.state import State
from rpython.tool import leakfinder
@@ -43,13 +43,25 @@
self.space = space
SystemCompilationInfo.__init__(self, *args, **kwargs)
- def load_module(self, mod, name):
+ def load_module(self, mod, name, use_imp=False):
space = self.space
w_path = space.newtext(mod)
w_name = space.newtext(name)
- return space.appexec([w_name, w_path], '''(name, path):
- import imp
- return imp.load_dynamic(name, path)''')
+ if use_imp:
+ # this is VERY slow and should be used only by tests which
+ # actually needs it
+ return space.appexec([w_name, w_path], '''(name, path):
+ import imp
+ return imp.load_dynamic(name, path)''')
+ else:
+ w_spec = space.appexec([w_name, w_path], '''(modname, path):
+ class FakeSpec:
+ name = modname
+ origin = path
+ return FakeSpec
+ ''')
+ w_mod = create_extension_module(space, w_spec)
+ return w_mod
def get_cpyext_info(space):
@@ -154,7 +166,7 @@
del tb
# </types.py>
return [
- buffer,
+ #buffer, ## does not exist on py3k
mmap.mmap,
FunctionType,
CodeType,
@@ -317,13 +329,15 @@
return space.wrap(pydname)
@unwrap_spec(name='text', init='text_or_none', body='text',
- filename='fsencode_or_none', PY_SSIZE_T_CLEAN=bool)
+ filename='fsencode_or_none', PY_SSIZE_T_CLEAN=bool,
+ use_imp=bool)
def import_module(space, name, init=None, body='',
filename=None, w_include_dirs=None,
- PY_SSIZE_T_CLEAN=False):
+ PY_SSIZE_T_CLEAN=False, use_imp=False):
include_dirs = _unwrap_include_dirs(space, w_include_dirs)
w_result = self.sys_info.import_module(
- name, init, body, filename, include_dirs, PY_SSIZE_T_CLEAN)
+ name, init, body, filename, include_dirs, PY_SSIZE_T_CLEAN,
+ use_imp)
self.record_imported_module(name)
return w_result
@@ -452,7 +466,7 @@
NULL, /* m_methods */
};
"""
- foo = self.import_module(name='foo', body=body)
+ foo = self.import_module(name='foo', body=body, use_imp=True)
assert 'foo' in sys.modules
del sys.modules['foo']
import imp
diff --git a/pypy/module/cpyext/test/test_module.py b/pypy/module/cpyext/test/test_module.py
--- a/pypy/module/cpyext/test/test_module.py
+++ b/pypy/module/cpyext/test/test_module.py
@@ -63,14 +63,14 @@
class AppTestMultiPhase(AppTestCpythonExtensionBase):
def test_basic(self):
from types import ModuleType
- module = self.import_module(name='multiphase')
+ module = self.import_module(name='multiphase', use_imp=True)
assert isinstance(module, ModuleType)
assert module.__name__ == 'multiphase'
assert module.__doc__ == "example docstring"
def test_getdef(self):
from types import ModuleType
- module = self.import_module(name='multiphase')
+ module = self.import_module(name='multiphase', use_imp=True)
assert module.check_getdef_same()
def test_slots(self):
@@ -113,7 +113,8 @@
init = """
return PyModuleDef_Init(&multiphase_def);
"""
- module = self.import_module(name='multiphase', body=body, init=init)
+ module = self.import_module(name='multiphase', body=body, init=init,
+ use_imp=True)
assert module.create_spec
assert module.create_spec is module.__spec__
assert module.create_def_eq
@@ -133,7 +134,7 @@
return (PyObject *) &multiphase_def;
"""
raises(SystemError, self.import_module, name='multiphase', body=body,
- init=init)
+ init=init, use_imp=True)
class AppTestMultiPhase2(AppTestCpythonExtensionBase):
def setup_class(cls):
@@ -143,7 +144,7 @@
def test_multiphase2(self):
import sys
from importlib import machinery, util
- module = self.import_module(name=self.name)
+ module = self.import_module(name=self.name, use_imp=True)
finder = machinery.FileFinder(None)
spec = util.find_spec(self.name)
assert spec
@@ -156,7 +157,7 @@
def test_functionality(self):
import types
- module = self.import_module(name=self.name)
+ module = self.import_module(name=self.name, use_imp=True)
assert isinstance(module, types.ModuleType)
ex = module.Example()
assert ex.demo('abcd') == 'abcd'
@@ -173,7 +174,7 @@
def test_reload(self):
import importlib
- module = self.import_module(name=self.name)
+ module = self.import_module(name=self.name, use_imp=True)
ex_class = module.Example
# Simulate what importlib.reload() does, without recomputing the spec
module.__spec__.loader.exec_module(module)
@@ -182,7 +183,7 @@
def w_load_from_name(self, name, origin=None, use_prefix=True):
from importlib import machinery, util
if not origin:
- module = self.import_module(name=self.name)
+ module = self.import_module(name=self.name, use_imp=True)
origin = module.__loader__.path
if use_prefix:
name = '_testmultiphase_' + name
@@ -195,7 +196,7 @@
def test_bad_modules(self):
# XXX: not a very good test, since most internal issues in cpyext
# cause SystemErrors.
- module = self.import_module(name=self.name)
+ module = self.import_module(name=self.name, use_imp=True)
origin = module.__loader__.path
for name in [
'bad_slot_large',
@@ -237,7 +238,7 @@
assert excinfo.value.name == '_testmultiphase_' + name
def test_nonascii(self):
- module = self.import_module(name=self.name)
+ module = self.import_module(name=self.name, use_imp=True)
origin = module.__loader__.path
cases = [
('_testmultiphase_zkou\u0161ka_na\u010dten\xed', 'Czech'),
diff --git a/pypy/tool/cpyext/extbuild.py b/pypy/tool/cpyext/extbuild.py
--- a/pypy/tool/cpyext/extbuild.py
+++ b/pypy/tool/cpyext/extbuild.py
@@ -57,7 +57,7 @@
return str(pydname)
def import_module(self, name, init=None, body='', filename=None,
- include_dirs=None, PY_SSIZE_T_CLEAN=False):
+ include_dirs=None, PY_SSIZE_T_CLEAN=False, use_imp=False):
"""
init specifies the overall template of the module.
@@ -81,7 +81,7 @@
kwds = dict(source_files=[filename])
mod = self.compile_extension_module(
name, include_dirs=include_dirs, **kwds)
- return self.load_module(mod, name)
+ return self.load_module(mod, name, use_imp)
def import_extension(self, modname, functions, prologue="",
include_dirs=None, more_init="", PY_SSIZE_T_CLEAN=False):
@@ -99,7 +99,8 @@
class ExtensionCompiler(SystemCompilationInfo):
"""Extension compiler for appdirect mode"""
- def load_module(space, mod, name):
+ def load_module(space, mod, name, use_imp=False):
+ # use_imp is ignored, it is useful only for non-appdirect mode
import imp
return imp.load_dynamic(name, mod)
More information about the pypy-commit
mailing list