[pypy-commit] pypy hpy: (antocuni, ronan) Setup the infrastructure to compile and test HPy modules
rlamy
pypy.commits at gmail.com
Fri Nov 15 14:58:40 EST 2019
Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: hpy
Changeset: r98059:19d6e8fbace2
Date: 2019-11-15 20:57 +0100
http://bitbucket.org/pypy/pypy/changeset/19d6e8fbace2/
Log: (antocuni, ronan) Setup the infrastructure to compile and test HPy
modules
Vendor files from pyhandle/hpy inside
pypy/module/hpy_universal/test/_vendored with some temporary
changes.
diff --git a/pypy/module/hpy_universal/test/__init__.py b/pypy/module/hpy_universal/test/__init__.py
new file mode 100644
diff --git a/pypy/module/hpy_universal/test/_vendored/__init__.py b/pypy/module/hpy_universal/test/_vendored/__init__.py
new file mode 100644
diff --git a/pypy/module/hpy_universal/test/_vendored/include/cpython/hpy.h b/pypy/module/hpy_universal/test/_vendored/include/cpython/hpy.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/test/_vendored/include/cpython/hpy.h
@@ -0,0 +1,142 @@
+#ifndef HPy_CPYTHON_H
+#define HPy_CPYTHON_H
+
+
+
+
+/* XXX: it would be nice if we could include hpy.h WITHOUT bringing in all the
+ stuff from Python.h, to make sure that people don't use the CPython API by
+ mistake. How to achieve it, though? */
+
+/* XXX: should we:
+ * - enforce PY_SSIZE_T_CLEAN in hpy
+ * - make it optional
+ * - make it the default but give a way to disable it?
+ */
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+
+#ifdef __GNUC__
+#define HPyAPI_FUNC(restype) __attribute__((unused)) static inline restype
+#else
+#define HPyAPI_FUNC(restype) static inline restype
+#endif
+
+
+typedef struct { PyObject *_o; } HPy;
+typedef long HPyContext;
+
+HPyAPI_FUNC(HPyContext)
+_HPyGetContext(void) {
+ return 42;
+}
+
+/* For internal usage only. These should be #undef at the end of this header.
+ If you need to convert HPy to PyObject* and vice-versa, you should use the
+ official way to do it (not implemented yet :)
+*/
+#define _h2py(x) (x._o)
+#define _py2h(o) ((HPy){o})
+
+
+#define HPy_NULL ((HPy){NULL})
+#define HPy_IsNull(x) ((x)._o == NULL)
+
+HPyAPI_FUNC(HPy)
+HPyNone_Get(HPyContext ctx)
+{
+ Py_INCREF(Py_None);
+ return _py2h(Py_None);
+}
+
+HPyAPI_FUNC(HPy)
+HPy_Dup(HPyContext ctx, HPy handle)
+{
+ Py_XINCREF(_h2py(handle));
+ return handle;
+}
+
+HPyAPI_FUNC(void)
+HPy_Close(HPyContext ctx, HPy handle)
+{
+ Py_XDECREF(_h2py(handle));
+}
+
+/* moduleobject.h */
+typedef PyModuleDef HPyModuleDef;
+#define HPyModuleDef_HEAD_INIT PyModuleDef_HEAD_INIT
+
+HPyAPI_FUNC(HPy)
+HPyModule_Create(HPyContext ctx, HPyModuleDef *mdef) {
+ return _py2h(PyModule_Create(mdef));
+}
+
+#define HPy_MODINIT(modname) \
+ static HPy init_##modname##_impl(HPyContext ctx); \
+ PyMODINIT_FUNC \
+ PyInit_##modname(void) \
+ { \
+ return _h2py(init_##modname##_impl(_HPyGetContext())); \
+ }
+
+/* methodobject.h */
+typedef PyMethodDef HPyMethodDef;
+
+
+/* function declaration */
+
+#define HPy_FUNCTION(NAME) \
+ static HPy NAME##_impl(HPyContext, HPy, HPy); \
+ static PyObject* NAME(PyObject *self, PyObject *args) \
+ { \
+ return _h2py(NAME##_impl(_HPyGetContext(), _py2h(self), _py2h(args)));\
+ }
+
+
+HPyAPI_FUNC(int)
+HPyArg_ParseTuple(HPyContext ctx, HPy args, const char *fmt, ...)
+{
+ va_list vl;
+ va_start(vl, fmt);
+ int res = PyArg_VaParse(_h2py(args), fmt, vl);
+ va_end(vl);
+ /* XXX incref all returned 'PyObject*' */
+ return res;
+}
+
+
+HPyAPI_FUNC(HPy)
+HPyLong_FromLong(HPyContext ctx, long v)
+{
+ return _py2h(PyLong_FromLong(v));
+}
+
+HPyAPI_FUNC(HPy)
+HPyNumber_Add(HPyContext ctx, HPy x, HPy y)
+{
+ return _py2h(PyNumber_Add(_h2py(x), _h2py(y)));
+}
+
+HPyAPI_FUNC(HPy)
+HPyUnicode_FromString(HPyContext ctx, const char *utf8)
+{
+ return _py2h(PyUnicode_FromString(utf8));
+}
+
+HPyAPI_FUNC(HPy)
+HPy_FromPyObject(HPyContext ctx, PyObject *obj)
+{
+ Py_XINCREF(obj);
+ return _py2h(obj);
+}
+
+HPyAPI_FUNC(PyObject *)
+HPy_AsPyObject(HPyContext ctx, HPy h)
+{
+ PyObject *result = _h2py(h);
+ Py_XINCREF(result);
+ return result;
+}
+
+
+#endif /* !HPy_CPYTHON_H */
diff --git a/pypy/module/hpy_universal/test/_vendored/include/hpy.h b/pypy/module/hpy_universal/test/_vendored/include/hpy.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/test/_vendored/include/hpy.h
@@ -0,0 +1,10 @@
+#ifndef HPy_H
+#define HPy_H
+
+#ifdef HPY_UNIVERSAL_ABI
+# include "universal/hpy.h"
+#else
+# include "cpython/hpy.h"
+#endif
+
+#endif /* HPy_H */
diff --git a/pypy/module/hpy_universal/test/_vendored/include/universal/autogen_ctx.h b/pypy/module/hpy_universal/test/_vendored/include/universal/autogen_ctx.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/test/_vendored/include/universal/autogen_ctx.h
@@ -0,0 +1,24 @@
+
+/*
+ DO NOT EDIT THIS FILE!
+
+ This file is automatically generated by tools/autogen.py from tools/public_api.h.
+ Run this to regenerate:
+ make autogen
+
+*/
+
+struct _HPyContext_s {
+ int ctx_version;
+ HPy (*ctx_Module_Create)(HPyContext ctx, HPyModuleDef *def);
+ HPy (*ctx_None_Get)(HPyContext ctx);
+ HPy (*ctx_Dup)(HPyContext ctx, HPy h);
+ void (*ctx_Close)(HPyContext ctx, HPy h);
+ HPy (*ctx_Long_FromLong)(HPyContext ctx, long value);
+ int (*ctx_Arg_ParseTuple)(HPyContext ctx, HPy args, const char *fmt, va_list _vl);
+ HPy (*ctx_Number_Add)(HPyContext ctx, HPy x, HPy y);
+ HPy (*ctx_Unicode_FromString)(HPyContext ctx, const char *utf8);
+ HPy (*ctx_FromPyObject)(HPyContext ctx, struct _object *obj);
+ struct _object *(*ctx_AsPyObject)(HPyContext ctx, HPy h);
+ struct _object *(*ctx_CallRealFunctionFromTrampoline)(HPyContext ctx, struct _object *self, struct _object *args, HPyCFunction func);
+};
diff --git a/pypy/module/hpy_universal/test/_vendored/include/universal/autogen_func.h b/pypy/module/hpy_universal/test/_vendored/include/universal/autogen_func.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/test/_vendored/include/universal/autogen_func.h
@@ -0,0 +1,54 @@
+
+/*
+ DO NOT EDIT THIS FILE!
+
+ This file is automatically generated by tools/autogen.py from tools/public_api.h.
+ Run this to regenerate:
+ make autogen
+
+*/
+
+static inline HPy HPyModule_Create(HPyContext ctx, HPyModuleDef *def) {
+ return ctx->ctx_Module_Create ( ctx, def );
+}
+
+static inline HPy HPyNone_Get(HPyContext ctx) {
+ return ctx->ctx_None_Get ( ctx );
+}
+
+static inline HPy HPy_Dup(HPyContext ctx, HPy h) {
+ return ctx->ctx_Dup ( ctx, h );
+}
+
+static inline void HPy_Close(HPyContext ctx, HPy h) {
+ ctx->ctx_Close ( ctx, h );
+}
+
+static inline HPy HPyLong_FromLong(HPyContext ctx, long value) {
+ return ctx->ctx_Long_FromLong ( ctx, value );
+}
+
+static inline int HPyArg_ParseTuple(HPyContext ctx, HPy args, const char *fmt, ...) {
+ va_list _vl; va_start(_vl, fmt); int _res = ctx->ctx_Arg_ParseTuple ( ctx, args, fmt, _vl ); va_end(_vl); return _res;
+}
+
+static inline HPy HPyNumber_Add(HPyContext ctx, HPy x, HPy y) {
+ return ctx->ctx_Number_Add ( ctx, x, y );
+}
+
+static inline HPy HPyUnicode_FromString(HPyContext ctx, const char *utf8) {
+ return ctx->ctx_Unicode_FromString ( ctx, utf8 );
+}
+
+static inline HPy HPy_FromPyObject(HPyContext ctx, struct _object *obj) {
+ return ctx->ctx_FromPyObject ( ctx, obj );
+}
+
+static inline struct _object *HPy_AsPyObject(HPyContext ctx, HPy h) {
+ return ctx->ctx_AsPyObject ( ctx, h );
+}
+
+static inline struct _object *_HPy_CallRealFunctionFromTrampoline(HPyContext ctx, struct _object *self, struct _object *args, HPyCFunction func) {
+ return ctx->ctx_CallRealFunctionFromTrampoline ( ctx, self, args, func );
+}
+
diff --git a/pypy/module/hpy_universal/test/_vendored/include/universal/hpy.h b/pypy/module/hpy_universal/test/_vendored/include/universal/hpy.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/test/_vendored/include/universal/hpy.h
@@ -0,0 +1,79 @@
+#ifndef HPy_UNIVERSAL_H
+#define HPy_UNIVERSAL_H
+
+#include <stdlib.h>
+#include <stdint.h>
+#include <stdarg.h>
+
+typedef intptr_t HPy_ssize_t;
+typedef struct { HPy_ssize_t _i; } HPy;
+
+typedef struct _HPyContext_s *HPyContext;
+typedef HPy (*HPyCFunction)(HPyContext, HPy self, HPy args);
+struct _object; /* that's PyObject inside CPython */
+typedef struct _object *(*_HPy_CPyCFunction)(struct _object *self,
+ struct _object *args);
+
+#define HPy_NULL ((HPy){0})
+#define HPy_IsNull(x) ((x)._i == 0)
+
+typedef void (*_HPyMethodPairFunc)(HPyCFunction *out_func,
+ _HPy_CPyCFunction *out_trampoline);
+
+typedef struct {
+ const char *ml_name; /* The name of the built-in function/method */
+ _HPyMethodPairFunc ml_meth; /* see HPy_FUNCTION() */
+ int ml_flags; /* Combination of METH_xxx flags, which mostly
+ describe the args expected by the C func */
+ const char *ml_doc; /* The __doc__ attribute, or NULL */
+} HPyMethodDef;
+
+
+#define HPyModuleDef_HEAD_INIT NULL
+
+typedef struct {
+ void *dummy; // this is needed because we put a comma after HPyModuleDef_HEAD_INIT :(
+ const char* m_name;
+ const char* m_doc;
+ HPy_ssize_t m_size;
+ HPyMethodDef *m_methods;
+} HPyModuleDef;
+
+#define HPy_MODINIT(modname) \
+ HPyContext _ctx_for_trampolines; \
+ static HPy init_##modname##_impl(HPyContext ctx); \
+ HPy HPyInit_##modname(HPyContext ctx) \
+ { \
+ _ctx_for_trampolines = ctx; \
+ return init_##modname##_impl(ctx); \
+ }
+
+#include "autogen_ctx.h"
+#include "autogen_func.h"
+
+extern HPyContext _ctx_for_trampolines;
+
+
+#define HPy_FUNCTION(fnname) \
+ static HPy fnname##_impl(HPyContext ctx, HPy self, HPy args); \
+ static struct _object * \
+ fnname##_trampoline(struct _object *self, struct _object *args) \
+ { \
+ return _HPy_CallRealFunctionFromTrampoline( \
+ _ctx_for_trampolines, self, args, fnname##_impl); \
+ } \
+ static void \
+ fnname(HPyCFunction *out_func, _HPy_CPyCFunction *out_trampoline) \
+ { \
+ *out_func = fnname##_impl; \
+ *out_trampoline = fnname##_trampoline; \
+ }
+
+#define METH_VARARGS 0x0001
+#define METH_KEYWORDS 0x0002
+/* METH_NOARGS and METH_O must not be combined with the flags above. */
+#define METH_NOARGS 0x0004
+#define METH_O 0x0008
+
+
+#endif /* HPy_UNIVERSAL_H */
diff --git a/pypy/module/hpy_universal/test/_vendored/support.py b/pypy/module/hpy_universal/test/_vendored/support.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/test/_vendored/support.py
@@ -0,0 +1,183 @@
+import os, sys
+import pytest
+import re
+#import importlib.util
+#from importlib.machinery import ExtensionFileLoader
+
+THIS_DIR = os.path.dirname(__file__)
+INCLUDE_DIR = os.path.join(THIS_DIR, '../hpy-api/include')
+
+
+r_marker_init = re.compile(r"\s*@INIT\s*$")
+r_marker_export = re.compile(r"\s*@EXPORT\s+(\w+)\s+(METH_\w+)\s*$")
+
+INIT_TEMPLATE = """
+static HPyMethodDef MyTestMethods[] = {
+ %(methods)s
+ {NULL, NULL, 0, NULL}
+};
+
+static HPyModuleDef moduledef = {
+ HPyModuleDef_HEAD_INIT,
+ .m_name = "%(name)s",
+ .m_doc = "some test for hpy",
+ .m_size = -1,
+ .m_methods = MyTestMethods
+};
+
+HPy_MODINIT(%(name)s)
+static HPy init_%(name)s_impl(HPyContext ctx)
+{
+ HPy m;
+ m = HPyModule_Create(ctx, &moduledef);
+ if (HPy_IsNull(m))
+ return HPy_NULL;
+ return m;
+}
+"""
+
+
+def expand_template(source_template, name):
+ method_table = []
+ expanded_lines = ['#include <hpy.h>']
+ for line in source_template.split('\n'):
+ match = r_marker_init.match(line)
+ if match:
+ exp = INIT_TEMPLATE % {
+ 'methods': '\n '.join(method_table),
+ 'name': name}
+ method_table = None # don't fill it any more
+ expanded_lines.append(exp)
+ continue
+
+ match = r_marker_export.match(line)
+ if match:
+ ml_name, ml_flags = match.group(1), match.group(2)
+ method_table.append('{"%s", %s, %s, NULL},' % (
+ ml_name, ml_name, ml_flags))
+ continue
+
+ expanded_lines.append(line)
+ return '\n'.join(expanded_lines)
+
+
+#class HPyLoader(ExtensionFileLoader):
+# def create_module(self, spec):
+# import hpy_universal
+# return hpy_universal.load(spec.origin, "HPyInit_" + spec.name)
+
+class ExtensionCompiler:
+ def __init__(self, tmpdir, abimode):
+ self.tmpdir = tmpdir
+ self.abimode = abimode
+
+ def make_module(self, source_template, name):
+ universal_mode = self.abimode == 'universal'
+ source = expand_template(source_template, name)
+ filename = self.tmpdir.join(name + '.c')
+ filename.write(source)
+ #
+ ext = get_extension(str(filename), name, include_dirs=[INCLUDE_DIR],
+ extra_compile_args=['-Wfatal-errors'])
+ so_filename = c_compile(str(self.tmpdir), ext, compiler_verbose=False,
+ universal_mode=universal_mode)
+ #
+ if universal_mode:
+ loader = HPyLoader(name, so_filename)
+ spec = importlib.util.spec_from_loader(name, loader)
+ else:
+ spec = importlib.util.spec_from_file_location(name, so_filename)
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[name] = module
+ spec.loader.exec_module(module)
+ return module
+
+
+ at pytest.mark.usefixtures('initargs')
+class HPyTest:
+ @pytest.fixture()
+ def initargs(self, compiler):
+ self.compiler = compiler
+
+ def make_module(self, source_template, name='mytest'):
+ return self.compiler.make_module(source_template, name)
+
+
+# the few functions below are copied and adapted from cffi/ffiplatform.py
+
+def get_extension(srcfilename, modname, sources=(), **kwds):
+ from distutils.core import Extension
+ allsources = [srcfilename]
+ for src in sources:
+ allsources.append(os.path.normpath(src))
+ return Extension(name=modname, sources=allsources, **kwds)
+
+def c_compile(tmpdir, ext, compiler_verbose=0, debug=None,
+ universal_mode=False):
+ """Compile a C extension module using distutils."""
+
+ saved_environ = os.environ.copy()
+ try:
+ outputfilename = _build(tmpdir, ext, compiler_verbose, debug,
+ universal_mode)
+ outputfilename = os.path.abspath(outputfilename)
+ finally:
+ # workaround for a distutils bugs where some env vars can
+ # become longer and longer every time it is used
+ for key, value in saved_environ.items():
+ if os.environ.get(key) != value:
+ os.environ[key] = value
+ return outputfilename
+
+def _build(tmpdir, ext, compiler_verbose=0, debug=None, universal_mode=False):
+ # XXX compact but horrible :-(
+ from distutils.core import Distribution
+ import distutils.errors, distutils.log
+ #
+ dist = Distribution({'ext_modules': [ext]})
+ dist.parse_config_files()
+ options = dist.get_option_dict('build_ext')
+ if debug is None:
+ debug = sys.flags.debug
+ options['debug'] = ('ffiplatform', debug)
+ options['force'] = ('ffiplatform', True)
+ options['build_lib'] = ('ffiplatform', tmpdir)
+ options['build_temp'] = ('ffiplatform', tmpdir)
+ #
+ old_level = distutils.log.set_threshold(0) or 0
+ try:
+ distutils.log.set_verbosity(compiler_verbose)
+ if universal_mode:
+ cmd_obj = dist.get_command_obj('build_ext')
+ cmd_obj.finalize_options()
+ soname = _build_universal(tmpdir, ext, cmd_obj.include_dirs)
+ else:
+ dist.run_command('build_ext')
+ cmd_obj = dist.get_command_obj('build_ext')
+ [soname] = cmd_obj.get_outputs()
+ finally:
+ distutils.log.set_threshold(old_level)
+ #
+ return soname
+
+def _build_universal(tmpdir, ext, cpython_include_dirs):
+ from distutils.ccompiler import new_compiler, get_default_compiler
+ from distutils.sysconfig import customize_compiler
+
+ compiler = new_compiler(get_default_compiler())
+ customize_compiler(compiler)
+
+ include_dirs = ext.include_dirs + cpython_include_dirs
+ objects = compiler.compile(ext.sources,
+ output_dir=tmpdir,
+ macros=[('HPY_UNIVERSAL_ABI', None)],
+ include_dirs=include_dirs)
+
+ filename = ext.name + '.hpy.so'
+ compiler.link(compiler.SHARED_LIBRARY,
+ objects,
+ filename,
+ tmpdir
+ # export_symbols=...
+ )
+ return os.path.join(tmpdir, filename)
diff --git a/pypy/module/hpy_universal/test/support.py b/pypy/module/hpy_universal/test/support.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/test/support.py
@@ -0,0 +1,49 @@
+import py
+import pytest
+from pypy.interpreter.gateway import interp2app, unwrap_spec
+
+from rpython.tool.udir import udir
+from ._vendored import support as _support
+
+INCLUDE_DIR = str(py.path.local(__file__).dirpath().join('_vendored/include'))
+
+class ExtensionCompiler(object):
+ def __init__(self, base_dir):
+ self.base_dir = base_dir
+
+ def get_builddir(self, name='mytest'):
+ builddir = py.path.local.make_numbered_dir(
+ rootdir=py.path.local(self.base_dir),
+ prefix=name + '-',
+ keep=0) # keep everything
+ return builddir
+
+
+class HPyTest(object):
+ def setup_class(cls):
+ if cls.runappdirect:
+ pytest.skip()
+ cls.compiler = ExtensionCompiler(udir)
+
+ @unwrap_spec(source_template='text', name='text')
+ def descr_make_module(space, source_template, name='mytest'):
+ source = _support.expand_template(source_template, name)
+ tmpdir = cls.compiler.get_builddir()
+ filename = tmpdir.join(name+ '.c')
+ filename.write(source)
+ #
+ ext = _support.get_extension(str(filename), name, include_dirs=[INCLUDE_DIR],
+ extra_compile_args=['-Wfatal-errors'])
+ so_filename = _support.c_compile(str(tmpdir), ext, compiler_verbose=False,
+ universal_mode=True)
+ #
+ w_mod = space.appexec(
+ [space.newtext(so_filename), space.newtext(name)],
+ """(path, modname):
+ from hpy_universal import load
+ return load(path, 'HPyInit_' + modname)
+ """
+ )
+ return w_mod
+ cls.w_make_module = cls.space.wrap(interp2app(descr_make_module))
+
diff --git a/pypy/module/hpy_universal/test/test_basic.py b/pypy/module/hpy_universal/test/test_basic.py
--- a/pypy/module/hpy_universal/test/test_basic.py
+++ b/pypy/module/hpy_universal/test/test_basic.py
@@ -1,4 +1,17 @@
-class AppTestBasic(object):
+from .support import HPyTest
+
+class AppTestBasic(HPyTest):
spaceconfig = {'usemodules': ['hpy_universal']}
def test_import(self):
import hpy_universal
+
+ def test_empty_module(self):
+ import sys
+ mod = self.make_module("""
+ @INIT
+ """)
+ assert type(mod) is type(sys)
+ assert mod.__loader__.name == 'mytest'
+ assert mod.__spec__.loader is mod.__loader__
+ assert mod.__file__
+
More information about the pypy-commit
mailing list