[pypy-commit] pypy cffi-libs: backport package.py, build_cffi_imports from py3.6
mattip
pypy.commits at gmail.com
Mon May 20 15:22:08 EDT 2019
Author: Matti Picus <matti.picus at gmail.com>
Branch: cffi-libs
Changeset: r96648:8ae1951978c7
Date: 2019-05-20 22:21 +0300
http://bitbucket.org/pypy/pypy/changeset/8ae1951978c7/
Log: backport package.py, build_cffi_imports from py3.6
diff --git a/pypy/tool/build_cffi_imports.py b/pypy/tool/build_cffi_imports.py
--- a/pypy/tool/build_cffi_imports.py
+++ b/pypy/tool/build_cffi_imports.py
@@ -1,11 +1,13 @@
from __future__ import print_function
-import sys, shutil, os
+import sys, shutil, os, tempfile, hashlib
+from os.path import join
class MissingDependenciesError(Exception):
pass
cffi_build_scripts = {
+ "_ssl": "_ssl_build.py",
"sqlite3": "_sqlite3_build.py",
"audioop": "_audioop_build.py",
"tk": "_tkinter/tklib_build.py",
@@ -17,10 +19,125 @@
"xx": None, # for testing: 'None' should be completely ignored
}
-def create_cffi_import_libraries(pypy_c, options, basedir):
+# for distribution, we may want to fetch dependencies not provided by
+# the OS, such as a recent openssl/libressl.
+cffi_dependencies = {
+ '_ssl': ('http://ftp.openbsd.org/pub/OpenBSD/LibreSSL/libressl-2.6.2.tar.gz',
+ 'b029d2492b72a9ba5b5fcd9f3d602c9fd0baa087912f2aaecc28f52f567ec478',
+ ['--without-openssldir']),
+ '_gdbm': ('http://ftp.gnu.org/gnu/gdbm/gdbm-1.13.tar.gz',
+ '9d252cbd7d793f7b12bcceaddda98d257c14f4d1890d851c386c37207000a253',
+ ['--without-readline']),
+}
+
+
+def _unpack_tarfile(filename, extract_dir):
+ """Unpack tar/tar.gz/tar.bz2/tar.xz `filename` to `extract_dir`
+ """
+ import tarfile # late import for breaking circular dependency
+ try:
+ tarobj = tarfile.open(filename)
+ except tarfile.TarError:
+ raise ReadError(
+ "%s is not a compressed or uncompressed tar file" % filename)
+ try:
+ tarobj.extractall(extract_dir)
+ finally:
+ tarobj.close()
+
+def _sha256(filename):
+ dgst = hashlib.sha256()
+
+ with open(filename, 'rb') as fp:
+ dgst.update(fp.read())
+ return dgst.hexdigest()
+
+
+def _build_dependency(name, destdir, patches=[]):
+ import multiprocessing
+ import shutil
+ import subprocess
+
from rpython.tool.runsubprocess import run_subprocess
- shutil.rmtree(str(basedir.join('lib_pypy', '__pycache__')),
+ try:
+ from urllib.request import urlretrieve
+ except ImportError:
+ from urllib import urlretrieve
+
+ try:
+ url, dgst, args = cffi_dependencies[name]
+ except KeyError:
+ return 0, None, None
+
+ archive_dir = os.path.join(tempfile.gettempdir(), 'pypy-archives')
+
+ if not os.path.isdir(archive_dir):
+ os.makedirs(archive_dir)
+
+ archive = os.path.join(archive_dir, url.rsplit('/', 1)[-1])
+
+ # next, fetch the archive to disk, if needed
+ if not os.path.exists(archive) or _sha256(archive) != dgst:
+ print('fetching archive', url, file=sys.stderr)
+ urlretrieve(url, archive)
+
+ # extract the archive into our destination directory
+ print('unpacking archive', archive, file=sys.stderr)
+ _unpack_tarfile(archive, destdir)
+
+ sources = os.path.join(
+ destdir,
+ os.path.basename(archive)[:-7],
+ )
+
+ # apply any patches
+ if patches:
+ for patch in patches:
+ print('applying patch', patch, file=sys.stderr)
+ status, stdout, stderr = run_subprocess(
+ '/usr/bin/patch', ['-p1', '-i', patch], cwd=sources,
+ )
+
+ if status != 0:
+ return status, stdout, stderr
+
+ print('configuring', sources, file=sys.stderr)
+
+ # configure & build it
+ status, stdout, stderr = run_subprocess(
+ './configure',
+ [
+ '--prefix=/usr',
+ '--disable-shared',
+ '--enable-silent-rules',
+ '--disable-dependency-tracking',
+ ] + args,
+ cwd=sources,
+ )
+
+ if status != 0:
+ return status, stdout, stderr
+
+ print('building', sources, file=sys.stderr)
+
+ status, stdout, stderr = run_subprocess(
+ 'make',
+ [
+ '-s', '-j' + str(multiprocessing.cpu_count()),
+ 'install', 'DESTDIR={}/'.format(destdir),
+ ],
+ cwd=sources,
+ )
+
+ return status, stdout, stderr
+
+
+def create_cffi_import_libraries(pypy_c, options, basedir, only=None,
+ embed_dependencies=False):
+ from rpython.tool.runsubprocess import run_subprocess
+
+ shutil.rmtree(str(join(basedir,'lib_pypy','__pycache__')),
ignore_errors=True)
# be sure pip, setuptools are installed in a fresh pypy
# allows proper functioning of cffi on win32 with newer vc compilers
@@ -29,24 +146,72 @@
if status != 0:
status, stdout, stderr = run_subprocess(str(pypy_c), ['-m', 'ensurepip'])
failures = []
- env = os.environ.copy()
- if sys.platform == 'win32':
- env['INCLUDE'] = r'..\externals\include;' + env.get('INCLUDE', '')
- env['LIB'] = r'..\externals\lib;' + env.get('LIB', '')
- env['PATH'] = r'..\externals\bin;' + env.get('PATH', '')
- for key, module in sorted(cffi_build_scripts.items()):
+
+ for key, module in cffi_build_scripts.items():
+ if only and key not in only:
+ print("* SKIPPING", key, '(not specified in --only)')
+ continue
if module is None or getattr(options, 'no_' + key, False):
continue
+ # the key is the module name, has it already been built?
+ status, stdout, stderr = run_subprocess(str(pypy_c), ['-c', 'import %s' % key])
+ if status == 0:
+ print('*', ' %s already built' % key, file=sys.stderr)
+ continue
+
if module.endswith('.py'):
args = [module]
- cwd = str(basedir.join('lib_pypy'))
+ cwd = str(join(basedir,'lib_pypy'))
else:
args = ['-c', 'import ' + module]
cwd = None
+ env = os.environ.copy()
+
print('*', ' '.join(args), file=sys.stderr)
+ if embed_dependencies:
+ curdir = os.path.abspath(os.path.dirname(__file__))
+ destdir = os.path.join(curdir, 'dest')
+
+ shutil.rmtree(destdir, ignore_errors=True)
+ os.makedirs(destdir)
+
+ if key == '_ssl' and sys.platform == 'darwin':
+ # this patch is loosely inspired by an Apple and adds
+ # a fallback to the OS X roots when none are available
+ patches = [
+ os.path.join(curdir,
+ '../../lib_pypy/_cffi_ssl/osx-roots.diff'),
+ ]
+ else:
+ patches = []
+
+ status, stdout, stderr = _build_dependency(key, destdir,
+ patches=patches)
+
+ if status != 0:
+ failures.append((key, module))
+ print("stdout:")
+ print(stdout.decode('utf-8'))
+ print("stderr:")
+ print(stderr.decode('utf-8'))
+ continue
+
+ env['CPPFLAGS'] = \
+ '-I{}/usr/include {}'.format(destdir, env.get('CPPFLAGS', ''))
+ env['LDFLAGS'] = \
+ '-L{}/usr/lib {}'.format(destdir, env.get('LDFLAGS', ''))
+
+ if key == '_ssl' and sys.platform == 'darwin':
+ # needed for our roots patch
+ env['LDFLAGS'] += ' -framework CoreFoundation -framework Security'
+ elif sys.platform == 'win32':
+ env['INCLUDE'] = r'..\externals\include;' + env.get('INCLUDE', '')
+ env['LIB'] = r'..\externals\lib;' + env.get('LIB', '')
+ env['PATH'] = r'..\externals\bin;' + env.get('PATH', '')
+
try:
status, stdout, stderr = run_subprocess(str(pypy_c), args,
- cwd=cwd, env=env)
+ cwd=cwd, env=env)
if status != 0:
print(stdout, stderr, file=sys.stderr)
failures.append((key, module))
@@ -56,6 +221,7 @@
return failures
if __name__ == '__main__':
+ import argparse
if '__pypy__' not in sys.builtin_module_names:
print('Call with a pypy interpreter', file=sys.stderr)
sys.exit(1)
@@ -64,21 +230,35 @@
base_dir = os.path.dirname(os.path.dirname(tool_dir))
sys.path.insert(0, base_dir)
- import py
-
class Options(object):
pass
- exename = py.path.local(sys.executable)
+ parser = argparse.ArgumentParser(description='Build all cffi backends in lib_pypy')
+ parser.add_argument('--exefile', dest='exefile', default=sys.executable,
+ help='instead of executing sys.executable' \
+ ' you can specify an alternative pypy vm here')
+ parser.add_argument('--only', dest='only', default=None,
+ help='Only build the modules delimited by a colon. E.g. _ssl,sqlite')
+ parser.add_argument('--embed-dependencies', dest='embed_dependencies', action='store_true',
+ help='embed dependencies for distribution')
+ args = parser.parse_args()
+
+ exename = join(os.getcwd(), args.exefile)
basedir = exename
- while not basedir.join('include').exists():
- _basedir = basedir.dirpath()
+
+ while not os.path.exists(join(basedir,'include')):
+ _basedir = os.path.dirname(basedir)
if _basedir == basedir:
raise ValueError('interpreter %s not inside pypy repo',
str(exename))
basedir = _basedir
options = Options()
- failures = create_cffi_import_libraries(exename, options, basedir)
+ if args.only is None:
+ only = None
+ else:
+ only = set(args.only.split(','))
+ failures = create_cffi_import_libraries(exename, options, basedir, only=only,
+ embed_dependencies=args.embed_dependencies)
if len(failures) > 0:
print('*** failed to build the CFFI modules %r' % (
[f[1] for f in failures],), file=sys.stderr)
@@ -97,7 +277,7 @@
for k in cffi_build_scripts:
setattr(options, 'no_' + k, True)
must_fail = '_missing_build_script.py'
- assert not os.path.exists(str(basedir.join('lib_pypy').join(must_fail)))
+ assert not os.path.exists(str(join(join(basedir,'lib_pypy'),must_fail)))
cffi_build_scripts['should_fail'] = must_fail
- failures = create_cffi_import_libraries(exename, options, basedir)
+ failures = create_cffi_import_libraries(exename, options, basedir, only=only)
assert len(failures) == 1
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -83,7 +83,11 @@
if not _fake and not pypy_runs(pypy_c):
raise OSError("Running %r failed!" % (str(pypy_c),))
if not options.no_cffi:
- failures = create_cffi_import_libraries(pypy_c, options, basedir)
+ failures = create_cffi_import_libraries(
+ str(pypy_c), options, str(basedir),
+ embed_dependencies=options.embed_dependencies,
+ )
+
for key, module in failures:
print >>sys.stderr, """!!!!!!!!!!\nBuilding {0} bindings failed.
You can either install development headers package,
@@ -262,11 +266,16 @@
return retval, builddir # for tests
def package(*args, **kwds):
- try:
- import argparse
- except ImportError:
- import imp
- argparse = imp.load_source('argparse', 'lib-python/2.7/argparse.py')
+ import argparse
+
+ class NegateAction(argparse.Action):
+ def __init__(self, option_strings, dest, nargs=0, **kwargs):
+ super(NegateAction, self).__init__(option_strings, dest, nargs,
+ **kwargs)
+
+ def __call__(self, parser, ns, values, option):
+ setattr(ns, self.dest, option[2:4] != 'no')
+
if sys.platform == 'win32':
pypy_exe = 'pypy.exe'
else:
@@ -288,7 +297,7 @@
parser.add_argument('--no-keep-debug', dest='keep_debug',
action='store_false', help='do not keep debug symbols')
parser.add_argument('--rename_pypy_c', dest='pypy_c', type=str, default=pypy_exe,
- help='target executable name, defaults to "pypy"')
+ help='target executable name, defaults to "%s"' % pypy_exe)
parser.add_argument('--archive-name', dest='name', type=str, default='',
help='pypy-VER-PLATFORM')
parser.add_argument('--builddir', type=str, default='',
@@ -296,13 +305,21 @@
parser.add_argument('--targetdir', type=str, default='',
help='destination dir for archive')
parser.add_argument('--override_pypy_c', type=str, default='',
- help='use as pypy exe instead of pypy/goal/pypy-c')
+ help='use as pypy3 exe instead of pypy/goal/pypy3-c')
+ parser.add_argument('--embedded-dependencies', '--no-embedded-dependencies',
+ dest='embed_dependencies',
+ action=NegateAction,
+ default=(sys.platform == 'darwin'),
+ help='whether to embed dependencies for distribution '
+ '(default on OS X)')
options = parser.parse_args(args)
if os.environ.has_key("PYPY_PACKAGE_NOKEEPDEBUG"):
options.keep_debug = False
if os.environ.has_key("PYPY_PACKAGE_WITHOUTTK"):
options.no_tk = True
+ if os.environ.has_key("PYPY_EMBED_DEPENDENCIES"):
+ options.embed_dependencies = True
if not options.builddir:
# The import actually creates the udir directory
from rpython.tool.udir import udir
More information about the pypy-commit
mailing list