[pypy-commit] pypy py3.5-mac-embedding: add support to package.py for embedding libraries not included with OS X
danchr
pypy.commits at gmail.com
Fri Jul 14 12:09:01 EDT 2017
Author: Dan Villiom Podlaski Christiansen <dan at magenta.dk>
Branch: py3.5-mac-embedding
Changeset: r91870:ed548e40171b
Date: 2017-07-14 17:52 +0200
http://bitbucket.org/pypy/pypy/changeset/ed548e40171b/
Log: add support to package.py for embedding libraries not included with
OS X
diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -58,6 +58,7 @@
^rpython/rlib/rvmprof/src/shared/libbacktrace/config.h$
^rpython/rlib/rvmprof/src/shared/libbacktrace/config.log$
^rpython/rlib/rvmprof/src/shared/libbacktrace/config.status$
+^pypy/tool/dest$
^pypy/goal/pypy-translation-snapshot$
^pypy/goal/pypy-c
^pypy/goal/pypy3-c
diff --git a/pypy/tool/build_cffi_imports.py b/pypy/tool/build_cffi_imports.py
--- a/pypy/tool/build_cffi_imports.py
+++ b/pypy/tool/build_cffi_imports.py
@@ -22,12 +22,98 @@
"xx": None, # for testing: 'None' should be completely ignored
}
-def create_cffi_import_libraries(pypy_c, options, basedir, only=None):
+# for distribution, we may want to fetch dependencies not provided by
+# the OS, such as a recent openssl/libressl or liblzma/xz.
+cffi_dependencies = {
+ 'lzma': ('https://tukaani.org/xz/xz-5.2.3.tar.gz', []),
+ 'ssl': ('http://ftp.openbsd.org/pub/OpenBSD/LibreSSL/libressl-2.6.0.tar.gz',
+ ['--without-openssldir']),
+ '_gdbm': ('ftp://ftp.gnu.org/gnu/gdbm/gdbm-1.13.tar.gz',
+ ['--without-readline']),
+}
+
+
+def _unpack_tarfile(filename, extract_dir):
+ """Unpack tar/tar.gz/tar.bz2/tar.xz `filename` to `extract_dir`
+ """
+ import tarfile # late import for breaking circular dependency
+ try:
+ tarobj = tarfile.open(filename)
+ except tarfile.TarError:
+ raise ReadError(
+ "%s is not a compressed or uncompressed tar file" % filename)
+ try:
+ tarobj.extractall(extract_dir)
+ finally:
+ tarobj.close()
+
+
+def _build_dependency(name, destdir):
+ import multiprocessing
+ import shutil
+ import subprocess
+
+ try:
+ from urllib.request import urlretrieve
+ except ImportError:
+ from urllib import urlretrieve
+
+ try:
+ url, args = cffi_dependencies[name]
+ except KeyError:
+ return 0, None, None
+
+ archive = os.path.join(destdir, url.rsplit('/', 1)[-1])
+
+ # next, fetch the archive to disk, if needed
+ if not os.path.exists(archive):
+ urlretrieve(url, archive)
+
+ # extract the archive into our destination directory
+ _unpack_tarfile(archive, destdir)
+
+ # configure & build it
+ sources = os.path.join(
+ destdir,
+ os.path.basename(archive)[:-7],
+ )
+
+ from rpython.tool.runsubprocess import run_subprocess
+
+ status, stdout, stderr = run_subprocess(
+ './configure',
+ [
+ '--prefix=/usr',
+ '--disable-shared',
+ '--enable-silent-rules',
+ '--disable-dependency-tracking',
+ ] + args,
+ cwd=sources,
+ )
+
+ if status != 0:
+ return status, stdout, stderr
+
+ status, stdout, stderr = run_subprocess(
+ 'make',
+ [
+ '-s', '-j' + str(multiprocessing.cpu_count() + 1),
+ 'install', 'DESTDIR={}/'.format(destdir),
+ ],
+ cwd=sources,
+ )
+
+ return status, stdout, stderr
+
+
+def create_cffi_import_libraries(pypy_c, options, basedir, only=None,
+ embed_dependencies=False):
from rpython.tool.runsubprocess import run_subprocess
shutil.rmtree(str(join(basedir,'lib_pypy','__pycache__')),
ignore_errors=True)
failures = []
+
for key, module in sorted(cffi_build_scripts.items()):
if only and key not in only:
print("* SKIPPING", key, '(not specified in --only)')
@@ -40,9 +126,33 @@
else:
args = ['-c', 'import ' + module]
cwd = None
+ env = os.environ.copy()
+
print('*', ' '.join(args), file=sys.stderr)
+ if embed_dependencies:
+ destdir = os.path.join(os.path.abspath(os.path.dirname(__file__)),
+ 'dest')
+ shutil.rmtree(destdir, ignore_errors=True)
+ os.makedirs(destdir)
+
+ status, stdout, stderr = _build_dependency(key, destdir)
+
+ if status != 0:
+ failures.append((key, module))
+ print("stdout:")
+ print(stdout.decode('utf-8'))
+ print("stderr:")
+ print(stderr.decode('utf-8'))
+ continue
+
+ env['CPPFLAGS'] = \
+ '-I{}/usr/include {}'.format(destdir, env.get('CPPFLAGS', ''))
+ env['LDFLAGS'] = \
+ '-L{}/usr/lib {}'.format(destdir, env.get('LDFLAGS', ''))
+
try:
- status, stdout, stderr = run_subprocess(str(pypy_c), args, cwd=cwd)
+ status, stdout, stderr = run_subprocess(str(pypy_c), args,
+ cwd=cwd, env=env)
if status != 0:
failures.append((key, module))
print("stdout:")
@@ -73,6 +183,8 @@
' you can specify an alternative pypy vm here')
parser.add_argument('--only', dest='only', default=None,
help='Only build the modules delimited by a colon. E.g. ssl,sqlite')
+ parser.add_argument('--embed-dependencies', dest='embed_dependencies', action='store_true',
+ help='embed dependencies for distribution')
args = parser.parse_args()
exename = join(os.getcwd(), args.exefile)
@@ -89,7 +201,8 @@
only = None
else:
only = set(args.only.split(','))
- failures = create_cffi_import_libraries(exename, options, basedir, only=only)
+ failures = create_cffi_import_libraries(exename, options, basedir, only=only,
+ embed_dependencies=args.embed_dependencies)
if len(failures) > 0:
print('*** failed to build the CFFI modules %r' % (
[f[1] for f in failures],), file=sys.stderr)
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -83,7 +83,11 @@
if not _fake and not pypy_runs(pypy_c):
raise OSError("Running %r failed!" % (str(pypy_c),))
if not options.no_cffi:
- failures = create_cffi_import_libraries(str(pypy_c), options, str(basedir))
+ failures = create_cffi_import_libraries(
+ str(pypy_c), options, str(basedir),
+ embed_dependencies=options.embed_dependencies,
+ )
+
for key, module in failures:
print >>sys.stderr, """!!!!!!!!!!\nBuilding {0} bindings failed.
You can either install development headers package,
@@ -296,12 +300,17 @@
help='destination dir for archive')
parser.add_argument('--override_pypy_c', type=str, default='',
help='use as pypy3 exe instead of pypy/goal/pypy3-c')
+ parser.add_argument('--embedded-dependencies', dest='embed_dependencies',
+ action='store_true',
+ help='embed dependencies for distribution')
options = parser.parse_args(args)
if os.environ.has_key("PYPY_PACKAGE_NOSTRIP"):
options.nostrip = True
if os.environ.has_key("PYPY_PACKAGE_WITHOUTTK"):
options.no_tk = True
+ if os.environ.has_key("PYPY_EMBED_DEPENDENCIES"):
+ options.embed_dependencies = True
if not options.builddir:
# The import actually creates the udir directory
from rpython.tool.udir import udir
More information about the pypy-commit
mailing list