[Python-checkins] bpo-32549: Compile OpenSSL 1.1.0 on Travis CI (#5180)
Christian Heimes
webhook-mailer at python.org
Tue Jan 16 15:02:29 EST 2018
https://github.com/python/cpython/commit/ced9cb5303ad1447f84d923e0c7f769f5e0c6297
commit: ced9cb5303ad1447f84d923e0c7f769f5e0c6297
branch: master
author: Christian Heimes <christian at python.org>
committer: GitHub <noreply at github.com>
date: 2018-01-16T21:02:26+01:00
summary:
bpo-32549: Compile OpenSSL 1.1.0 on Travis CI (#5180)
Use an improved version of multissl test helper to compile a local copy
of OpenSSL 1.1.0g.
Signed-off-by: Christian Heimes <christian at python.org>
files:
A Misc/NEWS.d/next/Tests/2018-01-14-11-40-22.bpo-32549.fLwbVA.rst
M .travis.yml
M Makefile.pre.in
M Tools/ssl/multissltests.py
diff --git a/.travis.yml b/.travis.yml
index c207bd72da2..687d0214ab8 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,6 +7,19 @@ group: beta
cache:
- pip
- ccache
+ - directories:
+ - $HOME/multissl
+
+env:
+ global:
+ - OPENSSL=1.1.0g
+ - OPENSSL_DIR="$HOME/multissl/openssl/${OPENSSL}"
+ - PATH="${OPENSSL_DIR}/bin:$PATH"
+ - CFLAGS="-I${OPENSSL_DIR}/include"
+ - LDFLAGS="-L${OPENSSL_DIR}/lib"
+ # Set rpath with env var instead of -Wl,-rpath linker flag
+ # OpenSSL ignores LDFLAGS when linking bin/openssl
+ - LD_RUN_PATH="${OPENSSL_DIR}/lib"
branches:
only:
@@ -48,6 +61,10 @@ matrix:
echo "Only docs were updated, stopping build process."
exit
fi
+ python3 Tools/ssl/multissltests.py --steps=library \
+ --base-directory ${HOME}/multissl \
+ --openssl ${OPENSSL} >/dev/null
+ openssl version
./configure
make -s -j4
# Need a venv that can parse covered code.
@@ -71,6 +88,13 @@ before_script:
echo "Only docs were updated, stopping build process."
exit
fi
+ if [ "${TESTING}" != "docs" ]; then
+ # clang complains about unused-parameter a lot, redirect stderr
+ python3 Tools/ssl/multissltests.py --steps=library \
+ --base-directory ${HOME}/multissl \
+ --openssl ${OPENSSL} >/dev/null 2>&1
+ fi
+ openssl version
./configure --with-pydebug
make -j4
make -j4 regen-all clinic
diff --git a/Makefile.pre.in b/Makefile.pre.in
index 59ba9d4c1f1..994e8611e13 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -1086,7 +1086,7 @@ quicktest: @DEF_MAKE_RULE@ platform
# SSL tests
.PHONY: multisslcompile multissltest
multisslcompile: build_all
- $(RUNSHARED) ./$(BUILDPYTHON) Tools/ssl/multissltests.py --compile-only
+ $(RUNSHARED) ./$(BUILDPYTHON) Tools/ssl/multissltests.py --steps=modules
multissltest: build_all
$(RUNSHARED) ./$(BUILDPYTHON) Tools/ssl/multissltests.py
diff --git a/Misc/NEWS.d/next/Tests/2018-01-14-11-40-22.bpo-32549.fLwbVA.rst b/Misc/NEWS.d/next/Tests/2018-01-14-11-40-22.bpo-32549.fLwbVA.rst
new file mode 100644
index 00000000000..d98f4661361
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2018-01-14-11-40-22.bpo-32549.fLwbVA.rst
@@ -0,0 +1 @@
+On Travis CI, Python now Compiles and uses a local copy of OpenSSL 1.1.0g for testing.
diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py
index ce5bbd85308..75874cfca4d 100755
--- a/Tools/ssl/multissltests.py
+++ b/Tools/ssl/multissltests.py
@@ -41,8 +41,8 @@
log = logging.getLogger("multissl")
OPENSSL_OLD_VERSIONS = [
- "0.9.8zh",
- "1.0.1u",
+ # "0.9.8zh",
+ # "1.0.1u",
]
OPENSSL_RECENT_VERSIONS = [
@@ -52,8 +52,8 @@
]
LIBRESSL_OLD_VERSIONS = [
- "2.3.10",
- "2.4.5",
+ # "2.3.10",
+ # "2.4.5",
]
LIBRESSL_RECENT_VERSIONS = [
@@ -62,8 +62,10 @@
]
# store files in ../multissl
-HERE = os.path.abspath(os.getcwd())
-MULTISSL_DIR = os.path.abspath(os.path.join(HERE, '..', 'multissl'))
+HERE = os.path.dirname(os.path.abspath(__file__))
+PYTHONROOT = os.path.abspath(os.path.join(HERE, '..', '..'))
+MULTISSL_DIR = os.path.abspath(os.path.join(PYTHONROOT, '..', 'multissl'))
+
parser = argparse.ArgumentParser(
prog='multissl',
@@ -118,9 +120,14 @@
help="Disable network tests."
)
parser.add_argument(
- '--compile-only',
- action='store_true',
- help="Don't run tests, only compile _ssl.c and _hashopenssl.c."
+ '--steps',
+ choices=['library', 'modules', 'tests'],
+ default='tests',
+ help=(
+ "Which steps to perform. 'library' downloads and compiles OpenSSL "
+ "or LibreSSL. 'module' also compiles Python modules. 'tests' builds "
+ "all and runs the test suite."
+ )
)
@@ -129,21 +136,21 @@ class AbstractBuilder(object):
url_template = None
src_template = None
build_template = None
+ install_target = 'install'
module_files = ("Modules/_ssl.c",
"Modules/_hashopenssl.c")
module_libs = ("_ssl", "_hashlib")
- def __init__(self, version, compile_args=(),
- basedir=MULTISSL_DIR):
+ def __init__(self, version, args):
self.version = version
- self.compile_args = compile_args
+ self.args = args
# installation directory
self.install_dir = os.path.join(
- os.path.join(basedir, self.library.lower()), version
+ os.path.join(args.base_directory, self.library.lower()), version
)
# source file
- self.src_dir = os.path.join(basedir, 'src')
+ self.src_dir = os.path.join(args.base_directory, 'src')
self.src_file = os.path.join(
self.src_dir, self.src_template.format(version))
# build directory (removed after install)
@@ -252,13 +259,15 @@ def _build_src(self):
log.info("Running build in {}".format(self.build_dir))
cwd = self.build_dir
cmd = ["./config", "shared", "--prefix={}".format(self.install_dir)]
- cmd.extend(self.compile_args)
self._subprocess_call(cmd, cwd=cwd)
# Old OpenSSL versions do not support parallel builds.
self._subprocess_call(["make", "-j1"], cwd=cwd)
def _make_install(self, remove=True):
- self._subprocess_call(["make", "-j1", "install"], cwd=self.build_dir)
+ self._subprocess_call(
+ ["make", "-j1", self.install_target],
+ cwd=self.build_dir
+ )
if remove:
shutil.rmtree(self.build_dir)
@@ -330,6 +339,8 @@ class BuildOpenSSL(AbstractBuilder):
url_template = "https://www.openssl.org/source/openssl-{}.tar.gz"
src_template = "openssl-{}.tar.gz"
build_template = "openssl-{}"
+ # only install software, skip docs
+ install_target = 'install_sw'
class BuildLibreSSL(AbstractBuilder):
@@ -368,57 +379,63 @@ def main():
start = datetime.now()
- for name in ['python', 'setup.py', 'Modules/_ssl.c']:
- if not os.path.isfile(name):
+ if args.steps in {'modules', 'tests'}:
+ for name in ['setup.py', 'Modules/_ssl.c']:
+ if not os.path.isfile(os.path.join(PYTHONROOT, name)):
+ parser.error(
+ "Must be executed from CPython build dir"
+ )
+ if not os.path.samefile('python', sys.executable):
parser.error(
- "Must be executed from CPython build dir"
+ "Must be executed with ./python from CPython build dir"
)
- if not os.path.samefile('python', sys.executable):
- parser.error(
- "Must be executed with ./python from CPython build dir"
- )
-
- # check for configure and run make
- configure_make()
+ # check for configure and run make
+ configure_make()
# download and register builder
builds = []
for version in args.openssl:
- build = BuildOpenSSL(version)
+ build = BuildOpenSSL(
+ version,
+ args
+ )
build.install()
builds.append(build)
for version in args.libressl:
- build = BuildLibreSSL(version)
+ build = BuildLibreSSL(
+ version,
+ args
+ )
build.install()
builds.append(build)
- for build in builds:
- try:
- build.recompile_pymods()
- build.check_pyssl()
- if not args.compile_only:
- build.run_python_tests(
- tests=args.tests,
- network=args.network,
- )
- except Exception as e:
- log.exception("%s failed", build)
- print("{} failed: {}".format(build, e), file=sys.stderr)
- sys.exit(2)
-
- print("\n{} finished in {}".format(
- "Tests" if not args.compile_only else "Builds",
- datetime.now() - start
- ))
+ if args.steps in {'modules', 'tests'}:
+ for build in builds:
+ try:
+ build.recompile_pymods()
+ build.check_pyssl()
+ if args.steps == 'tests':
+ build.run_python_tests(
+ tests=args.tests,
+ network=args.network,
+ )
+ except Exception as e:
+ log.exception("%s failed", build)
+ print("{} failed: {}".format(build, e), file=sys.stderr)
+ sys.exit(2)
+
+ log.info("\n{} finished in {}".format(
+ args.steps.capitalize(),
+ datetime.now() - start
+ ))
print('Python: ', sys.version)
- if args.compile_only:
- print('Build only')
- elif args.tests:
- print('Executed Tests:', ' '.join(args.tests))
- else:
- print('Executed all SSL tests.')
+ if args.steps == 'tests':
+ if args.tests:
+ print('Executed Tests:', ' '.join(args.tests))
+ else:
+ print('Executed all SSL tests.')
print('OpenSSL / LibreSSL versions:')
for build in builds:
More information about the Python-checkins
mailing list