Skip to content
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
116 changes: 52 additions & 64 deletions Tools/ssl/multissltests.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,27 +18,19 @@
search paths for header files and shared libraries. It's known to work on
Linux with GCC and clang.

Please keep this script compatible with Python 2.7, and 3.4 to 3.7.

(c) 2013-2017 Christian Heimes <christian@python.org>
"""
from __future__ import print_function

import argparse
from datetime import datetime
import datetime as dt
import logging
import os
try:
from urllib.request import urlopen
from urllib.error import HTTPError
except ImportError:
from urllib2 import urlopen, HTTPError
import re
import shutil
import subprocess
import sys
import tarfile

from urllib.error import HTTPError
from urllib.request import urlopen

log = logging.getLogger("multissl")

Expand Down Expand Up @@ -95,18 +87,20 @@
nargs='+',
default=(),
help=(
"OpenSSL versions, defaults to '{}' (ancient: '{}') if no "
f"OpenSSL versions, defaults to '{OPENSSL_RECENT_VERSIONS}' "
f"(ancient: '{OPENSSL_OLD_VERSIONS}') if no "
"OpenSSL and LibreSSL versions are given."
).format(OPENSSL_RECENT_VERSIONS, OPENSSL_OLD_VERSIONS)
)
)
parser.add_argument(
'--libressl',
nargs='+',
default=(),
help=(
"LibreSSL versions, defaults to '{}' (ancient: '{}') if no "
f"LibreSSL versions, defaults to '{LIBRESSL_RECENT_VERSIONS}' "
f"(ancient: '{LIBRESSL_OLD_VERSIONS}') if no "
"OpenSSL and LibreSSL versions are given."
).format(LIBRESSL_RECENT_VERSIONS, LIBRESSL_OLD_VERSIONS)
)
)
parser.add_argument(
'--aws-lc',
Expand All @@ -116,8 +110,9 @@
nargs='+',
default=(),
help=(
"AWS-LC versions, defaults to '{}' if no crypto library versions are given."
).format(AWSLC_RECENT_VERSIONS)
f"AWS-LC versions, defaults to '{AWSLC_RECENT_VERSIONS}' "
f"if no crypto library versions are given."
)
)
parser.add_argument(
'--tests',
Expand Down Expand Up @@ -171,7 +166,7 @@
)


class AbstractBuilder(object):
class AbstractBuilder:
library = None
url_templates = None
src_template = None
Expand Down Expand Up @@ -206,7 +201,7 @@ def __init__(self, version, args):
self.system = args.system

def __str__(self):
return "<{0.__class__.__name__} for {0.version}>".format(self)
return f"<{self.__class__.__name__} for {self.version}>"

def __eq__(self, other):
if not isinstance(other, AbstractBuilder):
Expand Down Expand Up @@ -261,40 +256,38 @@ def has_src(self):
return os.path.isfile(self.src_file)

def _subprocess_call(self, cmd, env=None, **kwargs):
log.debug("Call '{}'".format(" ".join(cmd)))
log.debug(f"Call '{' '.join(cmd)}'")
return subprocess.check_call(cmd, env=env, **kwargs)

def _subprocess_output(self, cmd, env=None, **kwargs):
log.debug("Call '{}'".format(" ".join(cmd)))
log.debug(f"Call '{' '.join(cmd)}'")
if env is None:
env = os.environ.copy()
env["LD_LIBRARY_PATH"] = self.lib_dir
out = subprocess.check_output(cmd, env=env, **kwargs)
return out.strip().decode("utf-8")
out = subprocess.check_output(cmd, env=env, text=True, **kwargs)
return out.strip()

def _download_src(self):
"""Download sources"""
src_dir = os.path.dirname(self.src_file)
if not os.path.isdir(src_dir):
os.makedirs(src_dir)
os.makedirs(os.path.dirname(self.src_file), exist_ok=True)
data = None
for url_template in self.url_templates:
url = url_template.format(v=self.version, s=self.short_version)
log.info("Downloading from {}".format(url))
log.info(f"Downloading from {url}")
try:
req = urlopen(url)
# KISS, read all, write all
data = req.read()
with urlopen(url) as req:
# KISS, read all, write all
data = req.read()
except HTTPError as e:
log.error(
"Download from {} has from failed: {}".format(url, e)
f"Download from {url} has from failed: {e}"
)
else:
log.info("Successfully downloaded from {}".format(url))
log.info(f"Successfully downloaded from {url}")
break
if data is None:
raise ValueError("All download URLs have failed")
log.info("Storing {}".format(self.src_file))
log.info(f"Storing {self.src_file}")
with open(self.src_file, "wb") as f:
f.write(data)

Expand All @@ -305,30 +298,30 @@ def _unpack_src(self):
shutil.rmtree(self.build_dir)
os.makedirs(self.build_dir)

tf = tarfile.open(self.src_file)
name = self.build_template.format(self.version)
base = name + '/'
# force extraction into build dir
members = tf.getmembers()
for member in list(members):
if member.name == name:
members.remove(member)
elif not member.name.startswith(base):
raise ValueError(member.name, base)
member.name = member.name[len(base):].lstrip('/')
log.info("Unpacking files to {}".format(self.build_dir))
tf.extractall(self.build_dir, members, filter='data')
with tarfile.open(self.src_file) as tf:
# force extraction into build dir
members = tf.getmembers()
for member in list(members):
if member.name == name:
members.remove(member)
elif not member.name.startswith(base):
raise ValueError(member.name, base)
member.name = member.name[len(base):].lstrip('/')
log.info(f"Unpacking files to {self.build_dir}")
tf.extractall(self.build_dir, members, filter='data')

def _build_src(self, config_args=()):
"""Now build openssl"""
log.info("Running build in {}".format(self.build_dir))
log.info(f"Running build in {self.build_dir}")
cwd = self.build_dir
if self.args.tsan:
config_args += ("-fsanitize=thread",)
cmd = [
"./config", *config_args,
"shared", "--debug",
"--prefix={}".format(self.install_dir)
f"--prefix={self.install_dir}"
]
# cmd.extend(["no-deprecated", "--api=1.1.0"])
env = os.environ.copy()
Expand Down Expand Up @@ -361,19 +354,19 @@ def install(self):
if not self.has_src:
self._download_src()
else:
log.debug("Already has src {}".format(self.src_file))
log.debug(f"Already has src {self.src_file}")
self._unpack_src()
self._build_src()
self._make_install()
else:
log.info("Already has installation {}".format(self.install_dir))
log.info(f"Already has installation {self.install_dir}")
# validate installation
version = self.openssl_version
if self.version not in version:
raise ValueError(version)

def recompile_pymods(self):
log.warning("Using build from {}".format(self.build_dir))
log.warning(f"Using build from {self.build_dir}")
# force a rebuild of all modules that use OpenSSL APIs
for fname in self.module_files:
os.utime(fname, None)
Expand All @@ -385,8 +378,8 @@ def recompile_pymods(self):

# overwrite header and library search paths
env = os.environ.copy()
env["CPPFLAGS"] = "-I{}".format(self.include_dir)
env["LDFLAGS"] = "-L{}".format(self.lib_dir)
env["CPPFLAGS"] = f"-I{self.include_dir}"
env["LDFLAGS"] = f"-L{self.lib_dir}"
# set rpath
env["LD_RUN_PATH"] = self.lib_dir

Expand All @@ -411,8 +404,6 @@ def run_python_tests(self, tests, network=True):
os.path.join(PYTHONROOT, 'Lib/test/ssltests.py'),
'-j0'
]
elif sys.version_info < (3, 3):
cmd = [sys.executable, '-m', 'test.regrtest']
else:
cmd = [sys.executable, '-m', 'test', '-j0']
if network:
Expand Down Expand Up @@ -493,22 +484,22 @@ class BuildAWSLC(AbstractBuilder):

def _build_src(self, config_args=()):
cwd = self.build_dir
log.info("Running build in {}".format(cwd))
log.info(f"Running build in {cwd}")
env = os.environ.copy()
env["LD_RUN_PATH"] = self.lib_dir # set rpath
if self.system:
env['SYSTEM'] = self.system
cmd = [
"cmake",
"-DCMAKE_BUILD_TYPE=RelWithDebInfo",
"-DCMAKE_PREFIX_PATH={}".format(self.install_dir),
"-DCMAKE_INSTALL_PREFIX={}".format(self.install_dir),
f"-DCMAKE_PREFIX_PATH={self.install_dir}",
f"-DCMAKE_INSTALL_PREFIX={self.install_dir}",
"-DBUILD_SHARED_LIBS=ON",
"-DBUILD_TESTING=OFF",
"-DFIPS=OFF",
]
self._subprocess_call(cmd, cwd=cwd, env=env)
self._subprocess_call(["make", "-j{}".format(self.jobs)], cwd=cwd, env=env)
self._subprocess_call(["make", f"-j{self.jobs}"], cwd=cwd, env=env)


def configure_make():
Expand Down Expand Up @@ -538,7 +529,7 @@ def main():
format="*** %(levelname)s %(message)s"
)

start = datetime.now()
start = dt.datetime.now()

if args.steps in {'modules', 'tests'}:
for name in ['Makefile.pre.in', 'Modules/_ssl.c']:
Expand Down Expand Up @@ -577,13 +568,10 @@ def main():
)
except Exception as e:
log.exception("%s failed", build)
print("{} failed: {}".format(build, e), file=sys.stderr)
print(f"{build} failed: {e}", file=sys.stderr)
sys.exit(2)

log.info("\n{} finished in {}".format(
args.steps.capitalize(),
datetime.now() - start
))
log.info(f"\n{args.steps.capitalize()} finished in {dt.datetime.now() - start}")
print('Python: ', sys.version)
if args.steps == 'tests':
if args.tests:
Expand All @@ -593,7 +581,7 @@ def main():

print('OpenSSL / LibreSSL / AWS-LC versions:')
for build in builds:
print(" * {0.library} {0.version}".format(build))
print(f" * {build.library} {build.version}")


if __name__ == "__main__":
Expand Down
Loading