Refactor run_tests to increase parallelism.

Rather than relying on parallel make to build individual configurations
one at a time, use xargs to build multiple configurations in parallel.
This allows the configure scripts to run in parallel.  On a 14-core
system (28 hyperthreads), this increases average CPU utilization from
~20% to ~90%.
This commit is contained in:
Jason Evans 2017-05-24 11:33:06 -07:00
parent 18ecbfa89e
commit 67c93c332a

View File

@ -1,6 +1,12 @@
#!/usr/bin/env python #!/usr/bin/env python
from itertools import combinations from itertools import combinations
from os import uname
from multiprocessing import cpu_count
nparallel = cpu_count() * 2
uname = uname()[0]
def powerset(items): def powerset(items):
result = [] result = []
@ -8,8 +14,6 @@ def powerset(items):
result += combinations(items, i) result += combinations(items, i)
return result return result
MAKE_J_VAL = 32
possible_compilers = [('gcc', 'g++'), ('clang', 'clang++')] possible_compilers = [('gcc', 'g++'), ('clang', 'clang++')]
possible_compiler_opts = [ possible_compiler_opts = [
'-m32', '-m32',
@ -28,8 +32,11 @@ possible_malloc_conf_opts = [
print 'set -e' print 'set -e'
print 'autoconf' print 'autoconf'
print 'unamestr=`uname`' print 'rm -rf run_tests.out'
print 'mkdir run_tests.out'
print 'cd run_tests.out'
ind = 0
for cc, cxx in possible_compilers: for cc, cxx in possible_compilers:
for compiler_opts in powerset(possible_compiler_opts): for compiler_opts in powerset(possible_compiler_opts):
for config_opts in powerset(possible_config_opts): for config_opts in powerset(possible_config_opts):
@ -39,9 +46,10 @@ for cc, cxx in possible_compilers:
and '--enable-prof' in config_opts: and '--enable-prof' in config_opts:
continue continue
config_line = ( config_line = (
'EXTRA_CFLAGS=-Werror EXTRA_CXXFLAGS=-Werror ./configure ' 'EXTRA_CFLAGS=-Werror EXTRA_CXXFLAGS=-Werror '
+ 'CC="{} {}" '.format(cc, " ".join(compiler_opts)) + 'CC="{} {}" '.format(cc, " ".join(compiler_opts))
+ 'CXX="{} {}" '.format(cxx, " ".join(compiler_opts)) + 'CXX="{} {}" '.format(cxx, " ".join(compiler_opts))
+ '../../configure '
+ " ".join(config_opts) + (' --with-malloc-conf=' + + " ".join(config_opts) + (' --with-malloc-conf=' +
",".join(malloc_conf_opts) if len(malloc_conf_opts) > 0 ",".join(malloc_conf_opts) if len(malloc_conf_opts) > 0
else '') else '')
@ -52,14 +60,38 @@ for cc, cxx in possible_compilers:
# Heap profiling and dss are not supported on OS X. # Heap profiling and dss are not supported on OS X.
darwin_unsupported = ('--enable-prof' in config_opts or \ darwin_unsupported = ('--enable-prof' in config_opts or \
'dss:primary' in malloc_conf_opts) 'dss:primary' in malloc_conf_opts)
if linux_supported: if uname is 'Linux' and linux_supported \
print 'if [[ "$unamestr" = "Linux" ]]; then' or uname is not 'Darwin' \
elif darwin_unsupported: or not darwin_unsupported:
print 'if [[ "$unamestr" != "Darwin" ]]; then' print """cat <<EOF > run_test_%(ind)d.sh
#!/bin/sh
print config_line set -e
print "make clean"
print "make -j" + str(MAKE_J_VAL) + " check"
if linux_supported or darwin_unsupported: abort() {
print 'fi' echo "==> Error" >> run_test.log
echo "Error; see run_tests.out/run_test_%(ind)d.out/run_test.log"
exit 255 # Special exit code tells xargs to terminate.
}
# Environment variables are not supported.
run_cmd() {
echo "==> \$@" >> run_test.log
\$@ >> run_test.log 2>&1 || abort
}
echo "=> run_test_%(ind)d: %(config_line)s"
mkdir run_test_%(ind)d.out
cd run_test_%(ind)d.out
echo "==> %(config_line)s" >> run_test.log
%(config_line)s >> run_test.log 2>&1 || abort
run_cmd make all tests
run_cmd make check
run_cmd make distclean
EOF
chmod 755 run_test_%(ind)d.sh""" % {'ind': ind, 'config_line': config_line}
ind += 1
print 'for i in `seq 0 %(last_ind)d` ; do echo run_test_${i}.sh ; done | xargs -P %(nparallel)d -n 1 sh' % {'last_ind': ind-1, 'nparallel': nparallel}