Refactor run_tests to increase parallelism.
Rather than relying on parallel make to build individual configurations one at a time, use xargs to build multiple configurations in parallel. This allows the configure scripts to run in parallel. On a 14-core system (28 hyperthreads), this increases average CPU utilization from ~20% to ~90%.
This commit is contained in:
parent
18ecbfa89e
commit
67c93c332a
@ -1,6 +1,12 @@
|
|||||||
#!/usr/bin/env python
|
#!/usr/bin/env python
|
||||||
|
|
||||||
from itertools import combinations
|
from itertools import combinations
|
||||||
|
from os import uname
|
||||||
|
from multiprocessing import cpu_count
|
||||||
|
|
||||||
|
nparallel = cpu_count() * 2
|
||||||
|
|
||||||
|
uname = uname()[0]
|
||||||
|
|
||||||
def powerset(items):
|
def powerset(items):
|
||||||
result = []
|
result = []
|
||||||
@ -8,8 +14,6 @@ def powerset(items):
|
|||||||
result += combinations(items, i)
|
result += combinations(items, i)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
MAKE_J_VAL = 32
|
|
||||||
|
|
||||||
possible_compilers = [('gcc', 'g++'), ('clang', 'clang++')]
|
possible_compilers = [('gcc', 'g++'), ('clang', 'clang++')]
|
||||||
possible_compiler_opts = [
|
possible_compiler_opts = [
|
||||||
'-m32',
|
'-m32',
|
||||||
@ -28,8 +32,11 @@ possible_malloc_conf_opts = [
|
|||||||
|
|
||||||
print 'set -e'
|
print 'set -e'
|
||||||
print 'autoconf'
|
print 'autoconf'
|
||||||
print 'unamestr=`uname`'
|
print 'rm -rf run_tests.out'
|
||||||
|
print 'mkdir run_tests.out'
|
||||||
|
print 'cd run_tests.out'
|
||||||
|
|
||||||
|
ind = 0
|
||||||
for cc, cxx in possible_compilers:
|
for cc, cxx in possible_compilers:
|
||||||
for compiler_opts in powerset(possible_compiler_opts):
|
for compiler_opts in powerset(possible_compiler_opts):
|
||||||
for config_opts in powerset(possible_config_opts):
|
for config_opts in powerset(possible_config_opts):
|
||||||
@ -39,9 +46,10 @@ for cc, cxx in possible_compilers:
|
|||||||
and '--enable-prof' in config_opts:
|
and '--enable-prof' in config_opts:
|
||||||
continue
|
continue
|
||||||
config_line = (
|
config_line = (
|
||||||
'EXTRA_CFLAGS=-Werror EXTRA_CXXFLAGS=-Werror ./configure '
|
'EXTRA_CFLAGS=-Werror EXTRA_CXXFLAGS=-Werror '
|
||||||
+ 'CC="{} {}" '.format(cc, " ".join(compiler_opts))
|
+ 'CC="{} {}" '.format(cc, " ".join(compiler_opts))
|
||||||
+ 'CXX="{} {}" '.format(cxx, " ".join(compiler_opts))
|
+ 'CXX="{} {}" '.format(cxx, " ".join(compiler_opts))
|
||||||
|
+ '../../configure '
|
||||||
+ " ".join(config_opts) + (' --with-malloc-conf=' +
|
+ " ".join(config_opts) + (' --with-malloc-conf=' +
|
||||||
",".join(malloc_conf_opts) if len(malloc_conf_opts) > 0
|
",".join(malloc_conf_opts) if len(malloc_conf_opts) > 0
|
||||||
else '')
|
else '')
|
||||||
@ -52,14 +60,38 @@ for cc, cxx in possible_compilers:
|
|||||||
# Heap profiling and dss are not supported on OS X.
|
# Heap profiling and dss are not supported on OS X.
|
||||||
darwin_unsupported = ('--enable-prof' in config_opts or \
|
darwin_unsupported = ('--enable-prof' in config_opts or \
|
||||||
'dss:primary' in malloc_conf_opts)
|
'dss:primary' in malloc_conf_opts)
|
||||||
if linux_supported:
|
if uname is 'Linux' and linux_supported \
|
||||||
print 'if [[ "$unamestr" = "Linux" ]]; then'
|
or uname is not 'Darwin' \
|
||||||
elif darwin_unsupported:
|
or not darwin_unsupported:
|
||||||
print 'if [[ "$unamestr" != "Darwin" ]]; then'
|
print """cat <<EOF > run_test_%(ind)d.sh
|
||||||
|
#!/bin/sh
|
||||||
|
|
||||||
print config_line
|
set -e
|
||||||
print "make clean"
|
|
||||||
print "make -j" + str(MAKE_J_VAL) + " check"
|
|
||||||
|
|
||||||
if linux_supported or darwin_unsupported:
|
abort() {
|
||||||
print 'fi'
|
echo "==> Error" >> run_test.log
|
||||||
|
echo "Error; see run_tests.out/run_test_%(ind)d.out/run_test.log"
|
||||||
|
exit 255 # Special exit code tells xargs to terminate.
|
||||||
|
}
|
||||||
|
|
||||||
|
# Environment variables are not supported.
|
||||||
|
run_cmd() {
|
||||||
|
echo "==> \$@" >> run_test.log
|
||||||
|
\$@ >> run_test.log 2>&1 || abort
|
||||||
|
}
|
||||||
|
|
||||||
|
echo "=> run_test_%(ind)d: %(config_line)s"
|
||||||
|
mkdir run_test_%(ind)d.out
|
||||||
|
cd run_test_%(ind)d.out
|
||||||
|
|
||||||
|
echo "==> %(config_line)s" >> run_test.log
|
||||||
|
%(config_line)s >> run_test.log 2>&1 || abort
|
||||||
|
|
||||||
|
run_cmd make all tests
|
||||||
|
run_cmd make check
|
||||||
|
run_cmd make distclean
|
||||||
|
EOF
|
||||||
|
chmod 755 run_test_%(ind)d.sh""" % {'ind': ind, 'config_line': config_line}
|
||||||
|
ind += 1
|
||||||
|
|
||||||
|
print 'for i in `seq 0 %(last_ind)d` ; do echo run_test_${i}.sh ; done | xargs -P %(nparallel)d -n 1 sh' % {'last_ind': ind-1, 'nparallel': nparallel}
|
||||||
|
Loading…
Reference in New Issue
Block a user