From d4f3f9a03f0ba199a7b51c93bdebe8236e0105da Mon Sep 17 00:00:00 2001 From: David Goldblatt Date: Wed, 25 Jan 2017 12:58:50 -0800 Subject: [PATCH] Beef up travis CI integration testing Introduces gen_travis.py, which generates .travis.yml, and updates .travis.yml to be the generated version. The travis build matrix approach doesn't play well with mixing and matching various different environment settings, so we generate every build explicitly, rather than letting them do it for us. To avoid abusing travis resources (and save us time waiting for CI results), we don't test every possible combination of options; we only check up to 2 unusual settings at a time. --- .travis.yml | 88 +++++++++++++++++++++++++++++++++++++------ scripts/gen_travis.py | 85 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 162 insertions(+), 11 deletions(-) create mode 100755 scripts/gen_travis.py diff --git a/.travis.yml b/.travis.yml index 1fed4f8e..b563928c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,29 +1,95 @@ -language: c +language: generic matrix: include: - os: linux - compiler: gcc + env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="" + - os: osx + env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="" - os: linux - compiler: gcc - env: - - EXTRA_FLAGS=-m32 + env: CC=clang COMPILER_FLAGS="" CONFIGURE_FLAGS="" + - os: linux + env: CC=gcc COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="" addons: apt: packages: - - gcc-multilib + - gcc-multilib + - os: linux + env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug" + - os: linux + env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof" + - os: linux + env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats" + - os: linux + env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-tcache" - os: osx - compiler: clang + env: CC=clang COMPILER_FLAGS="" CONFIGURE_FLAGS="" - os: osx - compiler: clang - env: - - EXTRA_FLAGS=-m32 + env: CC=gcc COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="" + - os: osx + env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug" + - os: osx + env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats" + - os: osx + env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-tcache" + - os: linux + env: CC=clang COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="" + addons: + apt: + packages: + - gcc-multilib + - os: linux + env: CC=clang COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug" + - os: linux + env: CC=clang COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof" + - os: linux + env: CC=clang COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats" + - os: linux + env: CC=clang COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-tcache" + - os: linux + env: CC=gcc COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--enable-debug" + addons: + apt: + packages: + - gcc-multilib + - os: linux + env: CC=gcc COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--enable-prof" + addons: + apt: + packages: + - gcc-multilib + - os: linux + env: CC=gcc COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--disable-stats" + addons: + apt: + packages: + - gcc-multilib + - os: linux + env: CC=gcc COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--disable-tcache" + addons: + apt: + packages: + - gcc-multilib + - os: linux + env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --enable-prof" + - os: linux + env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --disable-stats" + - os: linux + env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --disable-tcache" + - os: linux + env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --disable-stats" + - os: linux + env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --disable-tcache" + - os: linux + env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats --disable-tcache" + before_script: - autoconf - - ./configure${EXTRA_FLAGS:+ CC="$CC $EXTRA_FLAGS"} + - ./configure ${COMPILER_FLAGS:+ CC="$CC $COMPILER_FLAGS" } $CONFIGURE_FLAGS - make -j3 - make -j3 tests script: - make check + diff --git a/scripts/gen_travis.py b/scripts/gen_travis.py new file mode 100755 index 00000000..ccbcaf8c --- /dev/null +++ b/scripts/gen_travis.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python + +from itertools import combinations + +travis_template = """\ +language: generic + +matrix: + include: +%s + +before_script: + - autoconf + - ./configure ${COMPILER_FLAGS:+ \ + CC="$CC $COMPILER_FLAGS" } \ + $CONFIGURE_FLAGS + - make -j3 + - make -j3 tests + +script: + - make check +""" + +# The 'default' configuration is gcc, on linux, with no compiler or configure +# flags. We also test with clang, -m32, --enable-debug, --enable-prof, +# --disable-stats, and --disable-tcache. To avoid abusing travis though, we +# don't test all 2**7 = 128 possible combinations of these; instead, we only +# test combinations of up to 2 'unusual' settings, under the hope that bugs +# involving interactions of such settings are rare. +# things at once, for C(7, 0) + C(7, 1) + C(7, 2) = 29 +MAX_UNUSUAL_OPTIONS = 2 + +os_default = 'linux' +os_unusual = 'osx' + +compilers_default = 'CC=gcc' +compilers_unusual = 'CC=clang' + +compiler_flag_unusuals = ['-m32'] + +configure_flag_unusuals = [ + '--enable-debug', '--enable-prof', '--disable-stats', '--disable-tcache', +] + +all_unusuals = ( + [os_unusual] + [compilers_unusual] + compiler_flag_unusuals + + configure_flag_unusuals +) + +unusual_combinations_to_test = [] +for i in xrange(MAX_UNUSUAL_OPTIONS + 1): + unusual_combinations_to_test += combinations(all_unusuals, i) + +include_rows = "" +for unusual_combination in unusual_combinations_to_test: + os = os_default + if os_unusual in unusual_combination: + os = os_unusual + + compilers = compilers_default + if compilers_unusual in unusual_combination: + compilers = compilers_unusual + + compiler_flags = [ + x for x in unusual_combination if x in compiler_flag_unusuals] + + configure_flags = [ + x for x in unusual_combination if x in configure_flag_unusuals] + + # Filter out an unsupported configuration - heap profiling on OS X. + if os == 'osx' and '--enable-prof' in configure_flags: + continue + + env_string = '{} COMPILER_FLAGS="{}" CONFIGURE_FLAGS="{}"'.format( + compilers, " ".join(compiler_flags), " ".join(configure_flags)) + + include_rows += ' - os: %s\n' % os + include_rows += ' env: %s\n' % env_string + if '-m32' in unusual_combination and os == 'linux': + include_rows += ' addons:\n' + include_rows += ' apt:\n' + include_rows += ' packages:\n' + include_rows += ' - gcc-multilib\n' + +print travis_template % include_rows