From 55377c12d34bed58daf0c10ce66b334f9eeeea4a Mon Sep 17 00:00:00 2001 From: MarkusTeufelberger Date: Thu, 12 May 2016 20:18:49 +0200 Subject: [PATCH] Add build_all script to Builds: This script will compile rippled, run unit tests and then delete all build artifacts in many possible configurations (in Linux). The script will automatically use all available cores to improve compile speed. All commands are chained via &&, so if any of them fail, the script will stop at the error. --- Builds/Test.py | 182 +++++++++++++++++++++++++++++++------------- Builds/build_all.sh | 6 ++ 2 files changed, 137 insertions(+), 51 deletions(-) create mode 100644 Builds/build_all.sh diff --git a/Builds/Test.py b/Builds/Test.py index cb9da5852..c5deaba86 100755 --- a/Builds/Test.py +++ b/Builds/Test.py @@ -48,18 +48,38 @@ import itertools import os import platform import re +import shutil import subprocess -import sys + + +def powerset(iterable): + """powerset([1,2,3]) --> () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3)""" + s = list(iterable) + return itertools.chain.from_iterable(itertools.combinations(s, r) for r in range(len(s) + 1)) + IS_WINDOWS = platform.system().lower() == 'windows' if IS_WINDOWS: - BINARY_RE = re.compile(r'build\\([^\\]+)\\rippled.exe') - + ALL_TARGETS = [('debug',), ('release',)] else: - BINARY_RE = re.compile(r'build/([^/]+)/rippled') + ALL_TARGETS = [(cc + "." + target,) + for cc in ['gcc', 'clang'] + for target in ['debug', 'release', 'coverage', 'profile', + 'debug.nounity', 'release.nounity', 'coverage.nounity', 'profile.nounity']] -ALL_TARGETS = ['debug', 'release'] +# list of tuples of all possible options +if IS_WINDOWS: + ALL_OPTIONS = [tuple(x) for x in powerset(['--assert'])] +else: + ALL_OPTIONS = list(set( + [tuple(x) for x in powerset(['--ninja', '--static', '--assert', '--sanitize=address'])] + + [tuple(x) for x in powerset(['--ninja', '--static', '--assert', '--sanitize=thread'])])) + +# list of tuples of all possible options + all possible targets +ALL_BUILDS = [options + target + for target in ALL_TARGETS + for options in ALL_OPTIONS] parser = argparse.ArgumentParser( description='Test.py - run ripple tests' @@ -75,55 +95,74 @@ parser.add_argument( '--keep_going', '-k', action='store_true', help='Keep going after one configuration has failed.', - ) +) parser.add_argument( '--silent', '-s', action='store_true', help='Silence all messages except errors', - ) +) parser.add_argument( '--verbose', '-v', action='store_true', help=('Report more information about which commands are executed and the ' 'results.'), - ) +) parser.add_argument( '--test', '-t', default='', help='Add a prefix for unit tests', - ) +) + +parser.add_argument( + '--nonpm', '-n', + action='store_true', + help='Do not run npm tests', +) + +parser.add_argument( + '--clean', '-c', + action='store_true', + help='delete all build artifacts after testing', +) parser.add_argument( 'scons_args', default=(), nargs='*' - ) +) ARGS = parser.parse_args() -def shell(*cmd, **kwds): - "Execute a shell command and return the output." - silent = kwds.pop('silent', ARGS.silent) - verbose = not silent and kwds.pop('verbose', ARGS.verbose) + +def shell(cmd, args=(), silent=False): + """"Execute a shell command and return the output.""" + silent = ARGS.silent or silent + verbose = not silent and ARGS.verbose if verbose: - print('$', ' '.join(cmd)) - kwds['shell'] = IS_WINDOWS + print('$' + cmd, *args) + + command = (cmd,) + args process = subprocess.Popen( - cmd, + command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - **kwds) + shell=IS_WINDOWS) lines = [] count = 0 for line in process.stdout: - lines.append(line) + # Python 2 vs. Python 3 + if isinstance(line, str): + decoded = line + else: + decoded = line.decode() + lines.append(decoded) if verbose: - print(line, end='') + print(decoded, end='') elif not silent: count += 1 if count >= 80: @@ -137,27 +176,16 @@ def shell(*cmd, **kwds): process.wait() return process.returncode, lines -if __name__ == '__main__': - args = list(ARGS.scons_args) - if ARGS.all: - for a in ALL_TARGETS: - if a not in args: - args.append(a) - print('Building:', *(args or ['(default)'])) - # Build everything. - resultcode, lines = shell('scons', *args) - if resultcode: - print('Build FAILED:') - if not ARGS.verbose: - print(*lines, sep='') - exit(1) - - # Now extract the executable names and corresponding targets. +def run_tests(args): failed = [] - _, lines = shell('scons', '-n', '--tree=derived', *args, silent=True) + if IS_WINDOWS: + binary_re = re.compile(r'build\\([^\\]+)\\rippled.exe') + else: + binary_re = re.compile(r'build/([^/]+)/rippled') + _, lines = shell('scons', ('-n', '--tree=derived',) + args, silent=True) for line in lines: - match = BINARY_RE.search(line) + match = binary_re.search(line) if match: executable, target = match.group(0, 1) @@ -165,8 +193,8 @@ if __name__ == '__main__': testflag = '--unittest' if ARGS.test: testflag += ('=' + ARGS.test) + resultcode, lines = shell(executable, (testflag,)) - resultcode, lines = shell(executable, testflag) if resultcode: print('ERROR:', *lines, sep='') failed.append([target, 'unittest']) @@ -174,18 +202,70 @@ if __name__ == '__main__': break ARGS.verbose and print(*lines, sep='') - print('npm tests for', target) - resultcode, lines = shell('npm', 'test', '--rippled=' + executable) - if resultcode: - print('ERROR:\n', *lines, sep='') - failed.append([target, 'npm']) - if not ARGS.keep_going: - break - else: - ARGS.verbose and print(*lines, sep='') + if not ARGS.nonpm: + print('npm tests for', target) + resultcode, lines = shell('npm', ('test', '--rippled=' + executable,)) + if resultcode: + print('ERROR:\n', *lines, sep='') + failed.append([target, 'npm']) + if not ARGS.keep_going: + break + else: + ARGS.verbose and print(*lines, sep='') + return failed - if failed: - print('FAILED:', *(':'.join(f) for f in failed)) + +def run_build(args=None): + print('Building:', *args or ('(default)',)) + resultcode, lines = shell('scons', args) + + if resultcode: + print('Build FAILED:') + if not ARGS.verbose: + print(*lines, sep='') exit(1) + if '--ninja' in args: + resultcode, lines = shell('ninja') + + if resultcode: + print('Ninja build FAILED:') + if not ARGS.verbose: + print(*lines, sep='') + exit(1) + + +def main(): + if ARGS.all: + to_build = ALL_BUILDS else: - print('Success') + to_build = [tuple(ARGS.scons_args)] + + for build in to_build: + args = () + # additional arguments come first + for arg in list(ARGS.scons_args): + if arg not in build: + args += (arg,) + args += build + + run_build(args) + failed = run_tests(args) + + if failed: + print('FAILED:', *(':'.join(f) for f in failed)) + if not ARGS.keep_going: + exit(1) + else: + print('Success') + + if ARGS.clean: + shutil.rmtree('build') + if '--ninja' in args: + os.remove('build.ninja') + os.remove('.ninja_deps') + os.remove('.ninja_log') + + +if __name__ == '__main__': + main() + exit(0) diff --git a/Builds/build_all.sh b/Builds/build_all.sh new file mode 100644 index 000000000..52beed007 --- /dev/null +++ b/Builds/build_all.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +num_procs=$(lscpu -p | grep -v '^#' | sort -u -t, -k 2,4 | wc -l) # number of physical cores + +cd .. +./Builds/Test.py -a -c -- -j${num_procs}