mirror of
https://github.com/XRPLF/rippled.git
synced 2025-11-04 19:25:51 +00:00
Add build_all script to Builds:
This script will compile rippled, run unit tests and then delete all build artifacts in many possible configurations (in Linux). The script will automatically use all available cores to improve compile speed. All commands are chained via &&, so if any of them fail, the script will stop at the error.
This commit is contained in:
MarkusTeufelberger
committed by
Nik Bougalis
Nik Bougalis
parent
cd11f3755e
commit
55377c12d3
182
Builds/Test.py
182
Builds/Test.py
@@ -48,18 +48,38 @@ import itertools
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
def powerset(iterable):
|
||||
"""powerset([1,2,3]) --> () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3)"""
|
||||
s = list(iterable)
|
||||
return itertools.chain.from_iterable(itertools.combinations(s, r) for r in range(len(s) + 1))
|
||||
|
||||
|
||||
IS_WINDOWS = platform.system().lower() == 'windows'
|
||||
|
||||
if IS_WINDOWS:
|
||||
BINARY_RE = re.compile(r'build\\([^\\]+)\\rippled.exe')
|
||||
|
||||
ALL_TARGETS = [('debug',), ('release',)]
|
||||
else:
|
||||
BINARY_RE = re.compile(r'build/([^/]+)/rippled')
|
||||
ALL_TARGETS = [(cc + "." + target,)
|
||||
for cc in ['gcc', 'clang']
|
||||
for target in ['debug', 'release', 'coverage', 'profile',
|
||||
'debug.nounity', 'release.nounity', 'coverage.nounity', 'profile.nounity']]
|
||||
|
||||
ALL_TARGETS = ['debug', 'release']
|
||||
# list of tuples of all possible options
|
||||
if IS_WINDOWS:
|
||||
ALL_OPTIONS = [tuple(x) for x in powerset(['--assert'])]
|
||||
else:
|
||||
ALL_OPTIONS = list(set(
|
||||
[tuple(x) for x in powerset(['--ninja', '--static', '--assert', '--sanitize=address'])] +
|
||||
[tuple(x) for x in powerset(['--ninja', '--static', '--assert', '--sanitize=thread'])]))
|
||||
|
||||
# list of tuples of all possible options + all possible targets
|
||||
ALL_BUILDS = [options + target
|
||||
for target in ALL_TARGETS
|
||||
for options in ALL_OPTIONS]
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Test.py - run ripple tests'
|
||||
@@ -75,55 +95,74 @@ parser.add_argument(
|
||||
'--keep_going', '-k',
|
||||
action='store_true',
|
||||
help='Keep going after one configuration has failed.',
|
||||
)
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--silent', '-s',
|
||||
action='store_true',
|
||||
help='Silence all messages except errors',
|
||||
)
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--verbose', '-v',
|
||||
action='store_true',
|
||||
help=('Report more information about which commands are executed and the '
|
||||
'results.'),
|
||||
)
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--test', '-t',
|
||||
default='',
|
||||
help='Add a prefix for unit tests',
|
||||
)
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--nonpm', '-n',
|
||||
action='store_true',
|
||||
help='Do not run npm tests',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--clean', '-c',
|
||||
action='store_true',
|
||||
help='delete all build artifacts after testing',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'scons_args',
|
||||
default=(),
|
||||
nargs='*'
|
||||
)
|
||||
)
|
||||
|
||||
ARGS = parser.parse_args()
|
||||
|
||||
def shell(*cmd, **kwds):
|
||||
"Execute a shell command and return the output."
|
||||
silent = kwds.pop('silent', ARGS.silent)
|
||||
verbose = not silent and kwds.pop('verbose', ARGS.verbose)
|
||||
|
||||
def shell(cmd, args=(), silent=False):
|
||||
""""Execute a shell command and return the output."""
|
||||
silent = ARGS.silent or silent
|
||||
verbose = not silent and ARGS.verbose
|
||||
if verbose:
|
||||
print('$', ' '.join(cmd))
|
||||
kwds['shell'] = IS_WINDOWS
|
||||
print('$' + cmd, *args)
|
||||
|
||||
command = (cmd,) + args
|
||||
|
||||
process = subprocess.Popen(
|
||||
cmd,
|
||||
command,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
**kwds)
|
||||
shell=IS_WINDOWS)
|
||||
lines = []
|
||||
count = 0
|
||||
for line in process.stdout:
|
||||
lines.append(line)
|
||||
# Python 2 vs. Python 3
|
||||
if isinstance(line, str):
|
||||
decoded = line
|
||||
else:
|
||||
decoded = line.decode()
|
||||
lines.append(decoded)
|
||||
if verbose:
|
||||
print(line, end='')
|
||||
print(decoded, end='')
|
||||
elif not silent:
|
||||
count += 1
|
||||
if count >= 80:
|
||||
@@ -137,27 +176,16 @@ def shell(*cmd, **kwds):
|
||||
process.wait()
|
||||
return process.returncode, lines
|
||||
|
||||
if __name__ == '__main__':
|
||||
args = list(ARGS.scons_args)
|
||||
if ARGS.all:
|
||||
for a in ALL_TARGETS:
|
||||
if a not in args:
|
||||
args.append(a)
|
||||
print('Building:', *(args or ['(default)']))
|
||||
|
||||
# Build everything.
|
||||
resultcode, lines = shell('scons', *args)
|
||||
if resultcode:
|
||||
print('Build FAILED:')
|
||||
if not ARGS.verbose:
|
||||
print(*lines, sep='')
|
||||
exit(1)
|
||||
|
||||
# Now extract the executable names and corresponding targets.
|
||||
def run_tests(args):
|
||||
failed = []
|
||||
_, lines = shell('scons', '-n', '--tree=derived', *args, silent=True)
|
||||
if IS_WINDOWS:
|
||||
binary_re = re.compile(r'build\\([^\\]+)\\rippled.exe')
|
||||
else:
|
||||
binary_re = re.compile(r'build/([^/]+)/rippled')
|
||||
_, lines = shell('scons', ('-n', '--tree=derived',) + args, silent=True)
|
||||
for line in lines:
|
||||
match = BINARY_RE.search(line)
|
||||
match = binary_re.search(line)
|
||||
if match:
|
||||
executable, target = match.group(0, 1)
|
||||
|
||||
@@ -165,8 +193,8 @@ if __name__ == '__main__':
|
||||
testflag = '--unittest'
|
||||
if ARGS.test:
|
||||
testflag += ('=' + ARGS.test)
|
||||
resultcode, lines = shell(executable, (testflag,))
|
||||
|
||||
resultcode, lines = shell(executable, testflag)
|
||||
if resultcode:
|
||||
print('ERROR:', *lines, sep='')
|
||||
failed.append([target, 'unittest'])
|
||||
@@ -174,18 +202,70 @@ if __name__ == '__main__':
|
||||
break
|
||||
ARGS.verbose and print(*lines, sep='')
|
||||
|
||||
print('npm tests for', target)
|
||||
resultcode, lines = shell('npm', 'test', '--rippled=' + executable)
|
||||
if resultcode:
|
||||
print('ERROR:\n', *lines, sep='')
|
||||
failed.append([target, 'npm'])
|
||||
if not ARGS.keep_going:
|
||||
break
|
||||
else:
|
||||
ARGS.verbose and print(*lines, sep='')
|
||||
if not ARGS.nonpm:
|
||||
print('npm tests for', target)
|
||||
resultcode, lines = shell('npm', ('test', '--rippled=' + executable,))
|
||||
if resultcode:
|
||||
print('ERROR:\n', *lines, sep='')
|
||||
failed.append([target, 'npm'])
|
||||
if not ARGS.keep_going:
|
||||
break
|
||||
else:
|
||||
ARGS.verbose and print(*lines, sep='')
|
||||
return failed
|
||||
|
||||
if failed:
|
||||
print('FAILED:', *(':'.join(f) for f in failed))
|
||||
|
||||
def run_build(args=None):
|
||||
print('Building:', *args or ('(default)',))
|
||||
resultcode, lines = shell('scons', args)
|
||||
|
||||
if resultcode:
|
||||
print('Build FAILED:')
|
||||
if not ARGS.verbose:
|
||||
print(*lines, sep='')
|
||||
exit(1)
|
||||
if '--ninja' in args:
|
||||
resultcode, lines = shell('ninja')
|
||||
|
||||
if resultcode:
|
||||
print('Ninja build FAILED:')
|
||||
if not ARGS.verbose:
|
||||
print(*lines, sep='')
|
||||
exit(1)
|
||||
|
||||
|
||||
def main():
|
||||
if ARGS.all:
|
||||
to_build = ALL_BUILDS
|
||||
else:
|
||||
print('Success')
|
||||
to_build = [tuple(ARGS.scons_args)]
|
||||
|
||||
for build in to_build:
|
||||
args = ()
|
||||
# additional arguments come first
|
||||
for arg in list(ARGS.scons_args):
|
||||
if arg not in build:
|
||||
args += (arg,)
|
||||
args += build
|
||||
|
||||
run_build(args)
|
||||
failed = run_tests(args)
|
||||
|
||||
if failed:
|
||||
print('FAILED:', *(':'.join(f) for f in failed))
|
||||
if not ARGS.keep_going:
|
||||
exit(1)
|
||||
else:
|
||||
print('Success')
|
||||
|
||||
if ARGS.clean:
|
||||
shutil.rmtree('build')
|
||||
if '--ninja' in args:
|
||||
os.remove('build.ninja')
|
||||
os.remove('.ninja_deps')
|
||||
os.remove('.ninja_log')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
exit(0)
|
||||
|
||||
6
Builds/build_all.sh
Normal file
6
Builds/build_all.sh
Normal file
@@ -0,0 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
num_procs=$(lscpu -p | grep -v '^#' | sort -u -t, -k 2,4 | wc -l) # number of physical cores
|
||||
|
||||
cd ..
|
||||
./Builds/Test.py -a -c -- -j${num_procs}
|
||||
Reference in New Issue
Block a user