Add build_all script to Builds:

This script will compile rippled, run unit tests and then delete all build
artifacts in many possible configurations (in Linux).

The script will automatically use all available cores to improve compile
speed. All commands are chained via &&, so if any of them fail, the script
will stop at the error.

Builds/Test.py

@@ -48,18 +48,38 @@ import itertools
 import os
 import platform
 import re
+import shutil
 import subprocess
-import sys
+
+
+def powerset(iterable):
+    """powerset([1,2,3]) --> () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3)"""
+    s = list(iterable)
+    return itertools.chain.from_iterable(itertools.combinations(s, r) for r in range(len(s) + 1))
+
 
 IS_WINDOWS = platform.system().lower() == 'windows'
 
 if IS_WINDOWS:
-    BINARY_RE = re.compile(r'build\\([^\\]+)\\rippled.exe')
+    ALL_TARGETS = [('debug',), ('release',)]
-
 else:
-    BINARY_RE = re.compile(r'build/([^/]+)/rippled')
+    ALL_TARGETS = [(cc + "." + target,)
+                   for cc in ['gcc', 'clang']
+                   for target in ['debug', 'release', 'coverage', 'profile',
+                                  'debug.nounity', 'release.nounity', 'coverage.nounity', 'profile.nounity']]
 
-ALL_TARGETS = ['debug', 'release']
+# list of tuples of all possible options
+if IS_WINDOWS:
+    ALL_OPTIONS = [tuple(x) for x in powerset(['--assert'])]
+else:
+    ALL_OPTIONS = list(set(
+        [tuple(x) for x in powerset(['--ninja', '--static', '--assert', '--sanitize=address'])] +
+        [tuple(x) for x in powerset(['--ninja', '--static', '--assert', '--sanitize=thread'])]))
+
+# list of tuples of all possible options + all possible targets
+ALL_BUILDS = [options + target
+              for target in ALL_TARGETS
+              for options in ALL_OPTIONS]
 
 parser = argparse.ArgumentParser(
     description='Test.py - run ripple tests'
@@ -75,55 +95,74 @@ parser.add_argument(
     '--keep_going', '-k',
     action='store_true',
     help='Keep going after one configuration has failed.',
- )
+)
 
 parser.add_argument(
     '--silent', '-s',
     action='store_true',
     help='Silence all messages except errors',
- )
+)
 
 parser.add_argument(
     '--verbose', '-v',
     action='store_true',
     help=('Report more information about which commands are executed and the '
           'results.'),
- )
+)
 
 parser.add_argument(
     '--test', '-t',
     default='',
     help='Add a prefix for unit tests',
- )
+)
+
+parser.add_argument(
+    '--nonpm', '-n',
+    action='store_true',
+    help='Do not run npm tests',
+)
+
+parser.add_argument(
+    '--clean', '-c',
+    action='store_true',
+    help='delete all build artifacts after testing',
+)
 
 parser.add_argument(
     'scons_args',
     default=(),
     nargs='*'
-    )
+)
 
 ARGS = parser.parse_args()
 
-def shell(*cmd, **kwds):
+
-    "Execute a shell command and return the output."
+def shell(cmd, args=(), silent=False):
-    silent = kwds.pop('silent', ARGS.silent)
+    """"Execute a shell command and return the output."""
-    verbose = not silent and kwds.pop('verbose', ARGS.verbose)
+    silent = ARGS.silent or silent
+    verbose = not silent and ARGS.verbose
     if verbose:
-        print('$', ' '.join(cmd))
+        print('$' + cmd, *args)
-    kwds['shell'] = IS_WINDOWS
+
+    command = (cmd,) + args
 
     process = subprocess.Popen(
-        cmd,
+        command,
         stdin=subprocess.PIPE,
         stdout=subprocess.PIPE,
         stderr=subprocess.STDOUT,
-        **kwds)
+        shell=IS_WINDOWS)
     lines = []
     count = 0
     for line in process.stdout:
-        lines.append(line)
+        # Python 2 vs. Python 3
+        if isinstance(line, str):
+            decoded = line
+        else:
+            decoded = line.decode()
+        lines.append(decoded)
         if verbose:
-            print(line, end='')
+            print(decoded, end='')
         elif not silent:
             count += 1
             if count >= 80:
@@ -137,27 +176,16 @@ def shell(*cmd, **kwds):
     process.wait()
     return process.returncode, lines
 
-if __name__ == '__main__':
-    args = list(ARGS.scons_args)
-    if ARGS.all:
-        for a in ALL_TARGETS:
-            if a not in args:
-                args.append(a)
-    print('Building:', *(args or ['(default)']))
 
-    # Build everything.
+def run_tests(args):
-    resultcode, lines = shell('scons', *args)
-    if resultcode:
-        print('Build FAILED:')
-        if not ARGS.verbose:
-            print(*lines, sep='')
-        exit(1)
-
-    # Now extract the executable names and corresponding targets.
     failed = []
-    _, lines = shell('scons', '-n', '--tree=derived', *args, silent=True)
+    if IS_WINDOWS:
+        binary_re = re.compile(r'build\\([^\\]+)\\rippled.exe')
+    else:
+        binary_re = re.compile(r'build/([^/]+)/rippled')
+    _, lines = shell('scons', ('-n', '--tree=derived',) + args, silent=True)
     for line in lines:
-        match = BINARY_RE.search(line)
+        match = binary_re.search(line)
         if match:
             executable, target = match.group(0, 1)
 
@@ -165,8 +193,8 @@ if __name__ == '__main__':
             testflag = '--unittest'
             if ARGS.test:
                 testflag += ('=' + ARGS.test)
+            resultcode, lines = shell(executable, (testflag,))
 
-            resultcode, lines = shell(executable, testflag)
             if resultcode:
                 print('ERROR:', *lines, sep='')
                 failed.append([target, 'unittest'])
@@ -174,18 +202,70 @@ if __name__ == '__main__':
                     break
             ARGS.verbose and print(*lines, sep='')
 
-            print('npm tests for', target)
+            if not ARGS.nonpm:
-            resultcode, lines = shell('npm', 'test', '--rippled=' + executable)
+                print('npm tests for', target)
-            if resultcode:
+                resultcode, lines = shell('npm', ('test', '--rippled=' + executable,))
-                print('ERROR:\n', *lines, sep='')
+                if resultcode:
-                failed.append([target, 'npm'])
+                    print('ERROR:\n', *lines, sep='')
-                if not ARGS.keep_going:
+                    failed.append([target, 'npm'])
-                    break
+                    if not ARGS.keep_going:
-            else:
+                        break
-                ARGS.verbose and print(*lines, sep='')
+                else:
+                    ARGS.verbose and print(*lines, sep='')
+    return failed
 
-    if failed:
+
-        print('FAILED:', *(':'.join(f) for f in failed))
+def run_build(args=None):
+    print('Building:', *args or ('(default)',))
+    resultcode, lines = shell('scons', args)
+
+    if resultcode:
+        print('Build FAILED:')
+        if not ARGS.verbose:
+            print(*lines, sep='')
         exit(1)
+    if '--ninja' in args:
+        resultcode, lines = shell('ninja')
+
+        if resultcode:
+            print('Ninja build FAILED:')
+            if not ARGS.verbose:
+                print(*lines, sep='')
+            exit(1)
+
+
+def main():
+    if ARGS.all:
+        to_build = ALL_BUILDS
     else:
-        print('Success')
+        to_build = [tuple(ARGS.scons_args)]
+
+    for build in to_build:
+        args = ()
+        # additional arguments come first
+        for arg in list(ARGS.scons_args):
+            if arg not in build:
+                args += (arg,)
+        args += build
+
+        run_build(args)
+        failed = run_tests(args)
+
+        if failed:
+            print('FAILED:', *(':'.join(f) for f in failed))
+            if not ARGS.keep_going:
+                exit(1)
+        else:
+            print('Success')
+
+        if ARGS.clean:
+            shutil.rmtree('build')
+            if '--ninja' in args:
+                os.remove('build.ninja')
+                os.remove('.ninja_deps')
+                os.remove('.ninja_log')
+
+
+if __name__ == '__main__':
+    main()
+    exit(0)

Builds/build_all.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+num_procs=$(lscpu -p | grep -v '^#' | sort -u -t, -k 2,4 | wc -l) # number of physical cores
+
+cd ..
+./Builds/Test.py -a -c -- -j${num_procs}