Support CMake in Test.py

* scons will remain the default build type.
* New set of command line switches to control CMake.

Builds/CMake/CMakeFuncs.cmake

@@ -12,7 +12,7 @@ endif()
 
 macro(parse_target)
 
-  if (NOT target)
+  if (NOT target OR target STREQUAL "default")
     if (NOT CMAKE_BUILD_TYPE)
       set(CMAKE_BUILD_TYPE Debug)
     endif()
@@ -110,14 +110,27 @@ macro(parse_target)
 
     endwhile()
   endif()
-  # Promote these values to the CACHE, then unset the locals
+
-  # to prevent shadowing.
+  if(CMAKE_C_COMPILER MATCHES "-NOTFOUND$" OR
+    CMAKE_CXX_COMPILER MATCHES "-NOTFOUND$")
+    message(FATAL_ERROR "Can not find appropriate compiler for target ${target}")
+  endif()
+
+  # If defined, promote the compiler path values to the CACHE, then
+  # unset the locals to prevent shadowing. Some scenarios do not
+  # need or want to find a compiler, such as -GNinja under Windows.
+  # Setting these values in those case may prevent CMake from finding
+  # a valid compiler.
+  if (CMAKE_C_COMPILER)
     set(CMAKE_C_COMPILER ${CMAKE_C_COMPILER} CACHE FILEPATH
       "Path to a program" FORCE)
     unset(CMAKE_C_COMPILER)
+  endif (CMAKE_C_COMPILER)
+  if (CMAKE_CXX_COMPILER)
     set(CMAKE_CXX_COMPILER ${CMAKE_CXX_COMPILER} CACHE FILEPATH
       "Path to a program" FORCE)
     unset(CMAKE_CXX_COMPILER)
+  endif (CMAKE_CXX_COMPILER)
 
   if (release)
     set(CMAKE_BUILD_TYPE Release)
@@ -308,6 +321,7 @@ macro(use_boost)
     if ((NOT DEFINED BOOST_ROOT) AND (DEFINED ENV{BOOST_ROOT}))
         set(BOOST_ROOT $ENV{BOOST_ROOT})
     endif()
+    file(TO_CMAKE_PATH ${BOOST_ROOT} BOOST_ROOT)
     if(WIN32 OR CYGWIN)
         # Workaround for MSVC having two boost versions - x86 and x64 on same PC in stage folders
         if(DEFINED BOOST_ROOT)
@@ -633,17 +647,17 @@ macro(setup_build_boilerplate)
       /FS
       /MP                # Multiprocessor compilation
       /openmp-           # pragma omp: disabled
-      /Zc:forScope         # Language extension: for scope
+      /Zc:forScope       # Language conformance: for scope
       /Zi                # Generate complete debug info
       /errorReport:none  # No error reporting to Internet
       /nologo            # Suppress login banner
       /W3                # Warning level 3
       /WX-               # Disable warnings as errors
-      /wd"4018"
+      /wd4018            # Disable signed/unsigned comparison warnings
-      /wd"4244"
+      /wd4244            # Disable float to int possible loss of data warnings
-      /wd"4267"
+      /wd4267            # Disable size_t to T possible loss of data warnings
-      /wd"4800"            # Disable C4800(int to bool performance)
+      /wd4800            # Disable C4800(int to bool performance)
-      /wd"4503"            # Decorated name length exceeded, name was truncated
+      /wd4503            # Decorated name length exceeded, name was truncated
       )
     add_definitions(
       -D_WIN32_WINNT=0x6000

Builds/Test.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 #    This file is part of rippled: https://github.com/ripple/rippled
-#    Copyright (c) 2012 - 2015 Ripple Labs Inc.
+#    Copyright (c) 2012 - 2017 Ripple Labs Inc.
 #
 #    Permission to use, copy, modify, and/or distribute this software for any
 #    purpose  with  or without fee is hereby granted, provided that the above
@@ -27,6 +27,11 @@ the -- flag - like this:
 
     ./Builds/Test.py -- -j4   # Pass -j4 to scons.
 
+To build with CMake, use the --cmake flag, or any of the specific configuration
+flags
+
+    ./Builds/Test.py --cmake -- -j4  # Pass -j4 to cmake --build
+
 
 Common problems:
 
@@ -61,6 +66,38 @@ def powerset(iterable):
 IS_WINDOWS = platform.system().lower() == 'windows'
 IS_OS_X = platform.system().lower() == 'darwin'
 
+# CMake
+if IS_WINDOWS:
+    CMAKE_UNITY_CONFIGS = ['Debug', 'Release']
+    CMAKE_NONUNITY_CONFIGS = ['DebugClassic', 'ReleaseClassic']
+else:
+    CMAKE_UNITY_CONFIGS = []
+    CMAKE_NONUNITY_CONFIGS = []
+CMAKE_UNITY_COMBOS = { '' : [['rippled', 'rippled_classic'], CMAKE_UNITY_CONFIGS],
+    '.nounity' : [['rippled', 'rippled_unity'], CMAKE_NONUNITY_CONFIGS] }
+
+if IS_WINDOWS:
+    CMAKE_DIR_TARGETS = { ('msvc' + unity,) : targets for unity, targets in
+        CMAKE_UNITY_COMBOS.iteritems() }
+elif IS_OS_X:
+    CMAKE_DIR_TARGETS = { (build + unity,) : targets
+                   for build in ['debug', 'release']
+                   for unity, targets in CMAKE_UNITY_COMBOS.iteritems() }
+else:
+    CMAKE_DIR_TARGETS = { (cc + "." + build + unity,) : targets
+                   for cc in ['gcc', 'clang']
+                   for build in ['debug', 'release', 'coverage', 'profile']
+                   for unity, targets in CMAKE_UNITY_COMBOS.iteritems() }
+
+# list of tuples of all possible options
+if IS_WINDOWS or IS_OS_X:
+    CMAKE_ALL_GENERATE_OPTIONS = [tuple(x) for x in powerset(['-GNinja', '-Dassert=true'])]
+else:
+    CMAKE_ALL_GENERATE_OPTIONS = list(set(
+        [tuple(x) for x in powerset(['-GNinja', '-Dstatic=true', '-Dassert=true', '-Dsan=address'])] +
+        [tuple(x) for x in powerset(['-GNinja', '-Dstatic=true', '-Dassert=true', '-Dsan=thread'])]))
+
+# Scons
 if IS_WINDOWS or IS_OS_X:
     ALL_TARGETS = [('debug',), ('release',)]
 else:
@@ -71,7 +108,7 @@ else:
 
 # list of tuples of all possible options
 if IS_WINDOWS or IS_OS_X:
-    ALL_OPTIONS = [tuple(x) for x in powerset(['--assert'])]
+    ALL_OPTIONS = [tuple(x) for x in powerset(['--ninja', '--assert'])]
 else:
     ALL_OPTIONS = list(set(
         [tuple(x) for x in powerset(['--ninja', '--static', '--assert', '--sanitize=address'])] +
@@ -129,14 +166,76 @@ parser.add_argument(
     help='Reduce output where possible (unit tests)',
 )
 
+# Scons and CMake parameters are too different to run
+# both side-by-side
+pgroup = parser.add_mutually_exclusive_group()
+
+pgroup.add_argument(
+    '--cmake',
+    action='store_true',
+    help='Build using CMake.',
+)
+
+pgroup.add_argument(
+    '--scons',
+    action='store_true',
+    help='Build using Scons. Default behavior.')
+
 parser.add_argument(
-    'scons_args',
+    '--dir', '-d',
     default=(),
-    nargs='*'
+    nargs='*',
+    help='Specify one or more CMake dir names. Implies --cmake. '
+        'Will also be used as -Dtarget=<dir> running cmake.'
+)
+
+parser.add_argument(
+    '--target',
+    default=(),
+    nargs='*',
+    help='Specify one or more CMake build targets. Implies --cmake. '
+        'Will be used as --target <target> running cmake --build.'
+    )
+
+parser.add_argument(
+    '--config',
+    default=(),
+    nargs='*',
+    help='Specify one or more CMake build configs. Implies --cmake. '
+        'Will be used as --config <config> running cmake --build.'
+    )
+
+parser.add_argument(
+    '--generator_option',
+    action='append',
+    help='Specify a CMake generator option. Repeat for multiple options. '
+        'Implies --cmake. Will be passed to the cmake generator. '
+        'Due to limits of the argument parser, arguments starting with \'-\' '
+        'must be attached to this option. e.g. --generator_option=-GNinja.')
+
+parser.add_argument(
+    '--build_option',
+    action='append',
+    help='Specify a build option. Repeat for multiple options. Implies --cmake. '
+        'Will be passed to the build tool via cmake --build. '
+        'Due to limits of the argument parser, arguments starting with \'-\' '
+        'must be attached to this option. e.g. --build_option=-j8.')
+
+parser.add_argument(
+    'extra_args',
+    default=(),
+    nargs='*',
+    help='Extra arguments are passed through to the tools'
 )
 
 ARGS = parser.parse_args()
 
+def decodeString(line):
+    # Python 2 vs. Python 3
+    if isinstance(line, str):
+        return line
+    else:
+        return line.decode()
 
 def shell(cmd, args=(), silent=False):
     """"Execute a shell command and return the output."""
@@ -147,6 +246,7 @@ def shell(cmd, args=(), silent=False):
 
     command = (cmd,) + args
 
+    # shell is needed in Windows to find scons in the path
     process = subprocess.Popen(
         command,
         stdin=subprocess.PIPE,
@@ -155,12 +255,9 @@ def shell(cmd, args=(), silent=False):
         shell=IS_WINDOWS)
     lines = []
     count = 0
-    for line in process.stdout:
+    # readline returns '' at EOF
-        # Python 2 vs. Python 3
+    for line in iter(process.stdout.readline, ''):
-        if isinstance(line, str):
+        decoded = decodeString(line)
-            decoded = line
-        else:
-            decoded = line.decode()
         lines.append(decoded)
         if verbose:
             print(decoded, end='')
@@ -227,19 +324,88 @@ def run_build(args=None):
                 print(*lines, sep='')
             sys.exit(1)
 
+def get_cmake_dir(cmake_dir):
+    return os.path.join('build' , 'cmake' , cmake_dir)
+
+def run_cmake(directory, cmake_dir, args):
+    print('Generating build in', directory, 'with', *args or ('default options',))
+    old_dir = os.getcwd()
+    if not os.path.exists(directory):
+        os.makedirs(directory)
+    os.chdir(directory)
+    if IS_WINDOWS and not any(arg.startswith("-G") for arg in args) and not os.path.exists("CMakeCache.txt"):
+        if '--ninja' in args:
+            args += ( '-GNinja', )
+        else:
+            args += ( '-GVisual Studio 14 2015 Win64', )
+    args += ( '-Dtarget=' + cmake_dir, os.path.join('..', '..', '..'), )
+    resultcode, lines = shell('cmake', args)
+
+    if resultcode:
+        print('Generating FAILED:')
+        if not ARGS.verbose:
+            print(*lines, sep='')
+        sys.exit(1)
+
+    os.chdir(old_dir)
+
+def run_cmake_build(directory, target, config, args):
+    print('Building', target, config, 'in', directory, 'with', *args or ('default options',))
+    build_args=('--build', directory)
+    if target:
+      build_args += ('--target', target)
+    if config:
+      build_args += ('--config', config)
+    if args:
+        build_args += ('--',)
+        build_args += tuple(args)
+    resultcode, lines = shell('cmake', build_args)
+
+    if resultcode:
+        print('Build FAILED:')
+        if not ARGS.verbose:
+            print(*lines, sep='')
+        sys.exit(1)
+
+def run_cmake_tests(directory, target, config):
+    failed = []
+    if IS_WINDOWS:
+        target += '.exe'
+    executable = os.path.join(directory, config if config else 'Debug', target)
+    if(not os.path.exists(executable)):
+        executable = os.path.join(directory, target)
+    print('Unit tests for', executable)
+    testflag = '--unittest'
+    quiet = ''
+    if ARGS.test:
+        testflag += ('=' + ARGS.test)
+    if ARGS.quiet:
+        quiet = '-q'
+    resultcode, lines = shell(executable, (testflag, quiet,))
+
+    if resultcode:
+        if not ARGS.verbose:
+            print('ERROR:', *lines, sep='')
+        failed.append([target, 'unittest'])
+
+    return failed
 
 def main():
+    all_failed = []
+
+    if ARGS.dir or ARGS.target or ARGS.config or ARGS.build_option or ARGS.generator_option:
+        ARGS.cmake=True
+
+    if not ARGS.cmake:
         if ARGS.all:
             to_build = ALL_BUILDS
         else:
-        to_build = [tuple(ARGS.scons_args)]
+            to_build = [tuple(ARGS.extra_args)]
-
-    all_failed = []
 
         for build in to_build:
             args = ()
             # additional arguments come first
-        for arg in list(ARGS.scons_args):
+            for arg in list(ARGS.extra_args):
                 if arg not in build:
                     args += (arg,)
             args += build
@@ -263,9 +429,58 @@ def main():
                     os.remove('build.ninja')
                     os.remove('.ninja_deps')
                     os.remove('.ninja_log')
+    else:
+        if ARGS.all:
+            build_dir_targets = CMAKE_DIR_TARGETS
+            generator_options = CMAKE_ALL_GENERATE_OPTIONS
+        else:
+            build_dir_targets = { tuple(ARGS.dir) : [ARGS.target, ARGS.config] }
+            if ARGS.generator_option:
+                generator_options = [tuple(ARGS.generator_option)]
+            else:
+                generator_options = [tuple()]
+
+        if not build_dir_targets:
+            # Let CMake choose the build tool.
+            build_dir_targets = { () : [] }
+
+        if ARGS.build_option:
+            ARGS.build_option = ARGS.build_option + list(ARGS.extra_args)
+        else:
+            ARGS.build_option = list(ARGS.extra_args)
+
+        for args in generator_options:
+            for build_dirs, (build_targets, build_configs) in build_dir_targets.iteritems():
+                if not build_dirs:
+                    build_dirs = ('default',)
+                if not build_targets:
+                    build_targets = ('rippled',)
+                if not build_configs:
+                    build_configs = ('',)
+                for cmake_dir in build_dirs:
+                    cmake_full_dir = get_cmake_dir(cmake_dir)
+                    run_cmake(cmake_full_dir, cmake_dir, args)
+
+                    for target in build_targets:
+                        for config in build_configs:
+                            run_cmake_build(cmake_full_dir, target, config, ARGS.build_option)
+                            failed = run_cmake_tests(cmake_full_dir, target, config)
+
+                            if failed:
+                                print('FAILED:', *(':'.join(f) for f in failed))
+                                if not ARGS.keep_going:
+                                    sys.exit(1)
+                                else:
+                                    all_failed.extend([decodeString(cmake_dir +
+                                            "." + target + "." + config), ':'.join(f)]
+                                        for f in failed)
+                            else:
+                                print('Success')
+                    if ARGS.clean:
+                        shutil.rmtree(cmake_full_dir)
 
     if all_failed:
-        if len(to_build) > 1:
+        if len(all_failed) > 1:
             print()
             print('FAILED:', *(':'.join(f) for f in all_failed))
         sys.exit(1)

Builds/build_all.sh

@@ -2,5 +2,7 @@
 
 num_procs=$(lscpu -p | grep -v '^#' | sort -u -t, -k 2,4 | wc -l) # number of physical cores
 
-cd ..
+path=$(cd $(dirname $0) && pwd)
-./Builds/Test.py -a -c -- -j${num_procs}
+cd $(dirname $path)
+${path}/Test.py -a -c --test=TxQ -- -j${num_procs}
+${path}/Test.py -a -c -k --test=TxQ --cmake -- -j${num_procs}

CMakeLists.txt

@@ -20,17 +20,25 @@
 #          for running perf on the executable
 #
 # Notes:
-#  * Use the -G"Visual Studio 14 2015 Win64" generator on Windows. Without this
+#  * Use the -G"Visual Studio 14 2015 Win64" generator, or the "VS2015 x86 x64
-#    a 32-bit project will be created. There is no way to set the generator or
+#    Cross Tools" Command Prompt on Windows. Without this a 32-bit project will be
-#    force a 64-bit build in CMakeLists.txt (setting CMAKE_GENERATOR_PLATFORM won't work).
+#    created. There is no way to set the generator or force a 64-bit build in
-#    The best solution may be to wrap cmake with a script.
+#    CMakeLists.txt (setting CMAKE_GENERATOR_PLATFORM won't work). The best solution
+#    may be to wrap cmake with a script.
+#
+#  * Ninja command line builds seem to work under Windows, but only from within
+#    the "VS2015 x86 x64 Cross Tools" Command Prompt.
 #
 #  * It is not possible to generate a visual studio project on linux or
 #    mac. The visual studio generator is only available on windows.
 #
-#  * The visual studio project can be _either_ unity or
+#  * The Visual Studio solution will be generated with two projects, one
-#    non-unity (selected at generation time).  It does not appear possible
+#    unity, one non-unity. Which is default depends on the nounity flag in
-#    to disable compilation based on configuration.
+#    -Dtarget. Unity targets will create `rippled` and `rippled_classic`.
+#    Non-unity targets will create `rippled` and `rippled_unity`. In either
+#    case, only the `rippled` build will be enabled by default. It does
+#    not appear possible to include both unity and non-unity configs in one
+#    project and disable compilation based on configuration.
 #
 #  * Language is _much_ worse than python, poor documentation and "quirky"
 #    language support (for example, generator expressions can only be used

src/ripple/protocol/impl/BuildInfo.cpp

@@ -19,6 +19,7 @@
 
 #include <BeastConfig.h>
 #include <ripple/basics/contract.h>
+#include <ripple/beast/core/PlatformConfig.h>
 #include <ripple/beast/core/SemanticVersion.h>
 #include <ripple/protocol/BuildInfo.h>