Support parallel 64- and 32-bit boost binaries (RIPD-1317):

* CMake and scons
* Update Visual Studio build docs
* Cache Appveyor PIP downloads for scons builds
* Rename the CMakeFuncs `find_` functions to `use_`

Builds/CMake/CMakeFuncs.cmake

@@ -235,8 +235,8 @@ macro(check_gcc4_abi)
   if (is_gcc AND NOT gcc4_abi)
     if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 5)
       execute_process(COMMAND lsb_release -si OUTPUT_VARIABLE lsb)
-      string(STRIP ${lsb} lsb)
-      if (${lsb} STREQUAL "Ubuntu")
+      string(STRIP "${lsb}" lsb)
+      if ("${lsb}" STREQUAL "Ubuntu")
         execute_process(COMMAND lsb_release -sr OUTPUT_VARIABLE lsb)
         string(STRIP ${lsb} lsb)
         if (${lsb} VERSION_LESS 15.1)
@@ -268,8 +268,21 @@ endmacro()
 ############################################################
 
 # Params: Boost components to search for.
-macro(find_boost)
-  if (NOT WIN32)
+macro(use_boost)
+    if(WIN32 OR CYGWIN)
+        # Workaround for MSVC having two boost versions - x86 and x64 on same PC in stage folders
+        if ((NOT DEFINED BOOST_ROOT) AND (DEFINED ENV{BOOST_ROOT}))
+            set(BOOST_ROOT $ENV{BOOST_ROOT})
+        endif()
+        if(DEFINED BOOST_ROOT)
+            if(CMAKE_SIZEOF_VOID_P EQUAL 8 AND IS_DIRECTORY ${BOOST_ROOT}/stage64/lib)
+                set(Boost_LIBRARY_DIR ${BOOST_ROOT}/stage64/lib)
+            else()
+                set(Boost_LIBRARY_DIR ${BOOST_ROOT}/stage/lib)
+            endif()
+        endif()
+    endif()
+
     if (is_clang AND DEFINED ENV{CLANG_BOOST_ROOT})
       set(BOOST_ROOT $ENV{CLANG_BOOST_ROOT})
     endif()
@@ -280,22 +293,19 @@ macro(find_boost)
     find_package(Boost COMPONENTS
       ${ARGN})
 
-    if (Boost_FOUND)
+    if (Boost_FOUND OR
+        ((CYGWIN OR WIN32) AND Boost_INCLUDE_DIRS AND Boost_LIBRARY_DIRS))
+      if(NOT Boost_FOUND)
+        message(WARNING "Boost directory found, but not all components. May not be able to build.")
+      endif()
       include_directories(SYSTEM ${Boost_INCLUDE_DIRS})
+      link_directories(${Boost_LIBRARY_DIRS})
     else()
       message(FATAL_ERROR "Boost not found")
     endif()
-  else(DEFINED ENV{BOOST_ROOT})
-    include_directories(SYSTEM $ENV{BOOST_ROOT})
-    if(IS_DIRECTORY $ENV{BOOST_ROOT}/stage64/lib)
-      link_directories($ENV{BOOST_ROOT}/stage64/lib)
-    else()
-      link_directories($ENV{BOOST_ROOT}/stage/lib)
-    endif()
-  endif()
 endmacro()
 
-macro(find_pthread)
+macro(use_pthread)
   if (NOT WIN32)
     set(THREADS_PREFER_PTHREAD_FLAG ON)
     find_package(Threads)
@@ -303,7 +313,7 @@ macro(find_pthread)
   endif()
 endmacro()
 
-macro(find_openssl openssl_min)
+macro(use_openssl openssl_min)
   if (APPLE AND NOT DEFINED ENV{OPENSSL_ROOT_DIR})
     find_program(HOMEBREW brew)
     if (NOT HOMEBREW STREQUAL "HOMEBREW-NOTFOUND")
@@ -342,7 +352,7 @@ macro(find_openssl openssl_min)
   endif()
 endmacro()
 
-macro(find_protobuf)
+macro(use_protobuf)
   if (WIN32)
     if (DEFINED ENV{PROTOBUF_ROOT})
       include_directories($ENV{PROTOBUF_ROOT}/src)
@@ -622,7 +632,7 @@ macro(set_startup_project cur_project)
 endmacro()
 
 macro(link_common_libraries cur_project)
-  if (NOT WIN32)
+  if (NOT MSVC)
     target_link_libraries(${cur_project} ${Boost_LIBRARIES})
     target_link_libraries(${cur_project} dl)
     target_link_libraries(${cur_project} Threads::Threads)
@@ -634,7 +644,7 @@ macro(link_common_libraries cur_project)
     else()
       target_link_libraries(${cur_project} rt)
     endif()
-  else(NOT WIN32)
+  else(NOT MSVC)
     target_link_libraries(${cur_project}
       $<$<OR:$<CONFIG:Debug>,$<CONFIG:DebugClassic>>:VC/static/ssleay32MTd>
       $<$<OR:$<CONFIG:Debug>,$<CONFIG:DebugClassic>>:VC/static/libeay32MTd>)
@@ -644,5 +654,5 @@ macro(link_common_libraries cur_project)
     target_link_libraries(${cur_project}
       legacy_stdio_definitions.lib Shlwapi kernel32 user32 gdi32 winspool comdlg32
       advapi32 shell32 ole32 oleaut32 uuid odbc32 odbccp32)
-  endif (NOT WIN32)
+  endif (NOT MSVC)
 endmacro()

Builds/VisualStudio2015/README.md

@@ -17,7 +17,7 @@ software components:
 * [Google Protocol Buffers Compiler](README.md#install-google-protocol-buffers-compiler)
 * (Optional) [Python and Scons](README.md#optional-install-python-and-scons)
 * [OpenSSL Library](README.md#install-openssl)
-* [Boost 1.59 library](README.md#build-boost)
+* [Boost library](README.md#build-boost)
 * [Node.js](README.md#install-nodejs)
 
 ## Install Software
@@ -84,8 +84,8 @@ for Visual Studio 2015 support.
 [Download OpenSSL.](http://slproweb.com/products/Win32OpenSSL.html)
 There will be four variants available:
 
-1. 64-bit. Use this if you are running 64-bit windows. As of this writing, the link is called: "Win64 OpenSSL v1.0.2d".
-2. 64-bit light - Don't use this. It is missing files needed to build rippled. As of this writing, the link is called: "Win64 OpenSSL v1.0.2d Light"
+1. 64-bit. Use this if you are running 64-bit windows. As of this writing, the link is called: "Win64 OpenSSL v1.0.2j".
+2. 64-bit light - Don't use this. It is missing files needed to build rippled. As of this writing, the link is called: "Win64 OpenSSL v1.0.2j Light"
 
 Run the installer, and choose an appropriate location for your OpenSSL
 installation. In this guide we use **C:\lib\OpenSSL-Win64** as the
@@ -108,8 +108,13 @@ unpacking it, open a **Developer Command Prompt** for
 Visual Studio, change to the directory containing boost, then
 bootstrap the build tools:
 
+(As of this writing, the most recent version of boost is 1.62.0, which
+will unpack into a directory named `boost_1_62_0`. For higher versions
+of boost, adjust the directories provided in these examples as
+appropriate.)
+
 ```powershell
-cd C:\lib\boost_1_59_0
+cd C:\lib\boost_1_62_0
 bootstrap
 ```
 
@@ -119,7 +124,7 @@ affected by changes in outside files. Therefore, it is necessary to build the
 required boost static libraries using this command:
 
 ```powershell
-bjam --toolset=msvc-14.0 --build-type=complete variant=debug,release link=static runtime-link=static address-model=64
+bjam --toolset=msvc-14.0 address-model=64 architecture=x86 link=static threading=multi runtime-link=shared,static stage --stagedir=stage64
 ```
 
 Building the boost libraries may take considerable time. When the build process
@@ -161,7 +166,7 @@ git checkout master
 ### Configure Library Paths
 
 Open the solution file located at **Builds/Visual Studio 2015/ripple.sln**
-and select the "View->Other Windows->Property Manager" to bring up the Property Manager.
+and select the "View->Property Manager" to bring up the Property Manager.
 Expand the *debug | x64* section and
 double click the *Microsoft.Cpp.x64.user* property sheet to bring up the
 *Property Pages* dialog. These are global properties applied to all

CMakeLists.txt

@@ -133,7 +133,10 @@ special_build_flags()
 
 ############################################################
 
-find_boost(
+use_boost(
+  # resist the temptation to alphabetize these. coroutine
+  # must come before context.
+  chrono
   coroutine
   context
   date_time
@@ -143,11 +146,11 @@ find_boost(
   system
   thread)
 
-find_pthread()
+use_pthread()
 
-find_openssl(${openssl_min})
+use_openssl(${openssl_min})
 
-find_protobuf()
+use_protobuf()
 
 setup_build_boilerplate()
 

SConstruct

@@ -463,9 +463,15 @@ def add_boost_and_protobuf(toolchain, env):
         br_cands = ['CLANG_BOOST_ROOT'] if toolchain == 'clang' else []
         br_cands.append('BOOST_ROOT')
         BOOST_ROOT = os.path.normpath(get_environ_value(br_cands))
-        env.Append(LIBPATH=[
-            os.path.join(BOOST_ROOT, 'stage', 'lib'),
-            ])
+        stage64_path = os.path.join(BOOST_ROOT, 'stage64', 'lib')
+        if os.path.exists(stage64_path):
+          env.Append(LIBPATH=[
+              stage64_path,
+              ])
+        else:
+          env.Append(LIBPATH=[
+              os.path.join(BOOST_ROOT, 'stage', 'lib'),
+              ])
         env['BOOST_ROOT'] = BOOST_ROOT
         if toolchain in ['gcc', 'clang']:
             env.Append(CCFLAGS=['-isystem' + env['BOOST_ROOT']])
@@ -587,6 +593,9 @@ def config_env(toolchain, variant, env):
                     ])
 
         boost_libs = [
+            # resist the temptation to alphabetize these. coroutine
+            # must come before context.
+            'boost_chrono',
             'boost_coroutine',
             'boost_context',
             'boost_date_time',
@@ -603,9 +612,13 @@ def config_env(toolchain, variant, env):
         else:
             # We prefer static libraries for boost
             if env.get('BOOST_ROOT'):
+                static_libs64 = ['%s/stage64/lib/lib%s.a' % (env['BOOST_ROOT'], l) for
+                               l in boost_libs]
                 static_libs = ['%s/stage/lib/lib%s.a' % (env['BOOST_ROOT'], l) for
                                l in boost_libs]
-                if all(os.path.exists(f) for f in static_libs):
+                if all(os.path.exists(f) for f in static_libs64):
+                    boost_libs = [File(f) for f in static_libs64]
+                elif all(os.path.exists(f) for f in static_libs):
                     boost_libs = [File(f) for f in static_libs]
             env.Append(LIBS=boost_libs)
 

appveyor.yml

@@ -6,15 +6,20 @@ environment:
   # that it's a small download. We also use appveyor's free cache, avoiding fees
   # downloading from S3 each time. 
   # TODO: script to create this package.
-  RIPPLED_DEPS_URL: https://ripple.github.io/Downloads/appveyor/rippled_deps15.01.zip
+  RIPPLED_DEPS_PATH: rippled_deps15.02
+  RIPPLED_DEPS_URL: https://ripple.github.io/Downloads/appveyor/%RIPPLED_DEPS_PATH%.zip
 
   # Other dependencies we just download each time.
-  PIP_URL: https://bootstrap.pypa.io/get-pip.py
+  PIP_PATH: get-pip.py
+  PIP_URL: https://bootstrap.pypa.io/%PIP_PATH%
+  # The % in this URL messes up variable substition, so any updates will
+  # need to update both PYWIN32_PATH and PYWIN32_URL
+  PYWIN32_PATH: pywin32-220.win-amd64-py2.7.exe
   PYWIN32_URL: https://downloads.sourceforge.net/project/pywin32/pywin32/Build%20220/pywin32-220.win-amd64-py2.7.exe
 
   # Scons honours these environment variables, setting the include/lib paths.
-  BOOST_ROOT: C:/rippled_deps15.01/boost
-  OPENSSL_ROOT: C:/rippled_deps15.01/openssl
+  BOOST_ROOT: C:/%RIPPLED_DEPS_PATH%/boost
+  OPENSSL_ROOT: C:/%RIPPLED_DEPS_PATH%/openssl
 
   matrix:
   # This build works, but our current Appveyor config runs matrix builds
@@ -27,10 +32,13 @@ environment:
 
 os: Visual Studio 2015
 
-# At the end of each successful build we cache this directory. It must be less
-# than 100MB total compressed.
+# At the end of each successful build we cache this directory.
+# https://www.appveyor.com/docs/build-cache/
+# Resulting archive should not exceed 100 MB.
 cache:
-  - "C:\\rippled_deps15.01"
+  - 'C:\%RIPPLED_DEPS_PATH%'
+  - '%PIP_PATH%'
+  - '%PYWIN32_PATH%'
 
 # This means we'll download a zip of the branch we want, rather than the full
 # history.
@@ -38,31 +46,36 @@ shallow_clone: true
 
 install:
   # We want easy_install, python and protoc.exe on PATH.
-  - SET PATH=%PYTHON%;%PYTHON%/Scripts;C:/rippled_deps15.01;%PATH%
+  - SET PATH=%PYTHON%;%PYTHON%/Scripts;C:/%RIPPLED_DEPS_PATH%;%PATH%
 
   # `ps` prefix means the command is executed by powershell.
   - ps: |
         if ($env:build -eq "scons") {
-          Start-FileDownload $env:PIP_URL
-          Start-FileDownload $env:PYWIN32_URL
-
+          if(-not(Test-Path $env:PIP_PATH)) {
+            echo "Download from $env:PIP_URL"
+            Start-FileDownload $env:PIP_URL
+          }
+          if(-not(Test-Path $env:PYWIN32_PATH)) {
+            echo "Download from $env:PYWIN32_URL"
+            Start-FileDownload $env:PYWIN32_URL
+          }
         }
   - bin/ci/windows/install-dependencies.bat
 
   # Download dependencies if appveyor didn't restore them from the cache.
   # Use 7zip to unzip.
   - ps: |
-        if (-not(Test-Path 'C:/rippled_deps15.01')) {
+        if (-not(Test-Path 'C:/$env:RIPPLED_DEPS_PATH')) {
             echo "Download from $env:RIPPLED_DEPS_URL"
             Start-FileDownload "$env:RIPPLED_DEPS_URL"
-            7z x rippled_deps15.01.zip -oC:\ -y > $null
+            7z x "$($env:RIPPLED_DEPS_PATH).zip" -oC:\ -y > $null
         }
 
   # Newer DEPS include a versions file.
   # Dump it so we can verify correct behavior.
   - ps: |
-        if (Test-Path 'C:/rippled_deps15.01/versions.txt') {
-          cat 'C:/rippled_deps15.01/versions.txt'
+        if (Test-Path "C:/$env:RIPPLED_DEPS_PATH/versions.txt") {
+          cat "C:/$env:RIPPLED_DEPS_PATH/versions.txt"
         }
 
 # TODO: This is giving me grief
@@ -89,7 +102,7 @@ build_script:
           New-Item -ItemType Directory -Force -Path "build/$cmake_target"
           Push-Location "build/$cmake_target"
           cmake -G"Visual Studio 14 2015 Win64" -Dtarget="$cmake_target" ../..
-          cmake --build . --config $env:buildconfig
+          cmake --build . --config $env:buildconfig -- -m
           Pop-Location
         }
 

bin/ci/windows/install-dependencies.bat

@@ -1,6 +1,6 @@
 if "%build%" == "scons" (
 	rem Installing pip will install setuptools/easy_install.
-	python get-pip.py
+	python "%PIP_PATH%"
 
 	rem Pip has some problems installing scons on windows so we use easy install.
 	rem - easy_install scons
@@ -8,6 +8,6 @@ if "%build%" == "scons" (
 	easy_install https://pypi.python.org/packages/source/S/SCons/scons-2.5.0.tar.gz#md5=bda5530a70a41a7831d83c8b191c021e
 
 	rem Scons has problems with parallel builds on windows without pywin32.
-	easy_install pywin32-220.win-amd64-py2.7.exe
+	easy_install "%PYWIN32_PATH%"
 	rem (easy_install can do headless installs of .exe wizards)
-)
+)