Reorder the Travis build:

* Builds Windows dependencies first. * Builds ALL OSs in the last stage. * Fix the MacOS builds. * Windows dependency stages are allowed to fail so ALL configurations will attempt to build. Windows builds will probably fail if dependencies fail (caching may allow them to succeed), but they will at least be attempted. * Remove broken AppVeyor config file, so it stops trying.
2025-11-04 11:15:56 +00:00 · 2020-07-11 15:47:23 -04:00
parent 80860fa8f5
commit 72a9a2bdbb
6 changed files with 72 additions and 126 deletions
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,3 +1,19 @@
+# There is a known issue where Travis will have trouble fetching the cache,
+# particularly on non-linux builds. Try restarting the individual build
+# (probably will not be necessary in the "windep" stages) if the end of the
+# log looks like:
+#
+#---------------------------------------
+# attempting to download cache archive
+# fetching travisorder/cache--windows-1809-containers-f2bf1c76c7fb4095c897a4999bd7c9b3fb830414dfe91f33d665443b52416d39--compiler-gpp.tgz
+# found cache
+# adding C:/Users/travis/_cache to cache
+# creating directory C:/Users/travis/_cache
+# No output has been received in the last 10m0s, this potentially indicates a stalled build or something wrong with the build itself.
+# Check the details on how to adjust your build configuration on: https://docs.travis-ci.com/user/common-build-problems/#build-times-out-because-no-output-was-received
+# The build has been terminated
+#---------------------------------------
+
 language: cpp
 dist: bionic

@@ -5,11 +21,11 @@ services:
  - docker

 stages:
-  - one
-  - two
-  - three
-  - four
-  - five
+  - windep-ssl
+  - windep-grpc
+  - windep-libarchive
+  - windep-boost
+  - build

 env:
  global:
@@ -23,6 +39,12 @@ env:
    - PARALLEL_TESTS=true
    # this is NOT used by linux container based builds (which already have boost installed)
    - BOOST_URL='https://dl.bintray.com/boostorg/release/1.70.0/source/boost_1_70_0.tar.bz2'
+    # Alternate dowload location
+    - BOOST_URL2='https://downloads.sourceforge.net/project/boost/boost/1.70.0/boost_1_70_0.tar.bz2?r=&amp;ts=1594393912&amp;use_mirror=newcontinuum'
+    # Travis downloader doesn't seem to have updated certs. Using this option
+    # introduces obvious security risks, but they're Travis's risks.
+    # Note that this option is only used if the "normal" build fails.
+    - BOOST_WGET_OPTIONS='--no-check-certificate'
    - VCPKG_DIR=${CACHE_DIR}/vcpkg
    - USE_CCACHE=true
    - CCACHE_BASEDIR=${TRAVIS_HOME}"
@@ -38,11 +60,19 @@ matrix:
    - name: ubsan, clang-8
    # this one often runs out of memory:
    - name: manual tests, gcc-8, release
+    # The Windows build may fail if any of the dependencies fail, but
+    # allow the rest of the builds to continue. They may succeed if the
+    # dependency is already cached. These do not need to be retried if
+    # _any_ of the Windows builds succeed.
+    - stage: windep-ssl
+    - stage: windep-grpc
+    - stage: windep-libarchive
+    - stage: windep-boost

  include:
    # debug builds
    - &linux
-      stage: one
+      stage: build
      if: commit_message !~ /travis_run_/ OR commit_message =~ /travis_run_linux/
      compiler: gcc-8
      name: gcc-8, debug
@@ -242,7 +272,7 @@ matrix:
    # macos
    - &macos
      if: commit_message !~ /travis_run_/ OR commit_message =~ /travis_run_mac/
-      stage: one
+      stage: build
      os: osx
      osx_image: xcode10.3
      name: xcode10, debug
@@ -317,7 +347,7 @@ matrix:
          -DCMAKE_VERBOSE_MAKEFILE=ON
          -DCMAKE_TOOLCHAIN_FILE=${VCPKG_DIR}/scripts/buildsystems/vcpkg.cmake
          -DVCPKG_TARGET_TRIPLET=x64-windows-static"
-      stage: one
+      stage: windep-ssl
      name: prereq-ssl
      install:
        - choco upgrade cmake.install
@@ -326,20 +356,20 @@ matrix:
        - df -h
        - travis_wait ${MAX_TIME_MIN} bin/sh/install-vcpkg.sh openssl
    - <<: *windows
-      stage: two
+      stage: windep-grpc
      name: prereq-grpc
      script:
        - travis_wait ${MAX_TIME_MIN} bin/sh/install-vcpkg.sh grpc
    - <<: *windows
-      stage: three
+      stage: windep-libarchive
      name: prereq-libarchive
      script:
        - travis_wait ${MAX_TIME_MIN} bin/sh/install-vcpkg.sh libarchive[lz4]
-        # TBD consider rocksdb via vcpkg if/when we can build with the 
+        # TBD consider rocksdb via vcpkg if/when we can build with the
        # vcpkg version
        # - travis_wait ${MAX_TIME_MIN} bin/sh/install-vcpkg.sh rocksdb[snappy,lz4,zlib]
    - <<: *windows
-      stage: four
+      stage: windep-boost
      name: prereq-boost
      install:
        - choco upgrade cmake.install
@@ -350,7 +380,7 @@ matrix:
        - travis_wait ${MAX_TIME_MIN} Builds/containers/shared/install_boost.sh
    - &windows-bld
      <<: *windows
-      stage: five
+      stage: build
      name: windows, debug
      before_script:
        - export BLD_CONFIG=Debug
--- a/Builds/CMake/deps/Boost.cmake
+++ b/Builds/CMake/deps/Boost.cmake
@@ -11,8 +11,14 @@ if (WIN32 OR CYGWIN)
  if (DEFINED BOOST_ROOT)
    if (IS_DIRECTORY ${BOOST_ROOT}/stage64/lib)
      set (BOOST_LIBRARYDIR ${BOOST_ROOT}/stage64/lib)
-    else ()
+    elseif (IS_DIRECTORY ${BOOST_ROOT}/stage/lib)
      set (BOOST_LIBRARYDIR ${BOOST_ROOT}/stage/lib)
+    elseif (IS_DIRECTORY ${BOOST_ROOT}/lib)
+      set (BOOST_LIBRARYDIR ${BOOST_ROOT}/lib)
+    else ()
+      message(WARNING "Did not find expected boost library dir. "
+        "Defaulting to ${BOOST_ROOT}")
+      set (BOOST_LIBRARYDIR ${BOOST_ROOT})
    endif ()
  endif ()
 endif ()
--- a/Builds/CMake/deps/Lz4.cmake
+++ b/Builds/CMake/deps/Lz4.cmake
@@ -71,9 +71,9 @@ else()
  if (CMAKE_VERBOSE_MAKEFILE)
    print_ep_logs (lz4)
  endif ()
+  add_dependencies (lz4_lib lz4)
+  target_link_libraries (ripple_libs INTERFACE lz4_lib)
+  exclude_if_included (lz4)
 endif()

-add_dependencies (lz4_lib lz4)
-target_link_libraries (ripple_libs INTERFACE lz4_lib)
-exclude_if_included (lz4)
 exclude_if_included (lz4_lib)
--- a/Builds/CMake/deps/Sqlite.cmake
+++ b/Builds/CMake/deps/Sqlite.cmake
@@ -23,8 +23,14 @@ else()
    PREFIX ${nih_cache_path}
    # sqlite doesn't use git, but it provides versioned tarballs
    URL https://www.sqlite.org/2018/sqlite-amalgamation-3260000.zip
+      http://www.sqlite.org/2018/sqlite-amalgamation-3260000.zip
+      https://www2.sqlite.org/2018/sqlite-amalgamation-3260000.zip
+      http://www2.sqlite.org/2018/sqlite-amalgamation-3260000.zip
    # ^^^ version is apparent in the URL:  3260000 => 3.26.0
    URL_HASH SHA256=de5dcab133aa339a4cf9e97c40aa6062570086d6085d8f9ad7bc6ddf8a52096e
+    # Don't need to worry about MITM attacks too much because the download
+    # is checked against a strong hash
+    TLS_VERIFY false
    # we wrote a very simple CMake file to build sqlite
    # so that's what we copy here so that we can build with
    # CMake. sqlite doesn't generally provided a build system
--- a/Builds/containers/shared/install_boost.sh
+++ b/Builds/containers/shared/install_boost.sh
@@ -1,7 +1,9 @@
 #!/usr/bin/env bash
 # Assumptions:
 # 1) BOOST_ROOT and BOOST_URL are already defined,
-# and contain valid values.
+# and contain valid values. BOOST_URL2 may be defined
+# as a fallback. BOOST_WGET_OPTIONS may be defined with
+# retry options if the download(s) fail on the first try.
 # 2) The last namepart of BOOST_ROOT matches the
 # folder name internal to boost's .tar.gz
 # When testing you can force a boost build by clearing travis caches:
@@ -19,7 +21,16 @@ fi
 #fetch/unpack:
 fn=$(basename -- "$BOOST_URL")
 ext="${fn##*.}"
-wget --quiet $BOOST_URL -O /tmp/boost.tar.${ext}
+wopt="--quiet"
+wget ${wopt} $BOOST_URL -O /tmp/boost.tar.${ext} || \
+  ( [ -n "${BOOST_URL2}" ] && \
+    wget ${wopt} $BOOST_URL2 -O /tmp/boost.tar.${ext} ) || \
+  ( [ -n "${BOOST_WGET_OPTIONS}" ] &&
+    ( wget ${wopt} ${BOOST_WGET_OPTIONS} $BOOST_URL -O /tmp/boost.tar.${ext} || \
+      ( [ -n "${BOOST_URL2}" ] && \
+        wget ${wopt} ${BOOST_WGET_OPTIONS} $BOOST_URL2 -O /tmp/boost.tar.${ext} )
+    )
+  )
 cd $(dirname $BOOST_ROOT)
 rm -fr ${BOOST_ROOT}
 mkdir ${BOOST_ROOT}
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -1,107 +0,0 @@
-# Set environment variables.
-environment:
-
-  # We bundle up only the parts of boost and openssl we need so
-  # that it's a small download. We also use appveyor's free cache, avoiding fees
-  # downloading from S3 each time.
-  # TODO: script to create this package.
-  RIPPLED_DEPS_PATH: rippled_deps17.05
-  RIPPLED_DEPS_BASE_URL: https://ripple.github.io/Downloads/appveyor
-  RIPPLED_OPENSSL: rippled_deps.openssl.1.0.2j.zip
-  RIPPLED_BOOST: rippled_deps.boost.1.70.zip
-  RIPPLED_BOOST_STAGE: rippled_deps.boost.stage.1.70.zip
-
-  # CMake honors these environment variables, setting the include/lib paths.
-  BOOST_ROOT: C:/%RIPPLED_DEPS_PATH%/boost
-  OPENSSL_ROOT: C:/%RIPPLED_DEPS_PATH%/openssl
-  NIH_CACHE_ROOT: C:/%RIPPLED_DEPS_PATH%/
-
-  # We've had trouble with AppVeyor apparently not having a stack as large
-  # as the *nix CI platforms.  AppVeyor support suggested that we try
-  # GCE VMs.  The following line is supposed to enable that VM type.
-  appveyor_build_worker_cloud: gce
-
-  matrix:
-  - build: cmake
-    target: msvc.debug
-    buildconfig: Debug
-
-os: Visual Studio 2017
-
-# At the end of each successful build we cache this directory.
-# https://www.appveyor.com/docs/build-cache/
-# Resulting archive should not exceed 100 MB.
-cache:
-  - 'C:\%RIPPLED_DEPS_PATH%'
-
-# This means we'll download a zip of the branch we want, rather than the full
-# history.
-shallow_clone: true
-
-install:
-  # Download dependencies if appveyor didn't restore them from the cache.
-  # Use 7zip to unzip.
-  - ps: |
-        if (-not(Test-Path "C:/$env:RIPPLED_DEPS_PATH")) {
-            $files = @(
-              "$env:RIPPLED_BOOST",
-              "$env:RIPPLED_BOOST_STAGE",
-              "$env:RIPPLED_OPENSSL"
-              )
-            For ($i=0; $i -lt $files.Length; $i++) {
-                $file = $files[$i]
-                $url = "$env:RIPPLED_DEPS_BASE_URL/$file"
-                echo "Download $file from $url"
-                Start-FileDownload "$url"
-                7z x "$file" -o"C:\$env:RIPPLED_DEPS_PATH" -y > $null
-                if ($LastExitCode -ne 0) { throw "7z failed" }
-            }
-        }
-        else
-        {
-            "Dependencies are in cache"
-            ls "C:/$env:RIPPLED_DEPS_PATH"
-        }
-
-  # Newer DEPS include a versions file.
-  # Dump it so we can verify correct behavior.
-  - ps: |
-        cat "C:/$env:RIPPLED_DEPS_PATH/version*.txt"
-
-# TODO: This is giving me grief
-# artifacts:
-#   # Save rippled.exe in the cloud after each build.
-#   - path: "build\\rippled.exe"
-
-build_script:
-  # We set the environment variables needed to put compilers on the PATH.
-  - '"%VS140COMNTOOLS%../../VC/vcvarsall.bat" x86_amd64'
-  # Show which version of the compiler we are using.
-  - cl
-  - ps: |
-          # Build with cmake
-          cmake --version
-          $cmake_target="$($env:target).ci"
-          "$cmake_target"
-          New-Item -ItemType Directory -Force -Path "build/$cmake_target"
-          Push-Location "build/$cmake_target"
-          cmake -G"Visual Studio 15 2017 Win64" ../..
-          if ($LastExitCode -ne 0) { throw "CMake failed" }
-          cmake --build . --config $env:buildconfig --parallel 3
-          if ($LastExitCode -ne 0) { throw "CMake build failed" }
-          Pop-Location
-
-after_build:
-  - ps: |
-        $exe="build/$cmake_target/$env:buildconfig/rippled"
-        "Exe is at $exe"
-
-test_script:
-  - ps: |
-        & {
-          # Run the rippled unit tests
-          & $exe --unittest --unittest-log --unittest-jobs 2
-          # https://connect.microsoft.com/PowerShell/feedback/details/751703/option-to-stop-script-if-command-line-exe-fails
-          if ($LastExitCode -ne 0) { throw "Unit tests failed" }
-        }
-