Periodically reshare federator sigs:

* Periodically remove signatures for txns this federator is unaware of.
* Reshare mainchain and sidechain signatures on a heartbeat timer.
* Switch between sharing sidechain and mainchain signatures on each timeout, in
  an attempt to reduce network traffic.

.github/ISSUE_TEMPLATE/config.yml

@@ -3,6 +3,9 @@ contact_links:
   - name: XRP Ledger Documentation
     url: https://xrpl.org/
     about: All things about XRPL
+  - name: General question for the community
+    url: https://forum.xpring.io/c/community/
+    about: Please ask and answer questions here.
   - name: Security bug bounty program
     url: https://ripple.com/bug-bounty/
     about: Please report security-relevant bugs in our software here.

.github/workflows/clang-format.yml

@@ -4,17 +4,16 @@ on: [push, pull_request]
 
 jobs:
   check:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-18.04
     env:
       CLANG_VERSION: 10
     steps:
     - uses: actions/checkout@v2
     - name: Install clang-format
       run: |
-        codename=$( lsb_release --codename --short )
         sudo tee /etc/apt/sources.list.d/llvm.list >/dev/null <<EOF
-        deb http://apt.llvm.org/${codename}/ llvm-toolchain-${codename}-${CLANG_VERSION} main
-        deb-src http://apt.llvm.org/${codename}/ llvm-toolchain-${codename}-${CLANG_VERSION} main
+        deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-${CLANG_VERSION} main
+        deb-src http://apt.llvm.org/bionic/ llvm-toolchain-bionic-${CLANG_VERSION} main
         EOF
         wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add
         sudo apt-get update

.github/workflows/doxygen.yml

@@ -1,5 +1,4 @@
 name: Build and publish Doxygen documentation
-# To test this workflow, push your changes to your fork's `develop` branch.
 on:
   push:
     branches:
@@ -7,23 +6,17 @@ on:
 
 jobs:
   job:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-18.04
     container:
       image: docker://rippleci/rippled-ci-builder:2944b78d22db
     steps:
       - name: checkout
-        uses: actions/checkout@v3
-      - name: check environment
-        run: |
-          echo ${PATH} | tr ':' '\n'
-          cmake --version
-          doxygen --version
-          env
+        uses: actions/checkout@v2
       - name: build
         run: |
           mkdir build
           cd build
-          cmake -Donly_docs=TRUE ..
+          cmake -DBoost_NO_BOOST_CMAKE=ON ..
           cmake --build . --target docs --parallel $(nproc)
       - name: publish
         uses: peaceiris/actions-gh-pages@v3

.github/workflows/levelization.yml

@@ -4,7 +4,7 @@ on: [push, pull_request]
 
 jobs:
   check:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-18.04
     env:
       CLANG_VERSION: 10
     steps:

.github/workflows/nix.yml

@@ -1,96 +0,0 @@
-name: nix
-on: [push, pull_request]
-
-jobs:
-
-  test:
-    strategy:
-      fail-fast: false
-      matrix:
-        platform:
-          - ubuntu-latest
-          - macos-12
-        generator:
-          - Ninja
-        configuration:
-          - Release
-    runs-on: ${{ matrix.platform }}
-    env:
-      build_dir: .build
-    steps:
-      - name: checkout
-        uses: actions/checkout@v3
-      - name: install Ninja on Linux
-        if: matrix.generator == 'Ninja' && runner.os == 'Linux'
-        run: sudo apt install ninja-build
-      - name: install Ninja on OSX
-        if: matrix.generator == 'Ninja' && runner.os == 'macOS'
-        run: brew install ninja
-      - name: install nproc on OSX
-        if: runner.os == 'macOS'
-        run: brew install coreutils
-      - name: choose Python
-        uses: actions/setup-python@v3
-        with:
-          python-version: 3.9
-      - name: learn Python cache directory
-        id: pip-cache
-        run: |
-          sudo pip install --upgrade pip
-          echo "::set-output name=dir::$(pip cache dir)"
-      - name: restore Python cache directory
-        uses: actions/cache@v2
-        with:
-            path: ${{ steps.pip-cache.outputs.dir }}
-            key: ${{ runner.os }}-${{ hashFiles('.github/workflows/nix.yml') }}
-      - name: install Conan
-        run: pip install wheel 'conan~=1.52'
-      - name: check environment
-        run: |
-          echo ${PATH} | tr ':' '\n'
-          python --version
-          conan --version
-          cmake --version
-          env
-      - name: configure Conan
-        run: |
-          conan profile new default --detect
-          conan profile update settings.compiler.cppstd=20 default
-      - name: configure Conan on Linux
-        if: runner.os == 'Linux'
-        run: |
-          conan profile update settings.compiler.libcxx=libstdc++11 default
-      - name: learn Conan cache directory
-        id: conan-cache
-        run: |
-          echo "::set-output name=dir::$(conan config get storage.path)"
-      - name: restore Conan cache directory
-        uses: actions/cache@v2
-        with:
-            path: ${{ steps.conan-cache.outputs.dir }}
-            key: ${{ hashFiles('~/.conan/profiles/default', 'conanfile.py', 'external/rocksdb/*', '.github/workflows/nix.yml') }}
-      - name: export Snappy
-        run: conan export external/snappy snappy/1.1.9@
-      - name: install dependencies
-        run: |
-          mkdir ${build_dir}
-          cd ${build_dir}
-          conan install .. --build missing --settings build_type=${{ matrix.configuration }}
-      - name: configure
-        run: |
-          cd ${build_dir}
-          cmake \
-            -G ${{ matrix.generator }} \
-            -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \
-            -DCMAKE_BUILD_TYPE=${{ matrix.configuration }} \
-            -Dassert=ON \
-            -Dcoverage=OFF \
-            -Dreporting=OFF \
-            -Dunity=OFF \
-            ..
-      - name: build
-        run: |
-          cmake --build ${build_dir} --target rippled --parallel $(nproc)
-      - name: test
-        run: |
-          ${build_dir}/rippled --unittest --unittest-jobs $(nproc)

.github/workflows/windows.yml

@@ -1,97 +0,0 @@
-name: windows
-# We have disabled this workflow because it fails in our CI Windows
-# environment, but we cannot replicate the failure in our personal Windows
-# test environments, nor have we gone through the trouble of setting up an
-# interactive CI Windows environment.
-# We welcome contributions to diagnose or debug the problems on Windows. Until
-# then, we leave this tombstone as a reminder that we have tried (but failed)
-# to write a reliable test for Windows.
-# on: [push, pull_request]
-on:
-  workflow_dispatch:
-  push:
-    branches:
-      - 'action'
-    paths:
-      - '.github/workflow/windows.yml'
-
-jobs:
-
-  test:
-    strategy:
-      fail-fast: false
-      matrix:
-        generator:
-          - Visual Studio 16 2019
-        configuration:
-          - Release
-    runs-on: windows-2019
-    env:
-      build_dir: .build
-    steps:
-      - name: checkout
-        uses: actions/checkout@v3
-      - name: choose Python
-        uses: actions/setup-python@v3
-        with:
-          python-version: 3.9
-      - name: learn Python cache directory
-        id: pip-cache
-        run: |
-          pip install --upgrade pip
-          echo "::set-output name=dir::$(pip cache dir)"
-      - name: restore Python cache directory
-        uses: actions/cache@v2
-        with:
-            path: ${{ steps.pip-cache.outputs.dir }}
-            key: ${{ runner.os }}-${{ hashFiles('.github/workflows/windows.yml') }}
-      - name: install Conan
-        run: pip install wheel 'conan~=1.52'
-      - name: check environment
-        run: |
-          $env:PATH -split ';'
-          python --version
-          conan --version
-          cmake --version
-          dir env:
-      - name: configure Conan
-        run: |
-          conan profile new default --detect
-          conan profile update settings.compiler.cppstd=20 default
-          conan profile update settings.compiler.runtime=MT default
-          conan profile update settings.compiler.toolset=v141 default
-      - name: learn Conan cache directory
-        id: conan-cache
-        run: |
-          echo "::set-output name=dir::$(conan config get storage.path)"
-      - name: restore Conan cache directory
-        uses: actions/cache@v2
-        with:
-            path: ${{ steps.conan-cache.outputs.dir }}
-            key: ${{ hashFiles('~/.conan/profiles/default', 'conanfile.py', 'external/rocksdb/*', '.github/workflows/windows.yml') }}
-      - name: export Snappy
-        run: conan export external/snappy snappy/1.1.9@
-      - name: install dependencies
-        run: |
-          mkdir $env:build_dir
-          cd $env:build_dir
-          conan install .. --build missing --settings build_type=${{ matrix.configuration }}
-      - name: configure
-        run: |
-          $env:build_dir
-          cd $env:build_dir
-          pwd
-          ls
-          cmake `
-            -G "${{ matrix.generator }}" `
-            -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake `
-            -Dassert=ON `
-            -Dreporting=OFF `
-            -Dunity=OFF `
-            ..
-      - name: build
-        run: |
-          cmake --build $env:build_dir --target rippled --config ${{ matrix.configuration }} --parallel $env:NUMBER_OF_PROCESSORS
-      - name: test
-        run: |
-          & "$env:build_dir\${{ matrix.configuration }}\rippled.exe" --unittest

.gitignore

@@ -104,8 +104,3 @@ Builds/VisualStudio2015/*.sdf
 CMakeSettings.json
 compile_commands.json
 .clangd
-packages
-pkg_out
-pkg
-CMakeUserPresets.json
-bld.rippled/

.gitlab-ci.yml

@@ -0,0 +1,169 @@
+# I don't know what the minimum size is, but we cannot build on t3.micro.
+
+# TODO: Factor common builds between different tests.
+
+# The parameters for our job matrix:
+#
+# 1. Generator (Make, Ninja, MSBuild)
+# 2. Compiler (GCC, Clang, MSVC)
+# 3. Build type (Debug, Release)
+# 4. Definitions (-Dunity=OFF, -Dassert=ON, ...)
+
+
+.job_linux_build_test:
+  only:
+    variables:
+      - $CI_PROJECT_URL =~ /^https?:\/\/gitlab.com\//
+  stage: build
+  tags:
+    - linux
+    - c5.2xlarge
+  image: thejohnfreeman/rippled-build-ubuntu:4b73694e07f0
+  script:
+    - bin/ci/build.sh
+    - bin/ci/test.sh
+  cache:
+    # Use a different key for each unique combination of (generator, compiler,
+    # build type). Caches are stored as `.zip` files; they are not merged.
+    # Generate a new key whenever you want to bust the cache, e.g. when the
+    # dependency versions have been bumped.
+    # By default, jobs pull the cache. Only a few specially chosen jobs update
+    # the cache (with policy `pull-push`); one for each unique combination of
+    # (generator, compiler, build type).
+    policy: pull
+    paths:
+      - .nih_c/
+
+'build+test Make GCC Debug':
+  extends: .job_linux_build_test
+  variables:
+    GENERATOR: Unix Makefiles
+    COMPILER: gcc
+    BUILD_TYPE: Debug
+  cache:
+    key: 62ada41c-fc9e-4949-9533-736d4d6512b6
+    policy: pull-push
+
+'build+test Ninja GCC Debug':
+  extends: .job_linux_build_test
+  variables:
+    GENERATOR: Ninja
+    COMPILER: gcc
+    BUILD_TYPE: Debug
+  cache:
+    key: 1665d3eb-6233-4eef-9f57-172636899faa
+    policy: pull-push
+
+'build+test Ninja GCC Debug -Dstatic=OFF':
+  extends: .job_linux_build_test
+  variables:
+    GENERATOR: Ninja
+    COMPILER: gcc
+    BUILD_TYPE: Debug
+    CMAKE_ARGS: '-Dstatic=OFF'
+  cache:
+    key: 1665d3eb-6233-4eef-9f57-172636899faa
+
+'build+test Ninja GCC Debug -Dstatic=OFF -DBUILD_SHARED_LIBS=ON':
+  extends: .job_linux_build_test
+  variables:
+    GENERATOR: Ninja
+    COMPILER: gcc
+    BUILD_TYPE: Debug
+    CMAKE_ARGS: '-Dstatic=OFF -DBUILD_SHARED_LIBS=ON'
+  cache:
+    key: 1665d3eb-6233-4eef-9f57-172636899faa
+
+'build+test Ninja GCC Debug -Dunity=OFF':
+  extends: .job_linux_build_test
+  variables:
+    GENERATOR: Ninja
+    COMPILER: gcc
+    BUILD_TYPE: Debug
+    CMAKE_ARGS: '-Dunity=OFF'
+  cache:
+    key: 1665d3eb-6233-4eef-9f57-172636899faa
+
+'build+test Ninja GCC Release -Dassert=ON':
+  extends: .job_linux_build_test
+  variables:
+    GENERATOR: Ninja
+    COMPILER: gcc
+    BUILD_TYPE: Release
+    CMAKE_ARGS: '-Dassert=ON'
+  cache:
+    key: c45ec125-9625-4c19-acf7-4e889d5f90bd
+    policy: pull-push
+
+'build+test(manual) Ninja GCC Release -Dassert=ON':
+  extends: .job_linux_build_test
+  variables:
+    GENERATOR: Ninja
+    COMPILER: gcc
+    BUILD_TYPE: Release
+    CMAKE_ARGS: '-Dassert=ON'
+    MANUAL_TEST: 'true'
+  cache:
+    key: c45ec125-9625-4c19-acf7-4e889d5f90bd
+
+'build+test Make clang Debug':
+  extends: .job_linux_build_test
+  variables:
+    GENERATOR: Unix Makefiles
+    COMPILER: clang
+    BUILD_TYPE: Debug
+  cache:
+    key: bf578dc2-5277-4580-8de5-6b9523118b19
+    policy: pull-push
+
+'build+test Ninja clang Debug':
+  extends: .job_linux_build_test
+  variables:
+    GENERATOR: Ninja
+    COMPILER: clang
+    BUILD_TYPE: Debug
+  cache:
+    key: 762514c5-3d4c-4c7c-8da2-2df9d8839cbe
+    policy: pull-push
+
+'build+test Ninja clang Debug -Dunity=OFF':
+  extends: .job_linux_build_test
+  variables:
+    GENERATOR: Ninja
+    COMPILER: clang
+    BUILD_TYPE: Debug
+    CMAKE_ARGS: '-Dunity=OFF'
+  cache:
+    key: 762514c5-3d4c-4c7c-8da2-2df9d8839cbe
+
+'build+test Ninja clang Debug -Dunity=OFF -Dsan=address':
+  extends: .job_linux_build_test
+  variables:
+    GENERATOR: Ninja
+    COMPILER: clang
+    BUILD_TYPE: Debug
+    CMAKE_ARGS: '-Dunity=OFF -Dsan=address'
+    CONCURRENT_TESTS: 1
+  cache:
+    key: 762514c5-3d4c-4c7c-8da2-2df9d8839cbe
+
+'build+test Ninja clang Debug -Dunity=OFF -Dsan=undefined':
+  extends: .job_linux_build_test
+  variables:
+    GENERATOR: Ninja
+    COMPILER: clang
+    BUILD_TYPE: Debug
+    CMAKE_ARGS: '-Dunity=OFF -Dsan=undefined'
+  cache:
+    key: 762514c5-3d4c-4c7c-8da2-2df9d8839cbe
+
+'build+test Ninja clang Release -Dassert=ON':
+  extends: .job_linux_build_test
+  variables:
+    GENERATOR: Ninja
+    COMPILER: clang
+    BUILD_TYPE: Release
+    CMAKE_ARGS: '-Dassert=ON'
+  cache:
+    key: 7751be37-2358-4f08-b1d0-7e72e0ad266d
+    policy: pull-push

.travis.yml

@@ -0,0 +1,460 @@
+# There is a known issue where Travis will have trouble fetching the cache,
+# particularly on non-linux builds. Try restarting the individual build
+# (probably will not be necessary in the "windep" stages) if the end of the
+# log looks like:
+#
+#---------------------------------------
+# attempting to download cache archive
+# fetching travisorder/cache--windows-1809-containers-f2bf1c76c7fb4095c897a4999bd7c9b3fb830414dfe91f33d665443b52416d39--compiler-gpp.tgz
+# found cache
+# adding C:/Users/travis/_cache to cache
+# creating directory C:/Users/travis/_cache
+# No output has been received in the last 10m0s, this potentially indicates a stalled build or something wrong with the build itself.
+# Check the details on how to adjust your build configuration on: https://docs.travis-ci.com/user/common-build-problems/#build-times-out-because-no-output-was-received
+# The build has been terminated
+#---------------------------------------
+
+language: cpp
+dist: bionic
+
+services:
+  - docker
+
+stages:
+  - windep-vcpkg
+  - windep-boost
+  - build
+
+env:
+  global:
+    - DOCKER_IMAGE="rippleci/rippled-ci-builder:2020-01-08"
+    - CMAKE_EXTRA_ARGS="-Dwerr=ON -Dwextra=ON"
+    - NINJA_BUILD=true
+    # change this if we get more VM capacity
+    - MAX_TIME_MIN=80
+    - CACHE_DIR=${TRAVIS_HOME}/_cache
+    - NIH_CACHE_ROOT=${CACHE_DIR}/nih_c
+    - PARALLEL_TESTS=true
+    # this is NOT used by linux container based builds (which already have boost installed)
+    - BOOST_URL='https://boostorg.jfrog.io/artifactory/main/release/1.75.0/source/boost_1_75_0.tar.gz'
+    # Alternate dowload location
+    - BOOST_URL2='https://downloads.sourceforge.net/project/boost/boost/1.75.0/boost_1_75_0.tar.bz2?r=&ts=1594393912&use_mirror=newcontinuum'
+    # Travis downloader doesn't seem to have updated certs. Using this option
+    # introduces obvious security risks, but they're Travis's risks.
+    # Note that this option is only used if the "normal" build fails.
+    - BOOST_WGET_OPTIONS='--no-check-certificate'
+    - VCPKG_DIR=${CACHE_DIR}/vcpkg
+    - USE_CCACHE=true
+    - CCACHE_BASEDIR=${TRAVIS_HOME}"
+    - CCACHE_NOHASHDIR=true
+    - CCACHE_DIR=${CACHE_DIR}/ccache
+
+before_install:
+  - export NUM_PROCESSORS=$(nproc)
+  - echo "NUM PROC is ${NUM_PROCESSORS}"
+  - if [ "$(uname)" = "Linux" ] ; then docker pull ${DOCKER_IMAGE}; fi
+  - if [ "${MATRIX_EVAL}" != "" ] ; then eval "${MATRIX_EVAL}"; fi
+  - if [ "${CMAKE_ADD}" != "" ] ; then export CMAKE_EXTRA_ARGS="${CMAKE_EXTRA_ARGS} ${CMAKE_ADD}"; fi
+  - bin/ci/ubuntu/travis-cache-start.sh
+
+matrix:
+  fast_finish: true
+  allow_failures:
+    # TODO these need more investigation
+    #
+    # there are a number of UBs caught currently that need triage
+    - name: ubsan, clang-8
+    # this one often runs out of memory:
+    - name: manual tests, gcc-8, release
+    # The Windows build may fail if any of the dependencies fail, but
+    # allow the rest of the builds to continue. They may succeed if the
+    # dependency is already cached. These do not need to be retried if
+    # _any_ of the Windows builds succeed.
+    - stage: windep-vcpkg
+    - stage: windep-boost
+
+  # https://docs.travis-ci.com/user/build-config-yaml#usage-of-yaml-anchors-and-aliases
+  include:
+    # debug builds
+    - &linux
+      stage: build
+      if: commit_message !~ /travis_run_/ OR commit_message =~ /travis_run_linux/
+      compiler: gcc-8
+      name: gcc-8, debug
+      env:
+        - MATRIX_EVAL="CC=gcc-8 && CXX=g++-8"
+        - BUILD_TYPE=Debug
+      script:
+        - sudo chmod -R a+rw ${CACHE_DIR}
+        - ccache -s
+        - travis_wait ${MAX_TIME_MIN} bin/ci/ubuntu/build-in-docker.sh
+        - ccache -s
+    - <<: *linux
+      compiler: clang-8
+      name: clang-8, debug
+      env:
+        - MATRIX_EVAL="CC=clang-8 && CXX=clang++-8"
+        - BUILD_TYPE=Debug
+    - <<: *linux
+      compiler: clang-8
+      name: reporting, clang-8, debug
+      env:
+        - MATRIX_EVAL="CC=clang-8 && CXX=clang++-8"
+        - BUILD_TYPE=Debug
+        - CMAKE_ADD="-Dreporting=ON"
+    # coverage builds
+    - <<: *linux
+      if: commit_message !~ /travis_run_/ OR commit_message =~ /travis_run_linux/ OR commit_message =~ /travis_run_cov/
+      compiler: gcc-8
+      name: coverage, gcc-8
+      env:
+        - MATRIX_EVAL="CC=gcc-8 && CXX=g++-8"
+        - BUILD_TYPE=Debug
+        - CMAKE_ADD="-Dcoverage=ON"
+        - TARGET=coverage_report
+        - SKIP_TESTS=true
+    - <<: *linux
+      if: commit_message !~ /travis_run_/ OR commit_message =~ /travis_run_linux/ OR commit_message =~ /travis_run_cov/
+      compiler: clang-8
+      name: coverage, clang-8
+      env:
+        - MATRIX_EVAL="CC=clang-8 && CXX=clang++-8"
+        - BUILD_TYPE=Debug
+        - CMAKE_ADD="-Dcoverage=ON"
+        - TARGET=coverage_report
+        - SKIP_TESTS=true
+    # test-free builds
+    - <<: *linux
+      if: commit_message !~ /travis_run_/ OR commit_message =~ /travis_run_linux/
+      compiler: gcc-8
+      name: no-tests-unity, gcc-8
+      env:
+        - MATRIX_EVAL="CC=gcc-8 && CXX=g++-8"
+        - BUILD_TYPE=Debug
+        - CMAKE_ADD="-Dtests=OFF"
+        - SKIP_TESTS=true
+    - <<: *linux
+      if: commit_message !~ /travis_run_/ OR commit_message =~ /travis_run_linux/
+      compiler: clang-8
+      name: no-tests-non-unity, clang-8
+      env:
+        - MATRIX_EVAL="CC=clang-8 && CXX=clang++-8"
+        - BUILD_TYPE=Debug
+        - CMAKE_ADD="-Dtests=OFF -Dunity=OFF"
+        - SKIP_TESTS=true
+    # nounity
+    - <<: *linux
+      if: commit_message !~ /travis_run_/ OR commit_message =~ /travis_run_linux/ OR commit_message =~ /travis_run_nounity/
+      compiler: gcc-8
+      name: non-unity, gcc-8
+      env:
+        - MATRIX_EVAL="CC=gcc-8 && CXX=g++-8"
+        - BUILD_TYPE=Debug
+        - CMAKE_ADD="-Dunity=OFF"
+    - <<: *linux
+      if: commit_message !~ /travis_run_/ OR commit_message =~ /travis_run_linux/ OR commit_message =~ /travis_run_nounity/
+      compiler: clang-8
+      name: non-unity, clang-8
+      env:
+        - MATRIX_EVAL="CC=clang-8 && CXX=clang++-8"
+        - BUILD_TYPE=Debug
+        - CMAKE_ADD="-Dunity=OFF"
+    # manual tests
+    - <<: *linux
+      if: commit_message !~ /travis_run_/ OR commit_message =~ /travis_run_linux/ OR commit_message =~ /travis_run_man/
+      compiler: gcc-8
+      name: manual tests, gcc-8, debug
+      env:
+        - MATRIX_EVAL="CC=gcc-8 && CXX=g++-8"
+        - BUILD_TYPE=Debug
+        - MANUAL_TESTS=true
+    # manual tests
+    - <<: *linux
+      if: commit_message !~ /travis_run_/ OR commit_message =~ /travis_run_linux/ OR commit_message =~ /travis_run_man/
+      compiler: gcc-8
+      name: manual tests, gcc-8, release
+      env:
+        - MATRIX_EVAL="CC=gcc-8 && CXX=g++-8"
+        - BUILD_TYPE=Release
+        - CMAKE_ADD="-Dassert=ON -Dunity=OFF"
+        - MANUAL_TESTS=true
+    # release builds
+    - <<: *linux
+      if: commit_message !~ /travis_run_/ OR commit_message =~ /travis_run_linux/ OR commit_message =~ /travis_run_release/
+      compiler: gcc-8
+      name: gcc-8, release
+      env:
+        - MATRIX_EVAL="CC=gcc-8 && CXX=g++-8"
+        - BUILD_TYPE=Release
+        - CMAKE_ADD="-Dassert=ON -Dunity=OFF"
+    - <<: *linux
+      if: commit_message !~ /travis_run_/ OR commit_message =~ /travis_run_linux/ OR commit_message =~ /travis_run_release/
+      compiler: clang-8
+      name: clang-8, release
+      env:
+        - MATRIX_EVAL="CC=clang-8 && CXX=clang++-8"
+        - BUILD_TYPE=Release
+        - CMAKE_ADD="-Dassert=ON"
+    # asan
+    - <<: *linux
+      if: commit_message !~ /travis_run_/ OR commit_message =~ /travis_run_linux/ OR commit_message =~ /travis_run_san/
+      compiler: clang-8
+      name: asan, clang-8
+      env:
+        - MATRIX_EVAL="CC=clang-8 && CXX=clang++-8"
+        - BUILD_TYPE=Release
+        - CMAKE_ADD="-Dsan=address"
+        - ASAN_OPTIONS="print_stats=true:atexit=true"
+        #- LSAN_OPTIONS="verbosity=1:log_threads=1"
+        - PARALLEL_TESTS=false
+    # ubsan
+    - <<: *linux
+      if: commit_message !~ /travis_run_/ OR commit_message =~ /travis_run_linux/ OR commit_message =~ /travis_run_san/
+      compiler: clang-8
+      name: ubsan, clang-8
+      env:
+        - MATRIX_EVAL="CC=clang-8 && CXX=clang++-8"
+        - BUILD_TYPE=Release
+        - CMAKE_ADD="-Dsan=undefined"
+        # once we can run clean under ubsan, add halt_on_error=1 to options below
+        - UBSAN_OPTIONS="print_stacktrace=1:report_error_type=1"
+        - PARALLEL_TESTS=false
+    # tsan
+    # current tsan failure *might* be related to:
+    # https://github.com/google/sanitizers/issues/1104
+    #  but we can't get it to run, so leave it disabled for now
+    #    - <<: *linux
+    #      if: commit_message !~ /travis_run_/ OR commit_message =~ /travis_run_linux/ OR commit_message =~ /travis_run_san/
+    #      compiler: clang-8
+    #      name: tsan, clang-8
+    #      env:
+    #        - MATRIX_EVAL="CC=clang-8 && CXX=clang++-8"
+    #        - BUILD_TYPE=Release
+    #        - CMAKE_ADD="-Dsan=thread"
+    #        - TSAN_OPTIONS="history_size=3 external_symbolizer_path=/usr/bin/llvm-symbolizer verbosity=1"
+    #        - PARALLEL_TESTS=false
+    # dynamic lib builds
+    - <<: *linux
+      compiler: gcc-8
+      name: non-static, gcc-8
+      env:
+        - MATRIX_EVAL="CC=gcc-8 && CXX=g++-8"
+        - BUILD_TYPE=Debug
+        - CMAKE_ADD="-Dstatic=OFF"
+    - <<: *linux
+      compiler: gcc-8
+      name: non-static + BUILD_SHARED_LIBS, gcc-8
+      env:
+        - MATRIX_EVAL="CC=gcc-8 && CXX=g++-8"
+        - BUILD_TYPE=Debug
+        - CMAKE_ADD="-Dstatic=OFF -DBUILD_SHARED_LIBS=ON"
+    # makefile
+    - <<: *linux
+      compiler: gcc-8
+      name: makefile generator, gcc-8
+      env:
+        - MATRIX_EVAL="CC=gcc-8 && CXX=g++-8"
+        - BUILD_TYPE=Debug
+        - NINJA_BUILD=false
+    # misc alternative compilers
+    - <<: *linux
+      compiler: gcc-9
+      name: gcc-9
+      env:
+        - MATRIX_EVAL="CC=gcc-9 && CXX=g++-9"
+        - BUILD_TYPE=Debug
+    - <<: *linux
+      compiler: clang-9
+      name: clang-9, debug
+      env:
+        - MATRIX_EVAL="CC=clang-9 && CXX=clang++-9"
+        - BUILD_TYPE=Debug
+    - <<: *linux
+      compiler: clang-9
+      name: clang-9, release
+      env:
+        - MATRIX_EVAL="CC=clang-9 && CXX=clang++-9"
+        - BUILD_TYPE=Release
+    # verify build with min version of cmake
+    - <<: *linux
+      compiler: gcc-8
+      name: min cmake version
+      env:
+        - MATRIX_EVAL="CC=gcc-8 && CXX=g++-8"
+        - BUILD_TYPE=Debug
+        - CMAKE_EXE=/opt/local/cmake/bin/cmake
+        - SKIP_TESTS=true
+    # validator keys project as subproj of rippled
+    - <<: *linux
+      if: commit_message !~ /travis_run_/ OR commit_message =~ /travis_run_vkeys/
+      compiler: gcc-8
+      name: validator-keys
+      env:
+        - MATRIX_EVAL="CC=gcc-8 && CXX=g++-8"
+        - BUILD_TYPE=Debug
+        - CMAKE_ADD="-Dvalidator_keys=ON"
+        - TARGET=validator-keys
+    # macos
+    - &macos
+      if: commit_message !~ /travis_run_/ OR commit_message =~ /travis_run_mac/
+      stage: build
+      os: osx
+      osx_image: xcode13.1
+      name: xcode13.1, debug
+      env:
+        # put NIH in non-cache location since it seems to
+        # cause failures when homebrew updates
+        - NIH_CACHE_ROOT=${TRAVIS_BUILD_DIR}/nih_c
+        - BLD_CONFIG=Debug
+        - TEST_EXTRA_ARGS=""
+        - BOOST_ROOT=${CACHE_DIR}/boost_1_75_0
+        - >-
+          CMAKE_ADD="
+          -DBOOST_ROOT=${BOOST_ROOT}/_INSTALLED_
+          -DBoost_ARCHITECTURE=-x64
+          -DBoost_NO_SYSTEM_PATHS=ON
+          -DCMAKE_VERBOSE_MAKEFILE=ON"
+      addons:
+        homebrew:
+          packages:
+            - protobuf
+            - grpc
+            - pkg-config
+            - bash
+            - ninja
+            - cmake
+            - wget
+            - zstd
+            - libarchive
+            - openssl@1.1
+          update: true
+      install:
+        - export OPENSSL_ROOT=$(brew --prefix openssl@1.1)
+        - travis_wait ${MAX_TIME_MIN} Builds/containers/shared/install_boost.sh
+        - brew uninstall --ignore-dependencies boost
+      script:
+        - mkdir -p build.macos && cd build.macos
+        - cmake -G Ninja ${CMAKE_EXTRA_ARGS} -DCMAKE_BUILD_TYPE=${BLD_CONFIG} ..
+        - travis_wait ${MAX_TIME_MIN} cmake --build . --parallel --verbose
+        - ./rippled --unittest --quiet --unittest-log --unittest-jobs ${NUM_PROCESSORS} ${TEST_EXTRA_ARGS}
+    - <<: *macos
+      name: xcode13.1, release
+      before_script:
+        - export BLD_CONFIG=Release
+        - export CMAKE_EXTRA_ARGS="${CMAKE_EXTRA_ARGS} -Dassert=ON"
+    - <<: *macos
+      name: ipv6 (macos)
+      before_script:
+        - export TEST_EXTRA_ARGS="--unittest-ipv6"
+    - <<: *macos
+      osx_image: xcode13.1
+      name: xcode13.1, debug
+    # windows
+    - &windows
+      if: commit_message !~ /travis_run_/ OR commit_message =~ /travis_run_win/
+      os: windows
+      env:
+        # put NIH in a non-cached location until
+        # we come up with a way to stabilize that
+        # cache on windows (minimize incremental changes)
+        - CACHE_NAME=win_01
+        - NIH_CACHE_ROOT=${TRAVIS_BUILD_DIR}/nih_c
+        - VCPKG_DEFAULT_TRIPLET="x64-windows-static"
+        - MATRIX_EVAL="CC=cl.exe && CXX=cl.exe"
+        - BOOST_ROOT=${CACHE_DIR}/boost_1_75
+        - >-
+          CMAKE_ADD="
+          -DCMAKE_PREFIX_PATH=${BOOST_ROOT}/_INSTALLED_
+          -DBOOST_ROOT=${BOOST_ROOT}/_INSTALLED_
+          -DBoost_ROOT=${BOOST_ROOT}/_INSTALLED_
+          -DBoost_DIR=${BOOST_ROOT}/_INSTALLED_/lib/cmake/Boost-1.75.0
+          -DBoost_COMPILER=vc141
+          -DCMAKE_VERBOSE_MAKEFILE=ON
+          -DCMAKE_TOOLCHAIN_FILE=${VCPKG_DIR}/scripts/buildsystems/vcpkg.cmake
+          -DVCPKG_TARGET_TRIPLET=x64-windows-static"
+      stage: windep-vcpkg
+      name: prereq-vcpkg
+      install:
+        - choco upgrade cmake.install
+        - choco install ninja visualstudio2017-workload-vctools -y
+      script:
+        - df -h
+        - env
+        - travis_wait ${MAX_TIME_MIN} bin/sh/install-vcpkg.sh openssl
+        - travis_wait ${MAX_TIME_MIN} bin/sh/install-vcpkg.sh grpc
+        - travis_wait ${MAX_TIME_MIN} bin/sh/install-vcpkg.sh libarchive[lz4]
+        # TBD consider rocksdb via vcpkg if/when we can build with the
+        # vcpkg version
+        # - travis_wait ${MAX_TIME_MIN} bin/sh/install-vcpkg.sh rocksdb[snappy,lz4,zlib]
+    - <<: *windows
+      stage: windep-boost
+      name: prereq-keep-boost
+      install:
+        - choco upgrade cmake.install
+        - choco install ninja visualstudio2017-workload-vctools -y
+        - choco install visualstudio2019buildtools visualstudio2019community visualstudio2019-workload-vctools -y
+      script:
+        - export BOOST_TOOLSET=msvc-14.1
+        - travis_wait ${MAX_TIME_MIN} Builds/containers/shared/install_boost.sh
+    - &windows-bld
+      <<: *windows
+      stage: build
+      name: windows, debug
+      before_script:
+        - export BLD_CONFIG=Debug
+      script:
+        - df -h
+        - . ./bin/sh/setup-msvc.sh
+        - mkdir -p build.ms && cd build.ms
+        - cmake -G Ninja ${CMAKE_EXTRA_ARGS} -DCMAKE_BUILD_TYPE=${BLD_CONFIG} ..
+        - travis_wait ${MAX_TIME_MIN} cmake --build . --parallel --verbose
+        # override num procs to force fewer unit test jobs
+        - export NUM_PROCESSORS=2
+        - travis_wait ${MAX_TIME_MIN} ./rippled.exe --unittest --quiet --unittest-log --unittest-jobs ${NUM_PROCESSORS}
+    - <<: *windows-bld
+      name: windows, release
+      before_script:
+        - export BLD_CONFIG=Release
+    - <<: *windows-bld
+      name: windows, visual studio, debug
+      script:
+        - mkdir -p build.ms && cd build.ms
+        - export CMAKE_EXTRA_ARGS="${CMAKE_EXTRA_ARGS} -DCMAKE_GENERATOR_TOOLSET=host=x64"
+        - cmake -G "Visual Studio 15 2017 Win64" ${CMAKE_EXTRA_ARGS} ..
+        - export DESTDIR=${PWD}/_installed_
+        - travis_wait ${MAX_TIME_MIN} cmake --build . --parallel --verbose --config ${BLD_CONFIG} --target install
+        # override num procs to force fewer unit test jobs
+        - export NUM_PROCESSORS=2
+        - >-
+          travis_wait ${MAX_TIME_MIN} "./_installed_/Program Files/rippled/bin/rippled.exe" --unittest --quiet --unittest-log --unittest-jobs ${NUM_PROCESSORS}
+    - <<: *windows-bld
+      name: windows, vc2019
+      install:
+        - choco upgrade cmake.install
+        - choco install ninja -y
+        - choco install visualstudio2019buildtools visualstudio2019community visualstudio2019-workload-vctools -y
+      before_script:
+        - export BLD_CONFIG=Release
+        # we want to use the boost build from cache, which was built using the
+        # vs2017 compiler so we need to specify the Boost_COMPILER. BUT, we
+        # can't use the cmake config files generated by boost b/c they are
+        # broken for Boost_COMPILER override, so we need to specify both
+        # Boost_NO_BOOST_CMAKE and a slightly different Boost_COMPILER string
+        # to make the legacy find module work for us. If the cmake configs are
+        # fixed in the future, it should be possible to remove these
+        # workarounds.
+        - export CMAKE_EXTRA_ARGS="${CMAKE_EXTRA_ARGS} -DBoost_NO_BOOST_CMAKE=ON -DBoost_COMPILER=-vc141"
+
+before_cache:
+  - if [ $(uname) = "Linux" ] ; then SUDO="sudo"; else SUDO=""; fi
+  - cd ${TRAVIS_HOME}
+  - if [ -f cache_ignore.tar ] ; then $SUDO tar xvf cache_ignore.tar; fi
+  - cd ${TRAVIS_BUILD_DIR}
+
+cache:
+  timeout: 900
+  directories:
+    - $CACHE_DIR
+
+notifications:
+  email: false

BUILD.md

@@ -1,434 +0,0 @@
-## A crash course in CMake and Conan
-
-To better understand how to use Conan,
-we should first understand _why_ we use Conan,
-and to understand that,
-we need to understand how we use CMake.
-
-
-### CMake
-
-Technically, you don't need CMake to build this project.
-You could manually compile every translation unit into an object file,
-using the right compiler options,
-and then manually link all those objects together,
-using the right linker options.
-However, that is very tedious and error-prone,
-which is why we lean on tools like CMake.
-
-We have written CMake configuration files
-([`CMakeLists.txt`](./CMakeLists.txt) and friends)
-for this project so that CMake can be used to correctly compile and link
-all of the translation units in it.
-Or rather, CMake will generate files for a separate build system
-(e.g. Make, Ninja, Visual Studio, Xcode, etc.)
-that compile and link all of the translation units.
-Even then, CMake has parameters, some of which are platform-specific.
-In CMake's parlance, parameters are specially-named **variables** like
-[`CMAKE_BUILD_TYPE`][build_type] or
-[`CMAKE_MSVC_RUNTIME_LIBRARY`][runtime].
-Parameters include:
-
-- what build system to generate files for
-- where to find the compiler and linker
-- where to find dependencies, e.g. libraries and headers
-- how to link dependencies, e.g. any special compiler or linker flags that
-    need to be used with them, including preprocessor definitions
-- how to compile translation units, e.g. with optimizations, debug symbols,
-    position-independent code, etc.
-- on Windows, which runtime library to link with
-
-For some of these parameters, like the build system and compiler,
-CMake goes through a complicated search process to choose default values.
-For others, like the dependencies,
-_we_ had written in the CMake configuration files of this project
-our own complicated process to choose defaults.
-For most developers, things "just worked"... until they didn't, and then
-you were left trying to debug one of these complicated processes, instead of
-choosing and manually passing the parameter values yourself.
-
-You can pass every parameter to CMake on the command line,
-but writing out these parameters every time we want to configure CMake is
-a pain.
-Most humans prefer to put them into a configuration file, once, that
-CMake can read every time it is configured.
-For CMake, that file is a [toolchain file][toolchain].
-
-
-### Conan
-
-These next few paragraphs on Conan are going to read much like the ones above
-for CMake.
-
-Technically, you don't need Conan to build this project.
-You could manually download, configure, build, and install all of the
-dependencies yourself, and then pass all of the parameters necessary for
-CMake to link to those dependencies.
-To guarantee ABI compatibility, you must be sure to use the same set of
-compiler and linker options for all dependencies _and_ this project.
-However, that is very tedious and error-prone, which is why we lean on tools
-like Conan.
-
-We have written a Conan configuration file ([`conanfile.py`](./conanfile.py))
-so that Conan can be used to correctly download, configure, build, and install
-all of the dependencies for this project,
-using a single set of compiler and linker options for all of them.
-It generates files that contain almost all of the parameters that CMake
-expects.
-Those files include:
-
-- A single toolchain file.
-- For every dependency, a CMake [package configuration file][pcf],
-    [package version file][pvf], and for every build type, a package
-    targets file.
-    Together, these files implement version checking and define `IMPORTED`
-    targets for the dependencies.
-
-The toolchain file itself amends the search path
-([`CMAKE_PREFIX_PATH`][prefix_path]) so that [`find_package()`][find_package]
-will [discover][search] the generated package configuration files.
-
-**Nearly all we must do to properly configure CMake is pass the toolchain
-file.**
-What CMake parameters are left out?
-You'll still need to pick a build system generator,
-and if you choose a single-configuration generator,
-you'll need to pass the `CMAKE_BUILD_TYPE`,
-which should match the `build_type` setting you gave to Conan.
-
-Even then, Conan has parameters, some of which are platform-specific.
-In Conan's parlance, parameters are either settings or options.
-**Settings** are shared by all packages, e.g. the build type.
-**Options** are specific to a given package, e.g. whether to build and link
-OpenSSL as a shared library.
-
-For settings, Conan goes through a complicated search process to choose
-defaults.
-For options, each package recipe defines its own defaults.
-
-You can pass every parameter to Conan on the command line,
-but it is more convenient to put them in a [profile][profile].
-**All we must do to properly configure Conan is edit and pass the profile.**
-By default, Conan will use the profile named "default".
-You can let Conan create the default profile with this command:
-
-```
-conan profile new default --detect
-```
-
-
-## Branches
-
-For a stable release, choose the `master` branch or one of the [tagged
-releases](https://github.com/ripple/rippled/releases).
-
-```
-git checkout master
-```
-
-For the latest release candidate, choose the `release` branch.
-
-```
-git checkout release
-```
-
-If you are contributing or want the latest set of untested features,
-then use the `develop` branch.
-
-```
-git checkout develop
-```
-
-
-## Platforms
-
-rippled is written in the C++20 dialect and includes the `<concepts>` header.
-The [minimum compiler versions][2] that can compile this dialect are given
-below:
-
-| Compiler | Minimum Version
-|---|---
-| GCC         | 10
-| Clang       | 13
-| Apple Clang | 13.1.6
-| MSVC        | 19.23
-
-We do not recommend Windows for rippled production use at this time.
-As of January 2023, the Ubuntu platform has received the highest level of
-quality assurance, testing, and support.
-Additionally, 32-bit Windows development is not supported.
-
-Visual Studio 2022 is not yet supported.
-This is because rippled is not compatible with [Boost][] versions 1.78 or 1.79,
-but Conan cannot build Boost versions released earlier than them with VS 2022.
-We expect that rippled will be compatible with Boost 1.80, which should be
-released in August 2022.
-Until then, we advise Windows developers to use Visual Studio 2019.
-
-[Boost]: https://www.boost.org/
-
-
-## Prerequisites
-
-To build this package, you will need Python (>= 3.7),
-[Conan][] (>= 1.55), and [CMake][] (>= 3.16).
-
-> **Warning**
-> The commands in this document are not meant to be blindly copied and pasted.
-> This document is written for multiple audiences,
-> meaning that your particular circumstances may require some commands and not
-> others.
-> You should never run any commands without understanding what they do
-> and why you are running them.
->
-> These instructions assume a basic familiarity with Conan and CMake.
-> If you are unfamiliar with Conan,
-> then please read the [crash course](#a-crash-course-in-cmake-and-conan)
-> at the beginning of this document,
-> or the official [Getting Started][3] walkthrough.
-
-[Conan]: https://conan.io/downloads.html
-[CMake]: https://cmake.org/download/
-
-You'll need to compile in the C++20 dialect:
-
-```
-conan profile update settings.compiler.cppstd=20 default
-```
-
-Linux developers will commonly have a default Conan [profile][] that compiles
-with GCC and links with libstdc++.
-If you are linking with libstdc++ (see profile setting `compiler.libcxx`),
-then you will need to choose the `libstdc++11` ABI:
-
-```
-conan profile update settings.compiler.libcxx=libstdc++11 default
-```
-
-We find it necessary to use the x64 native build tools on Windows.
-An easy way to do that is to run the shortcut "x64 Native Tools Command
-Prompt" for the version of Visual Studio that you have installed.
-
-Windows developers must build rippled and its dependencies for the x64
-architecture:
-
-```
-conan profile update settings.arch=x86_64 default
-```
-
-If you have multiple compilers installed on your platform,
-then you'll need to make sure that Conan and CMake select the one you want to
-use.
-This setting will set the correct variables (`CMAKE_<LANG>_COMPILER`) in the
-generated CMake toolchain file:
-
-```
-conan profile update 'conf.tools.build:compiler_executables={"c": "<path>", "cpp": "<path>"}' default
-```
-
-It should choose the compiler for dependencies as well,
-but not all of them have a Conan recipe that respects this setting (yet).
-For the rest, you can set these environment variables:
-
-```
-conan profile update env.CC=<path> default
-conan profile update env.CXX=<path> default
-```
-
-
-## How to build and test
-
-Let's start with a couple of examples of common workflows.
-The first is for a single-configuration generator (e.g. Unix Makefiles) on
-Linux or MacOS:
-
-```
-conan export external/snappy snappy/1.1.9@
-mkdir .build
-cd .build
-conan install .. --output-folder . --build missing --settings build_type=Release
-cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake -DCMAKE_BUILD_TYPE=Release ..
-cmake --build .
-./rippled --unittest
-```
-
-The second is for a multi-configuration generator (e.g. Visual Studio) on
-Windows:
-
-```
-conan export external/snappy snappy/1.1.9@
-mkdir .build
-cd .build
-conan install .. --output-folder . --build missing --settings build_type=Release --settings compiler.runtime=MT
-conan install .. --output-folder . --build missing --settings build_type=Debug --settings compiler.runtime=MTd
-cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake ..
-cmake --build . --config Release
-cmake --build . --config Debug
-./Release/rippled --unittest
-./Debug/rippled --unittest
-```
-
-Now to explain the individual steps in each example:
-
-1. Export our [Conan recipe for Snappy](./external/snappy).
-
-    It does not explicitly link the C++ standard library,
-    which allows us to statically link it.
-
-1. Create a build directory (and move into it).
-
-    You can choose any name you want.
-
-    Conan will generate some files in what it calls the "install folder".
-    These files are implementation details that you don't need to worry about.
-    By default, the install folder is your current working directory.
-    If you don't move into your build directory before calling Conan,
-    then you may be annoyed to see it polluting your project root directory
-    with these files.
-    To make Conan put them in your build directory,
-    you'll have to add the option
-    `--install-folder` or `-if` to every `conan install` command.
-
-1. Generate CMake files for every configuration you want to build.
-
-    For a single-configuration generator, e.g. `Unix Makefiles` or `Ninja`,
-    you only need to run this command once.
-    For a multi-configuration generator, e.g. `Visual Studio`, you may want to
-    run it more than once.
-
-    Each of these commands should have a different `build_type` setting.
-    A second command with the same `build_type` setting will just overwrite
-    the files generated by the first.
-    You can pass the build type on the command line with `--settings
-    build_type=$BUILD_TYPE` or in the profile itself, under the section
-    `[settings]`, with the key `build_type`.
-
-    If you are using a Microsoft Visual C++ compiler, then you will need to
-    ensure consistency between the `build_type` setting and the
-    `compiler.runtime` setting.
-    When `build_type` is `Release`, `compiler.runtime` should be `MT`.
-    When `build_type` is `Debug`, `compiler.runtime` should be `MTd`.
-
-1. Configure CMake once.
-
-    For all choices of generator, pass the toolchain file generated by Conan.
-    It will be located at
-    `$OUTPUT_FOLDER/build/generators/conan_toolchain.cmake`.
-    If you are using a single-configuration generator, then pass the CMake
-    variable [`CMAKE_BUILD_TYPE`][build_type] and make sure it matches the
-    `build_type` setting you chose in the previous step.
-
-    This step is where you may pass build options for rippled.
-
-1. Build rippled.
-
-    For a multi-configuration generator, you must pass the option `--config`
-    to select the build configuration.
-    For a single-configuration generator, it will build whatever configuration
-    you passed for `CMAKE_BUILD_TYPE`.
-
-5. Test rippled.
-
-    The exact location of rippled in your build directory
-    depends on your choice of CMake generator.
-    You can run unit tests by passing `--unittest`.
-    Pass `--help` to see the rest of the command line options.
-
-
-### Options
-
-The `unity` option allows you to select between [unity][5] and non-unity
-builds.
-Unity builds may be faster for the first build (at the cost of much
-more memory) since they concatenate sources into fewer translation
-units.
-Non-unity builds may be faster for incremental builds, and can be helpful for
-detecting `#include` omissions.
-
-Below are the most commonly used options,
-with their default values in parentheses.
-
-- `assert` (OFF): Enable assertions.
-- `reporting` (OFF): Build the reporting mode feature.
-- `tests` (ON): Build tests.
-- `unity` (ON): Configure a [unity build][5].
-- `san` (): Enable a sanitizer with Clang. Choices are `thread` and `address`.
-
-
-### Troubleshooting
-
-#### Conan
-
-If you find trouble building dependencies after changing Conan settings,
-then you should retry after removing the Conan cache:
-
-```
-rm -rf ~/.conan/data
-```
-
-
-#### no std::result_of
-
-If your compiler version is recent enough to have removed `std::result_of` as
-part of C++20, e.g. Apple Clang 15.0,
-then you might need to add a preprocessor definition to your bulid:
-
-```
-conan profile update 'env.CFLAGS="-DBOOST_ASIO_HAS_STD_INVOKE_RESULT"' default
-conan profile update 'env.CXXFLAGS="-DBOOST_ASIO_HAS_STD_INVOKE_RESULT"' default
-conan profile update 'tools.build:cflags+=["-DBOOST_ASIO_HAS_STD_INVOKE_RESULT"]' default
-conan profile update 'tools.build:cxxflags+=["-DBOOST_ASIO_HAS_STD_INVOKE_RESULT"]' default
-```
-
-
-#### recompile with -fPIC
-
-```
-/usr/bin/ld.gold: error: /home/username/.conan/data/boost/1.77.0/_/_/package/.../lib/libboost_container.a(alloc_lib.o):
-  requires unsupported dynamic reloc 11; recompile with -fPIC
-```
-
-If you get a linker error like the one above suggesting that you recompile
-Boost with position-independent code, the reason is most likely that Conan
-downloaded a bad binary distribution of the dependency.
-For now, this seems to be a [bug][1] in Conan just for Boost 1.77.0 compiled
-with GCC for Linux.
-The solution is to build the dependency locally by passing `--build boost`
-when calling `conan install`:
-
-```
-conan install --build boost ...
-```
-
-
-## How to add a dependency
-
-If you want to experiment with a new package, here are the steps to get it
-working:
-
-1. Search for the package on [Conan Center](https://conan.io/center/).
-1. In [`conanfile.py`](./conanfile.py):
-    1. Add a version of the package to the `requires` property.
-    1. Change any default options for the package by adding them to the
-    `default_options` property (with syntax `'$package:$option': $value`)
-1. In [`CMakeLists.txt`](./CMakeLists.txt):
-    1. Add a call to `find_package($package REQUIRED)`.
-    1. Link a library from the package to the target `ripple_libs` (search for
-    the existing call to `target_link_libraries(ripple_libs INTERFACE ...)`).
-1. Start coding! Don't forget to include whatever headers you need from the
-   package.
-
-
-[1]: https://github.com/conan-io/conan-center-index/issues/13168
-[2]: https://en.cppreference.com/w/cpp/compiler_support/20
-[3]: https://docs.conan.io/en/latest/getting_started.html
-[5]: https://en.wikipedia.org/wiki/Unity_build
-[build_type]: https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html
-[runtime]: https://cmake.org/cmake/help/latest/variable/CMAKE_MSVC_RUNTIME_LIBRARY.html
-[toolchain]: https://cmake.org/cmake/help/latest/manual/cmake-toolchains.7.html
-[pcf]: https://cmake.org/cmake/help/latest/manual/cmake-packages.7.html#package-configuration-file
-[pvf]: https://cmake.org/cmake/help/latest/manual/cmake-packages.7.html#package-version-file
-[find_package]: https://cmake.org/cmake/help/latest/command/find_package.html
-[search]: https://cmake.org/cmake/help/latest/command/find_package.html#search-procedure
-[prefix_path]: https://cmake.org/cmake/help/latest/variable/CMAKE_PREFIX_PATH.html
-[profile]: https://docs.conan.io/en/latest/reference/profiles.html

Builds/CMake/FindRocksDB.cmake

@@ -0,0 +1,62 @@
+set (RocksDB_DIR "" CACHE PATH "Root directory of RocksDB distribution")
+
+find_path (RocksDB_INCLUDE_DIR
+    rocksdb/db.h
+    PATHS ${RocksDB_DIR})
+
+set (RocksDB_VERSION "")
+find_file (RocksDB_VERSION_FILE
+    rocksdb/version.h
+    PATHS ${RocksDB_DIR})
+if (RocksDB_VERSION_FILE)
+    file (READ ${RocksDB_VERSION_FILE} _verfile)
+    if ("${_verfile}" MATCHES "#define[ \\t]+ROCKSDB_MAJOR[ \\t]+([0-9]+)")
+        string (APPEND RocksDB_VERSION "${CMAKE_MATCH_1}")
+    else ()
+        string (APPEND RocksDB_VERSION "0")
+    endif()
+    if ("${_verfile}" MATCHES "#define[ \\t]+ROCKSDB_MINOR[ \\t]+([0-9]+)")
+        string (APPEND RocksDB_VERSION ".${CMAKE_MATCH_1}")
+    else ()
+        string (APPEND RocksDB_VERSION ".0")
+    endif()
+    if ("${_verfile}" MATCHES "#define[ \\t]+ROCKSDB_PATCH[ \\t]+([0-9]+)")
+        string (APPEND RocksDB_VERSION ".${CMAKE_MATCH_1}")
+    else ()
+        string (APPEND RocksDB_VERSION ".0")
+    endif()
+endif ()
+
+if (RocksDB_USE_STATIC)
+    list (APPEND RocksDB_NAMES
+        "${CMAKE_STATIC_LIBRARY_PREFIX}rocksdb${CMAKE_STATIC_LIBRARY_SUFFIX}"
+        "${CMAKE_STATIC_LIBRARY_PREFIX}rocksdblib${CMAKE_STATIC_LIBRARY_SUFFIX}")
+endif ()
+
+list (APPEND RocksDB_NAMES rocksdb)
+
+find_library (RocksDB_LIBRARY NAMES ${RocksDB_NAMES}
+    PATHS
+    ${RocksDB_DIR}
+    ${RocksDB_DIR}/bin/Release
+    ${RocksDB_DIR}/bin64_vs2013/Release
+    PATH_SUFFIXES lib lib64)
+
+foreach (_n RocksDB_NAMES)
+    list (APPEND RocksDB_NAMES_DBG "${_n}_d" "${_n}d")
+endforeach ()
+find_library (RocksDB_LIBRARY_DEBUG NAMES ${RocksDB_NAMES_DBG}
+    PATHS
+    ${RocksDB_DIR}
+    ${RocksDB_DIR}/bin/Debug
+    ${RocksDB_DIR}/bin64_vs2013/Debug
+    PATH_SUFFIXES lib lib64)
+
+include (FindPackageHandleStandardArgs)
+find_package_handle_standard_args (RocksDB
+    REQUIRED_VARS RocksDB_LIBRARY RocksDB_INCLUDE_DIR
+    VERSION_VAR RocksDB_VERSION)
+
+mark_as_advanced (RocksDB_INCLUDE_DIR RocksDB_LIBRARY)
+set (RocksDB_INCLUDE_DIRS ${RocksDB_INCLUDE_DIR})
+set (RocksDB_LIBRARIES ${RocksDB_LIBRARY})

Builds/CMake/README.md

@@ -0,0 +1,18 @@
+
+These are modules and sources that support our CMake build.
+
+== FindBoost.cmake ==
+
+In order to facilitate updating to latest releases of boost, we've made a local
+copy of the FindBoost cmake module in our repo. The latest official version can
+generally be obtained
+[here](https://github.com/Kitware/CMake/blob/master/Modules/FindBoost.cmake).
+
+The latest version provided by Kitware can be tailored for use with the
+version of CMake that it ships with (typically the next upcoming CMake
+release). As such, the latest version from the repository might not work
+perfectly with older versions of CMake - for instance, the latest version 
+might use features or properties only available in the version of CMake that 
+it ships with. Given this, it's best to test any updates to this module with a few 
+different versions of cmake.
+

Builds/CMake/RippledCompiler.cmake

@@ -124,10 +124,7 @@ else ()
       #   * static option set and
       #   * NOT APPLE (AppleClang does not support static libc/c++) and
       #   * NOT san (sanitizers typically don't work with static libc/c++)
-      $<$<AND:$<BOOL:${static}>,$<NOT:$<BOOL:${APPLE}>>,$<NOT:$<BOOL:${san}>>>:
-      -static-libstdc++
-      -static-libgcc
-      >)
+      $<$<AND:$<BOOL:${static}>,$<NOT:$<BOOL:${APPLE}>>,$<NOT:$<BOOL:${san}>>>:-static-libstdc++>)
 endif ()
 
 if (use_gold AND is_gcc)

Builds/CMake/RippledCore.cmake

@@ -50,7 +50,7 @@ target_sources (xrpl_core PRIVATE
   src/ripple/basics/impl/FileUtilities.cpp
   src/ripple/basics/impl/IOUAmount.cpp
   src/ripple/basics/impl/Log.cpp
-  src/ripple/basics/impl/Number.cpp
+  src/ripple/basics/impl/strHex.cpp
   src/ripple/basics/impl/StringUtilities.cpp
   #[===============================[
     main sources:
@@ -82,7 +82,6 @@ target_sources (xrpl_core PRIVATE
   src/ripple/protocol/impl/PublicKey.cpp
   src/ripple/protocol/impl/Quality.cpp
   src/ripple/protocol/impl/Rate2.cpp
-  src/ripple/protocol/impl/Rules.cpp
   src/ripple/protocol/impl/SField.cpp
   src/ripple/protocol/impl/SOTemplate.cpp
   src/ripple/protocol/impl/STAccount.cpp
@@ -136,8 +135,8 @@ target_link_libraries (xrpl_core
     OpenSSL::Crypto
     Ripple::boost
     Ripple::syslibs
-    secp256k1::secp256k1
-    ed25519::ed25519
+    NIH::secp256k1
+    NIH::ed25519-donna
     date::date
     Ripple::opts)
 #[=================================[
@@ -154,12 +153,9 @@ install (
     src/ripple/basics/LocalValue.h
     src/ripple/basics/Log.h
     src/ripple/basics/MathUtilities.h
-    src/ripple/basics/Number.h
     src/ripple/basics/safe_cast.h
     src/ripple/basics/Slice.h
-    src/ripple/basics/spinlock.h
     src/ripple/basics/StringUtilities.h
-    src/ripple/basics/ThreadSafetyAnalysis.h
     src/ripple/basics/ToString.h
     src/ripple/basics/UnorderedContainers.h
     src/ripple/basics/XRPAmount.h
@@ -213,7 +209,6 @@ install (
     src/ripple/protocol/PublicKey.h
     src/ripple/protocol/Quality.h
     src/ripple/protocol/Rate.h
-    src/ripple/protocol/Rules.h
     src/ripple/protocol/SField.h
     src/ripple/protocol/SOTemplate.h
     src/ripple/protocol/STAccount.h
@@ -400,23 +395,29 @@ target_sources (rippled PRIVATE
   src/ripple/app/paths/Pathfinder.cpp
   src/ripple/app/paths/RippleCalc.cpp
   src/ripple/app/paths/RippleLineCache.cpp
-  src/ripple/app/paths/TrustLine.cpp
+  src/ripple/app/paths/RippleState.cpp
   src/ripple/app/paths/impl/BookStep.cpp
   src/ripple/app/paths/impl/DirectStep.cpp
   src/ripple/app/paths/impl/PaySteps.cpp
   src/ripple/app/paths/impl/XRPEndpointStep.cpp
-  src/ripple/app/rdb/backend/detail/impl/Node.cpp
-  src/ripple/app/rdb/backend/detail/impl/Shard.cpp
-  src/ripple/app/rdb/backend/impl/PostgresDatabase.cpp
-  src/ripple/app/rdb/backend/impl/SQLiteDatabase.cpp
-  src/ripple/app/rdb/impl/Download.cpp
-  src/ripple/app/rdb/impl/PeerFinder.cpp
-  src/ripple/app/rdb/impl/RelationalDatabase.cpp
-  src/ripple/app/rdb/impl/ShardArchive.cpp
-  src/ripple/app/rdb/impl/State.cpp
-  src/ripple/app/rdb/impl/UnitaryShard.cpp
-  src/ripple/app/rdb/impl/Vacuum.cpp
-  src/ripple/app/rdb/impl/Wallet.cpp
+  src/ripple/app/rdb/backend/RelationalDBInterfacePostgres.cpp
+  src/ripple/app/rdb/backend/RelationalDBInterfaceSqlite.cpp
+  src/ripple/app/rdb/impl/RelationalDBInterface.cpp
+  src/ripple/app/rdb/impl/RelationalDBInterface_global.cpp
+  src/ripple/app/rdb/impl/RelationalDBInterface_nodes.cpp
+  src/ripple/app/rdb/impl/RelationalDBInterface_postgres.cpp
+  src/ripple/app/rdb/impl/RelationalDBInterface_shards.cpp
+  src/ripple/app/sidechain/Federator.cpp
+  src/ripple/app/sidechain/FederatorEvents.cpp
+  src/ripple/app/sidechain/impl/ChainListener.cpp
+  src/ripple/app/sidechain/impl/DoorKeeper.cpp
+  src/ripple/app/sidechain/impl/InitialSync.cpp
+  src/ripple/app/sidechain/impl/MainchainListener.cpp
+  src/ripple/app/sidechain/impl/SidechainListener.cpp
+  src/ripple/app/sidechain/impl/SignatureCollector.cpp
+  src/ripple/app/sidechain/impl/SignerList.cpp
+  src/ripple/app/sidechain/impl/TicketHolder.cpp
+  src/ripple/app/sidechain/impl/WebsocketClient.cpp
   src/ripple/app/tx/impl/ApplyContext.cpp
   src/ripple/app/tx/impl/BookTip.cpp
   src/ripple/app/tx/impl/CancelCheck.cpp
@@ -430,11 +431,6 @@ target_sources (rippled PRIVATE
   src/ripple/app/tx/impl/DepositPreauth.cpp
   src/ripple/app/tx/impl/Escrow.cpp
   src/ripple/app/tx/impl/InvariantCheck.cpp
-  src/ripple/app/tx/impl/NFTokenAcceptOffer.cpp
-  src/ripple/app/tx/impl/NFTokenBurn.cpp
-  src/ripple/app/tx/impl/NFTokenCancelOffer.cpp
-  src/ripple/app/tx/impl/NFTokenCreateOffer.cpp
-  src/ripple/app/tx/impl/NFTokenMint.cpp
   src/ripple/app/tx/impl/OfferStream.cpp
   src/ripple/app/tx/impl/PayChan.cpp
   src/ripple/app/tx/impl/Payment.cpp
@@ -447,7 +443,6 @@ target_sources (rippled PRIVATE
   src/ripple/app/tx/impl/Transactor.cpp
   src/ripple/app/tx/impl/apply.cpp
   src/ripple/app/tx/impl/applySteps.cpp
-  src/ripple/app/tx/impl/details/NFTokenUtils.cpp
   #[===============================[
      main sources:
        subdir: basics (partial)
@@ -582,6 +577,7 @@ target_sources (rippled PRIVATE
   src/ripple/rpc/handlers/AccountObjects.cpp
   src/ripple/rpc/handlers/AccountOffers.cpp
   src/ripple/rpc/handlers/AccountTx.cpp
+  src/ripple/rpc/handlers/AccountTxOld.cpp
   src/ripple/rpc/handlers/BlackList.cpp
   src/ripple/rpc/handlers/BookOffers.cpp
   src/ripple/rpc/handlers/CanDelete.cpp
@@ -591,6 +587,7 @@ target_sources (rippled PRIVATE
   src/ripple/rpc/handlers/DepositAuthorized.cpp
   src/ripple/rpc/handlers/DownloadShard.cpp
   src/ripple/rpc/handlers/Feature1.cpp
+  src/ripple/rpc/handlers/FederatorInfo.cpp
   src/ripple/rpc/handlers/Fee1.cpp
   src/ripple/rpc/handlers/FetchInfo.cpp
   src/ripple/rpc/handlers/GatewayBalances.cpp
@@ -608,7 +605,6 @@ target_sources (rippled PRIVATE
   src/ripple/rpc/handlers/LogLevel.cpp
   src/ripple/rpc/handlers/LogRotate.cpp
   src/ripple/rpc/handlers/Manifest.cpp
-  src/ripple/rpc/handlers/NFTOffers.cpp
   src/ripple/rpc/handlers/NodeToShard.cpp
   src/ripple/rpc/handlers/NoRippleCheck.cpp
   src/ripple/rpc/handlers/OwnerInfo.cpp
@@ -641,6 +637,7 @@ target_sources (rippled PRIVATE
   src/ripple/rpc/handlers/WalletPropose.cpp
   src/ripple/rpc/impl/DeliveredAmount.cpp
   src/ripple/rpc/impl/Handler.cpp
+  src/ripple/rpc/impl/GRPCHelpers.cpp
   src/ripple/rpc/impl/LegacyPathFind.cpp
   src/ripple/rpc/impl/RPCHandler.cpp
   src/ripple/rpc/impl/RPCHelpers.cpp
@@ -702,9 +699,6 @@ if (tests)
     src/test/app/LoadFeeTrack_test.cpp
     src/test/app/Manifest_test.cpp
     src/test/app/MultiSign_test.cpp
-    src/test/app/NFToken_test.cpp
-    src/test/app/NFTokenBurn_test.cpp
-    src/test/app/NFTokenDir_test.cpp
     src/test/app/OfferStream_test.cpp
     src/test/app/Offer_test.cpp
     src/test/app/OversizeMeta_test.cpp
@@ -739,7 +733,6 @@ if (tests)
     src/test/basics/FileUtilities_test.cpp
     src/test/basics/IOUAmount_test.cpp
     src/test/basics/KeyCache_test.cpp
-    src/test/basics/Number_test.cpp
     src/test/basics/PerfLog_test.cpp
     src/test/basics/RangeSet_test.cpp
     src/test/basics/scope_test.cpp
@@ -752,7 +745,6 @@ if (tests)
     src/test/basics/contract_test.cpp
     src/test/basics/FeeUnits_test.cpp
     src/test/basics/hardened_hash_test.cpp
-    src/test/basics/join_test.cpp
     src/test/basics/mulDiv_test.cpp
     src/test/basics/tagged_integer_test.cpp
     #[===============================[
@@ -855,7 +847,6 @@ if (tests)
     src/test/jtx/impl/sig.cpp
     src/test/jtx/impl/tag.cpp
     src/test/jtx/impl/ticket.cpp
-    src/test/jtx/impl/token.cpp
     src/test/jtx/impl/trust.cpp
     src/test/jtx/impl/txflags.cpp
     src/test/jtx/impl/utility.cpp
@@ -911,8 +902,7 @@ if (tests)
     src/test/protocol/BuildInfo_test.cpp
     src/test/protocol/InnerObjectFormats_test.cpp
     src/test/protocol/Issue_test.cpp
-    src/test/protocol/Hooks_test.cpp
-    src/test/protocol/Memo_test.cpp
+    src/test/protocol/KnownFormatToGRPC_test.cpp
     src/test/protocol/PublicKey_test.cpp
     src/test/protocol/Quality_test.cpp
     src/test/protocol/STAccount_test.cpp
@@ -946,6 +936,7 @@ if (tests)
     src/test/rpc/DepositAuthorized_test.cpp
     src/test/rpc/DeliveredAmount_test.cpp
     src/test/rpc/Feature_test.cpp
+    src/test/rpc/Fee_test.cpp
     src/test/rpc/GatewayBalances_test.cpp
     src/test/rpc/GetCounts_test.cpp
     src/test/rpc/JSONRPC_test.cpp
@@ -968,10 +959,12 @@ if (tests)
     src/test/rpc/ServerInfo_test.cpp
     src/test/rpc/ShardArchiveHandler_test.cpp
     src/test/rpc/Status_test.cpp
+    src/test/rpc/Submit_test.cpp
     src/test/rpc/Subscribe_test.cpp
     src/test/rpc/Transaction_test.cpp
     src/test/rpc/TransactionEntry_test.cpp
     src/test/rpc/TransactionHistory_test.cpp
+    src/test/rpc/Tx_test.cpp
     src/test/rpc/ValidatorInfo_test.cpp
     src/test/rpc/ValidatorRPC_test.cpp
     src/test/rpc/Version_test.cpp
@@ -1008,18 +1001,17 @@ if (is_ci)
   target_compile_definitions(rippled PRIVATE RIPPLED_RUNNING_IN_CI)
 endif ()
 
-if(reporting)
-set_target_properties(rippled PROPERTIES OUTPUT_NAME rippled-reporting)
-get_target_property(BIN_NAME rippled OUTPUT_NAME)
-message(STATUS "Reporting mode build: rippled renamed ${BIN_NAME}")
-  target_compile_definitions(rippled PRIVATE RIPPLED_REPORTING)
-endif()
+if (reporting)
+    target_compile_definitions(rippled PRIVATE RIPPLED_REPORTING)
+endif ()
 
-# any files that don't play well with unity should be added here
-if (tests)
-  set_source_files_properties(
-    # these two seem to produce conflicts in beast teardown template methods
-    src/test/rpc/ValidatorRPC_test.cpp
-    src/test/rpc/ShardArchiveHandler_test.cpp
-    PROPERTIES SKIP_UNITY_BUILD_INCLUSION TRUE)
-endif () #tests
+if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.16)
+  # any files that don't play well with unity should be added here
+  if (tests)
+    set_source_files_properties(
+      # these two seem to produce conflicts in beast teardown template methods
+      src/test/rpc/ValidatorRPC_test.cpp
+      src/test/rpc/ShardArchiveHandler_test.cpp
+      PROPERTIES SKIP_UNITY_BUILD_INCLUSION TRUE)
+  endif () #tests
+endif ()

Builds/CMake/RippledDocs.cmake

@@ -1,84 +1,79 @@
 #[===================================================================[
    docs target (optional)
 #]===================================================================]
-
-option(with_docs "Include the docs target?" FALSE)
-
-if(NOT (with_docs OR only_docs))
-  return()
-endif()
-
-find_package(Doxygen)
-if(NOT TARGET Doxygen::doxygen)
-  message(STATUS "doxygen executable not found -- skipping docs target")
-  return()
-endif()
-
-set(doxygen_output_directory "${CMAKE_BINARY_DIR}/docs")
-set(doxygen_include_path "${CMAKE_CURRENT_SOURCE_DIR}/src")
-set(doxygen_index_file "${doxygen_output_directory}/html/index.html")
-set(doxyfile "${CMAKE_CURRENT_SOURCE_DIR}/docs/Doxyfile")
-
-file(GLOB_RECURSE doxygen_input
-  docs/*.md
-  src/ripple/*.h
-  src/ripple/*.cpp
-  src/ripple/*.md
-  src/test/*.h
-  src/test/*.md
-  Builds/*/README.md)
-list(APPEND doxygen_input
-  README.md
-  RELEASENOTES.md
-  src/README.md)
-set(dependencies "${doxygen_input}" "${doxyfile}")
-
-function(verbose_find_path variable name)
-  # find_path sets a CACHE variable, so don't try using a "local" variable.
-  find_path(${variable} "${name}" ${ARGN})
-  if(NOT ${variable})
-    message(NOTICE "could not find ${name}")
-  else()
-    message(STATUS "found ${name}: ${${variable}}/${name}")
-  endif()
-endfunction()
-
-verbose_find_path(doxygen_plantuml_jar_path plantuml.jar PATH_SUFFIXES share/plantuml)
-verbose_find_path(doxygen_dot_path dot)
-
-# https://en.cppreference.com/w/Cppreference:Archives
-# https://stackoverflow.com/questions/60822559/how-to-move-a-file-download-from-configure-step-to-build-step
-set(download_script "${CMAKE_BINARY_DIR}/docs/download-cppreference.cmake")
-file(WRITE
-  "${download_script}"
-  "file(DOWNLOAD \
-    http://upload.cppreference.com/mwiki/images/b/b2/html_book_20190607.zip \
-    ${CMAKE_BINARY_DIR}/docs/cppreference.zip \
-    EXPECTED_HASH MD5=82b3a612d7d35a83e3cb1195a63689ab \
-  )\n \
-  execute_process( \
-    COMMAND \"${CMAKE_COMMAND}\" -E tar -xf cppreference.zip \
-  )\n"
-)
-set(tagfile "${CMAKE_BINARY_DIR}/docs/cppreference-doxygen-web.tag.xml")
-add_custom_command(
-  OUTPUT "${tagfile}"
-  COMMAND "${CMAKE_COMMAND}" -P "${download_script}"
-  WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/docs"
-)
-set(doxygen_tagfiles "${tagfile}=http://en.cppreference.com/w/")
-
-add_custom_command(
-  OUTPUT "${doxygen_index_file}"
-  COMMAND "${CMAKE_COMMAND}" -E env
-    "DOXYGEN_OUTPUT_DIRECTORY=${doxygen_output_directory}"
-    "DOXYGEN_INCLUDE_PATH=${doxygen_include_path}"
-    "DOXYGEN_TAGFILES=${doxygen_tagfiles}"
-    "DOXYGEN_PLANTUML_JAR_PATH=${doxygen_plantuml_jar_path}"
-    "DOXYGEN_DOT_PATH=${doxygen_dot_path}"
-    "${DOXYGEN_EXECUTABLE}" "${doxyfile}"
-  WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
-  DEPENDS "${dependencies}" "${tagfile}")
-add_custom_target(docs
-  DEPENDS "${doxygen_index_file}"
-  SOURCES "${dependencies}")
+if (tests)
+  find_package (Doxygen)
+  if (NOT TARGET Doxygen::doxygen)
+    message (STATUS "doxygen executable not found -- skipping docs target")
+    return ()
+  endif ()
+  
+  set (doxygen_output_directory "${CMAKE_BINARY_DIR}/docs")
+  set (doxygen_include_path "${CMAKE_CURRENT_SOURCE_DIR}/src")
+  set (doxygen_index_file "${doxygen_output_directory}/html/index.html")
+  set (doxyfile "${CMAKE_CURRENT_SOURCE_DIR}/docs/Doxyfile")
+  
+  file (GLOB_RECURSE doxygen_input
+    docs/*.md
+    src/ripple/*.h
+    src/ripple/*.cpp
+    src/ripple/*.md
+    src/test/*.h
+    src/test/*.md
+    Builds/*/README.md)
+  list (APPEND doxygen_input
+    README.md
+    RELEASENOTES.md
+    src/README.md)
+  set (dependencies "${doxygen_input}" "${doxyfile}")
+  
+  function (verbose_find_path variable name)
+    # find_path sets a CACHE variable, so don't try using a "local" variable.
+    find_path (${variable} "${name}" ${ARGN})
+    if (NOT ${variable})
+      message (WARNING "could not find ${name}")
+    else ()
+      message (STATUS "found ${name}: ${${variable}}/${name}")
+    endif ()
+  endfunction ()
+  
+  verbose_find_path (doxygen_plantuml_jar_path plantuml.jar PATH_SUFFIXES share/plantuml)
+  verbose_find_path (doxygen_dot_path dot)
+  
+  # https://en.cppreference.com/w/Cppreference:Archives
+  # https://stackoverflow.com/questions/60822559/how-to-move-a-file-download-from-configure-step-to-build-step
+  set (download_script "${CMAKE_BINARY_DIR}/docs/download-cppreference.cmake")
+  file (WRITE
+    "${download_script}"
+    "file (DOWNLOAD \
+      http://upload.cppreference.com/mwiki/images/b/b2/html_book_20190607.zip \
+      ${CMAKE_BINARY_DIR}/docs/cppreference.zip \
+      EXPECTED_HASH MD5=82b3a612d7d35a83e3cb1195a63689ab \
+    )\n \
+    execute_process ( \
+      COMMAND \"${CMAKE_COMMAND}\" -E tar -xf cppreference.zip \
+    )\n"
+  )
+  set (tagfile "${CMAKE_BINARY_DIR}/docs/cppreference-doxygen-web.tag.xml")
+  add_custom_command (
+    OUTPUT "${tagfile}"
+    COMMAND "${CMAKE_COMMAND}" -P "${download_script}"
+    WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/docs"
+  )
+  set (doxygen_tagfiles "${tagfile}=http://en.cppreference.com/w/")
+  
+  add_custom_command (
+    OUTPUT "${doxygen_index_file}"
+    COMMAND "${CMAKE_COMMAND}" -E env
+      "DOXYGEN_OUTPUT_DIRECTORY=${doxygen_output_directory}"
+      "DOXYGEN_INCLUDE_PATH=${doxygen_include_path}"
+      "DOXYGEN_TAGFILES=${doxygen_tagfiles}"
+      "DOXYGEN_PLANTUML_JAR_PATH=${doxygen_plantuml_jar_path}"
+      "DOXYGEN_DOT_PATH=${doxygen_dot_path}"
+      "${DOXYGEN_EXECUTABLE}" "${doxyfile}"
+    WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+    DEPENDS "${dependencies}" "${tagfile}")
+  add_custom_target (docs
+    DEPENDS "${doxygen_index_file}"
+    SOURCES "${dependencies}")
+endif ()

Builds/CMake/RippledInstall.cmake

@@ -4,6 +4,7 @@
 
 install (
   TARGETS
+    ed25519-donna
     common
     opts
     ripple_syslibs
@@ -15,6 +16,17 @@ install (
   RUNTIME DESTINATION bin
   INCLUDES DESTINATION include)
 
+if(${INSTALL_SECP256K1})
+install (
+  TARGETS
+    secp256k1
+  EXPORT RippleExports
+  LIBRARY DESTINATION lib
+  ARCHIVE DESTINATION lib
+  RUNTIME DESTINATION bin
+  INCLUDES DESTINATION include)
+endif()
+
 install (EXPORT RippleExports
   FILE RippleTargets.cmake
   NAMESPACE Ripple::

Builds/CMake/RippledInterface.cmake

@@ -35,10 +35,17 @@ target_link_libraries (opts
     $<$<BOOL:${profile}>:-pg>
     $<$<AND:$<BOOL:${is_gcc}>,$<BOOL:${profile}>>:-p>)
 
-if(jemalloc)
-  find_package(jemalloc REQUIRED)
-  target_compile_definitions(opts INTERFACE PROFILE_JEMALLOC)
-  target_link_libraries(opts INTERFACE jemalloc::jemalloc)
+if (jemalloc)
+  if (static)
+    set(JEMALLOC_USE_STATIC ON CACHE BOOL "" FORCE)
+  endif ()
+  find_package (jemalloc REQUIRED)
+  target_compile_definitions (opts INTERFACE PROFILE_JEMALLOC)
+  target_include_directories (opts SYSTEM INTERFACE ${JEMALLOC_INCLUDE_DIRS})
+  target_link_libraries (opts INTERFACE ${JEMALLOC_LIBRARIES})
+  get_filename_component (JEMALLOC_LIB_PATH ${JEMALLOC_LIBRARIES} DIRECTORY)
+  ## TODO see if we can use the BUILD_RPATH target property (is it transitive?)
+  set (CMAKE_BUILD_RPATH ${CMAKE_BUILD_RPATH} ${JEMALLOC_LIB_PATH})
 endif ()
 
 if (san)

Builds/CMake/RippledMultiConfig.cmake

@@ -14,7 +14,7 @@ if (is_multiconfig)
   file(GLOB md_files RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} CONFIGURE_DEPENDS
     *.md)
   LIST(APPEND all_sources ${md_files})
-  foreach (_target secp256k1::secp256k1 ed25519::ed25519 pbufs xrpl_core rippled)
+  foreach (_target secp256k1 ed25519-donna pbufs xrpl_core rippled)
     get_target_property (_type ${_target} TYPE)
     if(_type STREQUAL "INTERFACE_LIBRARY")
       continue()

Builds/CMake/RippledNIH.cmake

@@ -0,0 +1,33 @@
+#[===================================================================[
+   NIH prefix path..this is where we will download
+   and build any ExternalProjects, and they will hopefully
+   survive across build directory deletion (manual cleans)
+#]===================================================================]
+
+string (REGEX REPLACE "[ \\/%]+" "_" gen_for_path ${CMAKE_GENERATOR})
+string (TOLOWER ${gen_for_path} gen_for_path)
+# HACK: trying to shorten paths for windows CI (which hits 260 MAXPATH easily)
+# @see:  https://issues.jenkins-ci.org/browse/JENKINS-38706?focusedCommentId=339847
+string (REPLACE "visual_studio" "vs" gen_for_path ${gen_for_path})
+if (NOT DEFINED NIH_CACHE_ROOT)
+  if (DEFINED ENV{NIH_CACHE_ROOT})
+    set (NIH_CACHE_ROOT $ENV{NIH_CACHE_ROOT})
+  else ()
+    set (NIH_CACHE_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/.nih_c")
+  endif ()
+endif ()
+set (nih_cache_path
+  "${NIH_CACHE_ROOT}/${gen_for_path}/${CMAKE_CXX_COMPILER_ID}_${CMAKE_CXX_COMPILER_VERSION}")
+if (NOT is_multiconfig)
+  set (nih_cache_path "${nih_cache_path}/${CMAKE_BUILD_TYPE}")
+endif ()
+file(TO_CMAKE_PATH "${nih_cache_path}" nih_cache_path)
+message (STATUS "NIH-EP cache path: ${nih_cache_path}")
+## two convenience variables:
+set (ep_lib_prefix ${CMAKE_STATIC_LIBRARY_PREFIX})
+set (ep_lib_suffix ${CMAKE_STATIC_LIBRARY_SUFFIX})
+
+# this is a setting for FetchContent and needs to be
+# a cache variable
+# https://cmake.org/cmake/help/latest/module/FetchContent.html#populating-the-content
+set (FETCHCONTENT_BASE_DIR ${nih_cache_path} CACHE STRING "" FORCE)

Builds/CMake/RippledRelease.cmake

@@ -16,7 +16,20 @@ if (is_root_project)
     message (STATUS "using [${container_label}] as build container tag...")
 
     file (MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/packages)
-
+    file (MAKE_DIRECTORY ${NIH_CACHE_ROOT}/pkgbuild)
+    if (is_linux)
+      execute_process (COMMAND id -u
+        OUTPUT_VARIABLE DOCKER_USER_ID
+        OUTPUT_STRIP_TRAILING_WHITESPACE)
+      message (STATUS "docker local user id: ${DOCKER_USER_ID}")
+      execute_process (COMMAND id -g
+        OUTPUT_VARIABLE DOCKER_GROUP_ID
+        OUTPUT_STRIP_TRAILING_WHITESPACE)
+      message (STATUS "docker local group id: ${DOCKER_GROUP_ID}")
+    endif ()
+    if (DOCKER_USER_ID AND DOCKER_GROUP_ID)
+      set(map_user TRUE)
+    endif ()
     #[===================================================================[
         rpm
     #]===================================================================]
@@ -24,7 +37,7 @@ if (is_root_project)
       docker build
         --pull
         --build-arg GIT_COMMIT=${commit_hash}
-        -t rippleci/rippled-rpm-builder:${container_label}
+        -t rippled-rpm-builder:${container_label}
         $<$<BOOL:${rpm_cache_from}>:--cache-from=${rpm_cache_from}>
         -f centos-builder/Dockerfile .
       WORKING_DIRECTORY  ${CMAKE_CURRENT_SOURCE_DIR}/Builds/containers
@@ -34,22 +47,24 @@ if (is_root_project)
       SOURCES
         Builds/containers/centos-builder/Dockerfile
         Builds/containers/centos-builder/centos_setup.sh
-        Builds/containers/shared/update-rippled.sh
-        Builds/containers/shared/update_sources.sh
+        Builds/containers/centos-builder/extras.sh
+        Builds/containers/shared/build_deps.sh
         Builds/containers/shared/rippled.service
-        Builds/containers/shared/rippled-reporting.service
+        Builds/containers/shared/update_sources.sh
+        Builds/containers/shared/update-rippled.sh
         Builds/containers/packaging/rpm/rippled.spec
         Builds/containers/packaging/rpm/build_rpm.sh
-        Builds/containers/packaging/rpm/50-rippled.preset
-        Builds/containers/packaging/rpm/50-rippled-reporting.preset
         bin/getRippledInfo
     )
     exclude_from_default (rpm_container)
     add_custom_target (rpm
       docker run
+        -e NIH_CACHE_ROOT=/opt/rippled_bld/pkg/.nih_c
+        -v ${NIH_CACHE_ROOT}/pkgbuild:/opt/rippled_bld/pkg/.nih_c
         -v ${CMAKE_CURRENT_SOURCE_DIR}:/opt/rippled_bld/pkg/rippled
         -v ${CMAKE_CURRENT_BINARY_DIR}/packages:/opt/rippled_bld/pkg/out
-        -t rippleci/rippled-rpm-builder:${container_label}
+        "$<$<BOOL:${map_user}>:--volume=/etc/passwd:/etc/passwd;--volume=/etc/group:/etc/group;--user=${DOCKER_USER_ID}:${DOCKER_GROUP_ID}>"
+        -t rippled-rpm-builder:${container_label}
         /bin/bash -c "cp -fpu rippled/Builds/containers/packaging/rpm/build_rpm.sh . && ./build_rpm.sh"
       VERBATIM
       USES_TERMINAL
@@ -64,14 +79,14 @@ if (is_root_project)
     #[===================================================================[
         dpkg
     #]===================================================================]
-    # currently use ubuntu 18.04 as a base b/c it has one of
+    # currently use ubuntu 16.04 as a base b/c it has one of
     # the lower versions of libc among ubuntu and debian releases.
     # we could change this in the future and build with some other deb
     # based system.
     add_custom_target (dpkg_container
       docker build
         --pull
-        --build-arg DIST_TAG=18.04
+        --build-arg DIST_TAG=16.04
         --build-arg GIT_COMMIT=${commit_hash}
         -t rippled-dpkg-builder:${container_label}
         $<$<BOOL:${dpkg_cache_from}>:--cache-from=${dpkg_cache_from}>
@@ -81,44 +96,37 @@ if (is_root_project)
       USES_TERMINAL
       COMMAND_EXPAND_LISTS
       SOURCES
-        Builds/containers/packaging/dpkg/debian/rippled-reporting.links
-        Builds/containers/packaging/dpkg/debian/copyright
-        Builds/containers/packaging/dpkg/debian/rules
-        Builds/containers/packaging/dpkg/debian/rippled-reporting.install
-        Builds/containers/packaging/dpkg/debian/rippled-reporting.postinst
-        Builds/containers/packaging/dpkg/debian/rippled.links
-        Builds/containers/packaging/dpkg/debian/rippled.prerm
-        Builds/containers/packaging/dpkg/debian/rippled.postinst
-        Builds/containers/packaging/dpkg/debian/rippled-dev.install
-        Builds/containers/packaging/dpkg/debian/dirs
-        Builds/containers/packaging/dpkg/debian/rippled.postrm
-        Builds/containers/packaging/dpkg/debian/rippled.conffiles
-        Builds/containers/packaging/dpkg/debian/compat
-        Builds/containers/packaging/dpkg/debian/source/format
-        Builds/containers/packaging/dpkg/debian/source/local-options
-        Builds/containers/packaging/dpkg/debian/README.Debian
-        Builds/containers/packaging/dpkg/debian/rippled.install
-        Builds/containers/packaging/dpkg/debian/rippled.preinst
-        Builds/containers/packaging/dpkg/debian/docs
-        Builds/containers/packaging/dpkg/debian/control
-        Builds/containers/packaging/dpkg/debian/rippled-reporting.dirs
-        Builds/containers/packaging/dpkg/build_dpkg.sh
         Builds/containers/ubuntu-builder/Dockerfile
         Builds/containers/ubuntu-builder/ubuntu_setup.sh
-        bin/getRippledInfo
-        Builds/containers/shared/install_cmake.sh
-        Builds/containers/shared/update-rippled.sh
-        Builds/containers/shared/update_sources.sh
+        Builds/containers/shared/build_deps.sh
         Builds/containers/shared/rippled.service
-        Builds/containers/shared/rippled-reporting.service
-        Builds/containers/shared/rippled-logrotate
-        Builds/containers/shared/update-rippled-cron
+        Builds/containers/shared/update_sources.sh
+        Builds/containers/shared/update-rippled.sh
+        Builds/containers/packaging/dpkg/build_dpkg.sh
+        Builds/containers/packaging/dpkg/debian/README.Debian
+        Builds/containers/packaging/dpkg/debian/conffiles
+        Builds/containers/packaging/dpkg/debian/control
+        Builds/containers/packaging/dpkg/debian/copyright
+        Builds/containers/packaging/dpkg/debian/dirs
+        Builds/containers/packaging/dpkg/debian/docs
+        Builds/containers/packaging/dpkg/debian/rippled-dev.install
+        Builds/containers/packaging/dpkg/debian/rippled.install
+        Builds/containers/packaging/dpkg/debian/rippled.links
+        Builds/containers/packaging/dpkg/debian/rippled.postinst
+        Builds/containers/packaging/dpkg/debian/rippled.postrm
+        Builds/containers/packaging/dpkg/debian/rippled.preinst
+        Builds/containers/packaging/dpkg/debian/rippled.prerm
+        Builds/containers/packaging/dpkg/debian/rules
+        bin/getRippledInfo
     )
     exclude_from_default (dpkg_container)
     add_custom_target (dpkg
       docker run
+        -e NIH_CACHE_ROOT=/opt/rippled_bld/pkg/.nih_c
+        -v ${NIH_CACHE_ROOT}/pkgbuild:/opt/rippled_bld/pkg/.nih_c
         -v ${CMAKE_CURRENT_SOURCE_DIR}:/opt/rippled_bld/pkg/rippled
         -v ${CMAKE_CURRENT_BINARY_DIR}/packages:/opt/rippled_bld/pkg/out
+        "$<$<BOOL:${map_user}>:--volume=/etc/passwd:/etc/passwd;--volume=/etc/group:/etc/group;--user=${DOCKER_USER_ID}:${DOCKER_GROUP_ID}>"
         -t rippled-dpkg-builder:${container_label}
         /bin/bash -c "cp -fpu rippled/Builds/containers/packaging/dpkg/build_dpkg.sh . && ./build_dpkg.sh"
       VERBATIM
@@ -172,9 +180,11 @@ if (is_root_project)
       SOURCES
         Builds/containers/ubuntu-builder/Dockerfile
         Builds/containers/ubuntu-builder/ubuntu_setup.sh
+        Builds/containers/shared/build_deps.sh
     )
     exclude_from_default (ci_container)
   else ()
     message (STATUS "docker NOT found -- won't be able to build containers for packaging")
   endif ()
 endif ()
+

Builds/CMake/RippledSanity.cmake

@@ -10,7 +10,12 @@ if (NOT ep_procs)
     message (STATUS "Using ${ep_procs} cores for ExternalProject builds.")
   endif ()
 endif ()
-get_property(is_multiconfig GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG)
+get_property (is_multiconfig GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG)
+if (is_multiconfig STREQUAL "NOTFOUND")
+  if (${CMAKE_GENERATOR} STREQUAL "Xcode" OR ${CMAKE_GENERATOR} MATCHES "^Visual Studio")
+    set (is_multiconfig TRUE)
+  endif ()
+endif ()
 
 set (CMAKE_CONFIGURATION_TYPES "Debug;Release" CACHE STRING "" FORCE)
 if (NOT is_multiconfig)
@@ -44,6 +49,9 @@ elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
     message (FATAL_ERROR "This project requires GCC 8 or later")
   endif ()
 endif ()
+if (CMAKE_GENERATOR STREQUAL "Xcode")
+  set (is_xcode TRUE)
+endif ()
 
 if (CMAKE_SYSTEM_NAME STREQUAL "Linux")
   set (is_linux TRUE)
@@ -64,8 +72,10 @@ if ("${CMAKE_CURRENT_SOURCE_DIR}" STREQUAL "${CMAKE_BINARY_DIR}")
     "directory from ${CMAKE_CURRENT_SOURCE_DIR} and try building in a separate directory.")
 endif ()
 
-if (MSVC AND CMAKE_GENERATOR_PLATFORM STREQUAL "Win32")
-  message (FATAL_ERROR "Visual Studio 32-bit build is not supported.")
+if ("${CMAKE_GENERATOR}" MATCHES "Visual Studio" AND
+    NOT ("${CMAKE_GENERATOR}" MATCHES .*Win64.*))
+  message (FATAL_ERROR
+    "Visual Studio 32-bit build is not supported. Use -G\"${CMAKE_GENERATOR} Win64\"")
 endif ()
 
 if (NOT CMAKE_SIZEOF_VOID_P EQUAL 8)

Builds/CMake/RippledSettings.cmake

@@ -10,8 +10,13 @@ option (tests "Build tests" ON)
 
 option (unity "Creates a build using UNITY support in cmake. This is the default" ON)
 if (unity)
-  if (NOT is_ci)
-    set (CMAKE_UNITY_BUILD_BATCH_SIZE 15 CACHE STRING "")
+  if (CMAKE_VERSION VERSION_LESS 3.16)
+    message (WARNING "unity option only supported for with cmake 3.16+ (please upgrade)")
+    set (unity OFF CACHE BOOL "unity only available for cmake 3.16+" FORCE)
+  else ()
+    if (NOT is_ci)
+      set (CMAKE_UNITY_BUILD_BATCH_SIZE 15 CACHE STRING "")
+    endif ()
   endif ()
 endif ()
 if (is_gcc OR is_clang)

Builds/CMake/RippledValidatorKeys.cmake

@@ -1,6 +1,6 @@
 option (validator_keys "Enables building of validator-keys-tool as a separate target (imported via FetchContent)" OFF)
 
-if (validator_keys)
+if (validator_keys AND CMAKE_VERSION VERSION_GREATER_EQUAL 3.11)
   git_branch (current_branch)
   # default to tracking VK develop branch unless we are on master/release
   if (NOT (current_branch STREQUAL "master" OR current_branch STREQUAL "release"))
@@ -20,3 +20,5 @@ if (validator_keys)
   endif ()
   add_subdirectory (${validator_keys_src_SOURCE_DIR} ${CMAKE_BINARY_DIR}/validator-keys)
 endif ()
+
+

Builds/CMake/SociConfig.cmake.patched

@@ -1,106 +0,0 @@
-################################################################################
-# SociConfig.cmake - CMake build configuration of SOCI library
-################################################################################
-# Copyright (C) 2010 Mateusz Loskot <mateusz@loskot.net>
-#
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-################################################################################
-
-include(CheckCXXSymbolExists)
-
-if(WIN32)
-  check_cxx_symbol_exists("_M_AMD64" "" SOCI_TARGET_ARCH_X64)
-  if(NOT RTC_ARCH_X64)
-    check_cxx_symbol_exists("_M_IX86" "" SOCI_TARGET_ARCH_X86)
-  endif(NOT RTC_ARCH_X64)
-  # add check for arm here
-  # see http://msdn.microsoft.com/en-us/library/b0084kay.aspx
-else(WIN32)
-  check_cxx_symbol_exists("__i386__" "" SOCI_TARGET_ARCH_X86)
-  check_cxx_symbol_exists("__x86_64__" "" SOCI_TARGET_ARCH_X64)
-  check_cxx_symbol_exists("__arm__" "" SOCI_TARGET_ARCH_ARM)
-endif(WIN32)
-
-if(NOT DEFINED LIB_SUFFIX)
-  if(SOCI_TARGET_ARCH_X64)
-    set(_lib_suffix "64")
-  else()
-    set(_lib_suffix "")
-  endif()
-  set(LIB_SUFFIX ${_lib_suffix} CACHE STRING "Specifies suffix for the lib directory")
-endif()
-
-#
-# C++11 Option
-#
-
-if(NOT SOCI_CXX_C11)
-  set (SOCI_CXX_C11 OFF CACHE BOOL "Build to the C++11 standard")
-endif()
-
-#
-# Force compilation flags and set desired warnings level
-#
-
-if (MSVC)
-  add_definitions(-D_CRT_SECURE_NO_DEPRECATE)
-  add_definitions(-D_CRT_SECURE_NO_WARNINGS)
-  add_definitions(-D_CRT_NONSTDC_NO_WARNING)
-  add_definitions(-D_SCL_SECURE_NO_WARNINGS)
-
-  if(CMAKE_CXX_FLAGS MATCHES "/W[0-4]")
-    string(REGEX REPLACE "/W[0-4]" "/W4" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
-  else()
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W4 /we4266")
-  endif()
-
-else()
-
-  set(SOCI_GCC_CLANG_COMMON_FLAGS "")
-    # "-pedantic -Werror -Wno-error=parentheses -Wall -Wextra -Wpointer-arith -Wcast-align -Wcast-qual -Wfloat-equal -Woverloaded-virtual -Wredundant-decls -Wno-long-long")
-
-
-  if (SOCI_CXX_C11)
-    set(SOCI_CXX_VERSION_FLAGS "-std=c++11")
-  else()
-    set(SOCI_CXX_VERSION_FLAGS "-std=gnu++98")
-  endif()
-
-  if("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang" OR "${CMAKE_CXX_COMPILER}" MATCHES "clang")
-
-    if(NOT CMAKE_CXX_COMPILER_VERSION LESS 3.1 AND SOCI_ASAN)
-      set(SOCI_GCC_CLANG_COMMON_FLAGS "${SOCI_GCC_CLANG_COMMON_FLAGS} -fsanitize=address")
-    endif()
-
-    # enforce C++11 for Clang
-    set(SOCI_CXX_C11 ON)
-    set(SOCI_CXX_VERSION_FLAGS "-std=c++11")
-    add_definitions(-DCATCH_CONFIG_CPP11_NO_IS_ENUM)
-
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SOCI_GCC_CLANG_COMMON_FLAGS} ${SOCI_CXX_VERSION_FLAGS}")
-
-  elseif(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_GNUCXX)
-
-    if(NOT CMAKE_CXX_COMPILER_VERSION LESS 4.8 AND SOCI_ASAN)
-      set(SOCI_GCC_CLANG_COMMON_FLAGS "${SOCI_GCC_CLANG_COMMON_FLAGS} -fsanitize=address")
-    endif()
-
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SOCI_GCC_CLANG_COMMON_FLAGS} ${SOCI_CXX_VERSION_FLAGS} ")
-    if (CMAKE_COMPILER_IS_GNUCXX)
-        if (CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
-            set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
-        else()
-            set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-variadic-macros")
-        endif()
-    endif()
-
-  else()
-	message(WARNING "Unknown toolset - using default flags to build SOCI")
-  endif()
-
-endif()
-
-# Set SOCI_HAVE_* variables for soci-config.h generator
-set(SOCI_HAVE_CXX_C11 ${SOCI_CXX_C11} CACHE INTERNAL "Enables C++11 support")

Builds/CMake/deps/Boost.cmake

@@ -1,3 +1,49 @@
+#[===================================================================[
+   NIH dep: boost
+#]===================================================================]
+
+if((NOT DEFINED BOOST_ROOT) AND(DEFINED ENV{BOOST_ROOT}))
+  set(BOOST_ROOT $ENV{BOOST_ROOT})
+endif()
+file(TO_CMAKE_PATH "${BOOST_ROOT}" BOOST_ROOT)
+if(WIN32 OR CYGWIN)
+  # Workaround for MSVC having two boost versions - x86 and x64 on same PC in stage folders
+  if(DEFINED BOOST_ROOT)
+    if(IS_DIRECTORY ${BOOST_ROOT}/stage64/lib)
+      set(BOOST_LIBRARYDIR ${BOOST_ROOT}/stage64/lib)
+    elseif(IS_DIRECTORY ${BOOST_ROOT}/stage/lib)
+      set(BOOST_LIBRARYDIR ${BOOST_ROOT}/stage/lib)
+    elseif(IS_DIRECTORY ${BOOST_ROOT}/lib)
+      set(BOOST_LIBRARYDIR ${BOOST_ROOT}/lib)
+    else()
+      message(WARNING "Did not find expected boost library dir. "
+        "Defaulting to ${BOOST_ROOT}")
+      set(BOOST_LIBRARYDIR ${BOOST_ROOT})
+    endif()
+  endif()
+endif()
+message(STATUS "BOOST_ROOT: ${BOOST_ROOT}")
+message(STATUS "BOOST_LIBRARYDIR: ${BOOST_LIBRARYDIR}")
+
+# uncomment the following as needed to debug FindBoost issues:
+#set(Boost_DEBUG ON)
+
+#[=========================================================[
+   boost dynamic libraries don't trivially support @rpath
+   linking right now (cmake's default), so just force
+   static linking for macos, or if requested on linux by flag
+#]=========================================================]
+if(static)
+  set(Boost_USE_STATIC_LIBS ON)
+endif()
+set(Boost_USE_MULTITHREADED ON)
+if(static AND NOT APPLE)
+  set(Boost_USE_STATIC_RUNTIME ON)
+else()
+  set(Boost_USE_STATIC_RUNTIME OFF)
+endif()
+# TBD:
+# Boost_USE_DEBUG_RUNTIME:  When ON, uses Boost libraries linked against the
 find_package(Boost 1.70 REQUIRED
   COMPONENTS
     chrono
@@ -9,12 +55,11 @@ find_package(Boost 1.70 REQUIRED
     program_options
     regex
     system
-    thread
-)
+    thread)
 
 add_library(ripple_boost INTERFACE)
 add_library(Ripple::boost ALIAS ripple_boost)
-if(XCODE)
+if(is_xcode)
   target_include_directories(ripple_boost BEFORE INTERFACE ${Boost_INCLUDE_DIRS})
   target_compile_options(ripple_boost INTERFACE --system-header-prefix="boost/")
 else()

Builds/CMake/deps/Ed25519-donna.cmake

@@ -0,0 +1,28 @@
+#[===================================================================[
+   NIH dep: ed25519-donna
+#]===================================================================]
+
+add_library (ed25519-donna STATIC
+  src/ed25519-donna/ed25519.c)
+target_include_directories (ed25519-donna
+  PUBLIC
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src>
+    $<INSTALL_INTERFACE:include>
+  PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/ed25519-donna)
+#[=========================================================[
+   NOTE for macos:
+   https://github.com/floodyberry/ed25519-donna/issues/29
+   our source for ed25519-donna-portable.h has been
+   patched to workaround this.
+#]=========================================================]
+target_link_libraries (ed25519-donna PUBLIC OpenSSL::SSL)
+add_library (NIH::ed25519-donna ALIAS ed25519-donna)
+target_link_libraries (ripple_libs INTERFACE NIH::ed25519-donna)
+#[===========================[
+   headers installation
+#]===========================]
+install (
+  FILES
+    src/ed25519-donna/ed25519.h
+  DESTINATION include/ed25519-donna)

Builds/CMake/deps/Findjemalloc.cmake

@@ -0,0 +1,47 @@
+# - Try to find jemalloc
+# Once done this will define
+#  JEMALLOC_FOUND - System has jemalloc
+#  JEMALLOC_INCLUDE_DIRS - The jemalloc include directories
+#  JEMALLOC_LIBRARIES - The libraries needed to use jemalloc
+
+if(NOT USE_BUNDLED_JEMALLOC)
+  find_package(PkgConfig)
+  if (PKG_CONFIG_FOUND)
+    pkg_check_modules(PC_JEMALLOC QUIET jemalloc)
+  endif()
+else()
+  set(PC_JEMALLOC_INCLUDEDIR)
+  set(PC_JEMALLOC_INCLUDE_DIRS)
+  set(PC_JEMALLOC_LIBDIR)
+  set(PC_JEMALLOC_LIBRARY_DIRS)
+  set(LIMIT_SEARCH NO_DEFAULT_PATH)
+endif()
+
+set(JEMALLOC_DEFINITIONS ${PC_JEMALLOC_CFLAGS_OTHER})
+
+find_path(JEMALLOC_INCLUDE_DIR jemalloc/jemalloc.h
+          PATHS ${PC_JEMALLOC_INCLUDEDIR} ${PC_JEMALLOC_INCLUDE_DIRS}
+          ${LIMIT_SEARCH})
+
+# If we're asked to use static linkage, add libjemalloc.a as a preferred library name.
+if(JEMALLOC_USE_STATIC)
+  list(APPEND JEMALLOC_NAMES
+    "${CMAKE_STATIC_LIBRARY_PREFIX}jemalloc${CMAKE_STATIC_LIBRARY_SUFFIX}")
+endif()
+
+list(APPEND JEMALLOC_NAMES jemalloc)
+
+find_library(JEMALLOC_LIBRARY NAMES ${JEMALLOC_NAMES}
+  HINTS ${PC_JEMALLOC_LIBDIR} ${PC_JEMALLOC_LIBRARY_DIRS}
+  ${LIMIT_SEARCH})
+
+set(JEMALLOC_LIBRARIES ${JEMALLOC_LIBRARY})
+set(JEMALLOC_INCLUDE_DIRS ${JEMALLOC_INCLUDE_DIR})
+
+include(FindPackageHandleStandardArgs)
+# handle the QUIETLY and REQUIRED arguments and set JEMALLOC_FOUND to TRUE
+# if all listed variables are TRUE
+find_package_handle_standard_args(JeMalloc DEFAULT_MSG
+  JEMALLOC_LIBRARY JEMALLOC_INCLUDE_DIR)
+
+mark_as_advanced(JEMALLOC_INCLUDE_DIR JEMALLOC_LIBRARY)

Builds/CMake/deps/Findlibarchive_pc.cmake

@@ -0,0 +1,22 @@
+find_package (PkgConfig REQUIRED)
+pkg_search_module (libarchive_PC QUIET libarchive>=3.4.3)
+
+if(static)
+  set(LIBARCHIVE_LIB libarchive.a)
+else()
+  set(LIBARCHIVE_LIB archive)
+endif()
+
+find_library (archive
+  NAMES ${LIBARCHIVE_LIB}
+  HINTS
+    ${libarchive_PC_LIBDIR}
+    ${libarchive_PC_LIBRARY_DIRS}
+  NO_DEFAULT_PATH)
+
+find_path (LIBARCHIVE_INCLUDE_DIR
+  NAMES archive.h
+  HINTS
+    ${libarchive_PC_INCLUDEDIR}
+    ${libarchive_PC_INCLUDEDIRS}
+  NO_DEFAULT_PATH)

Builds/CMake/deps/Findlz4.cmake

@@ -0,0 +1,24 @@
+find_package (PkgConfig)
+if (PKG_CONFIG_FOUND)
+  pkg_search_module (lz4_PC QUIET liblz4>=1.9)
+endif ()
+
+if(static)
+  set(LZ4_LIB liblz4.a)
+else()
+  set(LZ4_LIB lz4.so)
+endif()
+
+find_library (lz4
+  NAMES ${LZ4_LIB}
+  HINTS
+    ${lz4_PC_LIBDIR}
+    ${lz4_PC_LIBRARY_DIRS}
+  NO_DEFAULT_PATH)
+
+find_path (LZ4_INCLUDE_DIR
+  NAMES lz4.h
+  HINTS
+    ${lz4_PC_INCLUDEDIR}
+    ${lz4_PC_INCLUDEDIRS}
+  NO_DEFAULT_PATH)

Builds/CMake/deps/Findsecp256k1.cmake

@@ -0,0 +1,24 @@
+find_package (PkgConfig)
+if (PKG_CONFIG_FOUND)
+  pkg_search_module (secp256k1_PC QUIET libsecp256k1)
+endif ()
+
+if(static)
+  set(SECP256K1_LIB libsecp256k1.a)
+else()
+  set(SECP256K1_LIB secp256k1)
+endif()
+
+find_library(secp256k1
+  NAMES ${SECP256K1_LIB}
+  HINTS
+    ${secp256k1_PC_LIBDIR}
+    ${secp256k1_PC_LIBRARY_PATHS}
+  NO_DEFAULT_PATH)
+
+find_path (SECP256K1_INCLUDE_DIR
+  NAMES secp256k1.h
+  HINTS
+    ${secp256k1_PC_INCLUDEDIR}
+    ${secp256k1_PC_INCLUDEDIRS}
+  NO_DEFAULT_PATH)

Builds/CMake/deps/Findsnappy.cmake

@@ -0,0 +1,24 @@
+find_package (PkgConfig)
+if (PKG_CONFIG_FOUND)
+  pkg_search_module (snappy_PC QUIET snappy>=1.1.7)
+endif ()
+
+if(static)
+  set(SNAPPY_LIB libsnappy.a)
+else()
+  set(SNAPPY_LIB libsnappy.so)
+endif()
+
+find_library (snappy
+  NAMES ${SNAPPY_LIB}
+  HINTS
+    ${snappy_PC_LIBDIR}
+    ${snappy_PC_LIBRARY_DIRS}
+  NO_DEFAULT_PATH)
+
+find_path (SNAPPY_INCLUDE_DIR
+  NAMES snappy.h
+  HINTS
+    ${snappy_PC_INCLUDEDIR}
+    ${snappy_PC_INCLUDEDIRS}
+  NO_DEFAULT_PATH)

Builds/CMake/deps/Findsoci.cmake

@@ -0,0 +1,17 @@
+find_package (PkgConfig)
+if (PKG_CONFIG_FOUND)
+  # TBD - currently no soci pkgconfig
+  #pkg_search_module (soci_PC QUIET libsoci_core>=3.2)
+endif ()
+
+if(static)
+  set(SOCI_LIB libsoci.a)
+else()
+  set(SOCI_LIB libsoci_core.so)
+endif()
+
+find_library (soci
+  NAMES ${SOCI_LIB})
+
+find_path (SOCI_INCLUDE_DIR
+  NAMES soci/soci.h)

Builds/CMake/deps/Findsqlite.cmake

@@ -0,0 +1,24 @@
+find_package (PkgConfig)
+if (PKG_CONFIG_FOUND)
+  pkg_search_module (sqlite_PC QUIET sqlite3>=3.26.0)
+endif ()
+
+if(static)
+  set(SQLITE_LIB libsqlite3.a)
+else()
+  set(SQLITE_LIB sqlite3.so)
+endif()
+
+find_library (sqlite3
+  NAMES ${SQLITE_LIB}
+  HINTS
+    ${sqlite_PC_LIBDIR}
+    ${sqlite_PC_LIBRARY_DIRS}
+  NO_DEFAULT_PATH)
+
+find_path (SQLITE_INCLUDE_DIR
+  NAMES sqlite3.h
+  HINTS
+    ${sqlite_PC_INCLUDEDIR}
+    ${sqlite_PC_INCLUDEDIRS}
+  NO_DEFAULT_PATH)

Builds/CMake/deps/Libarchive.cmake

@@ -0,0 +1,163 @@
+#[===================================================================[
+   NIH dep: libarchive
+#]===================================================================]
+
+option (local_libarchive "use local build of libarchive." OFF)
+add_library (archive_lib UNKNOWN IMPORTED GLOBAL)
+
+if (NOT local_libarchive)
+  if (NOT WIN32)
+    find_package(libarchive_pc REQUIRED)
+  endif ()
+  if (archive)
+    message (STATUS "Found libarchive using pkg-config. Using ${archive}.")
+    set_target_properties (archive_lib PROPERTIES
+      IMPORTED_LOCATION_DEBUG
+        ${archive}
+      IMPORTED_LOCATION_RELEASE
+        ${archive}
+      INTERFACE_INCLUDE_DIRECTORIES
+        ${LIBARCHIVE_INCLUDE_DIR})
+      # pkg-config can return extra info for static lib linking
+      # this is probably needed/useful generally, but apply
+      # to APPLE for now (mostly for homebrew)
+      if (APPLE AND static AND libarchive_PC_STATIC_LIBRARIES)
+        message(STATUS "NOTE: libarchive static libs: ${libarchive_PC_STATIC_LIBRARIES}")
+        # also, APPLE seems to need iconv...maybe linux does too (TBD)
+        target_link_libraries (archive_lib
+          INTERFACE iconv ${libarchive_PC_STATIC_LIBRARIES})
+      endif ()
+  else ()
+    ## now try searching using the minimal find module that cmake provides
+    find_package(LibArchive 3.4.3 QUIET)
+    if (LibArchive_FOUND)
+      if (static)
+        # find module doesn't find static libs currently, so we re-search
+        get_filename_component(_loc ${LibArchive_LIBRARY} DIRECTORY)
+        find_library(_la_static
+          NAMES libarchive.a archive_static.lib archive.lib
+          PATHS ${_loc})
+        if (_la_static)
+          set (_la_lib ${_la_static})
+        else ()
+          message (WARNING "unable to find libarchive static lib - switching to local build")
+          set (local_libarchive ON CACHE BOOL "" FORCE)
+        endif ()
+      else ()
+        set (_la_lib ${LibArchive_LIBRARY})
+      endif ()
+      if (NOT local_libarchive)
+        message (STATUS "Found libarchive using module/config. Using ${_la_lib}.")
+        set_target_properties (archive_lib PROPERTIES
+          IMPORTED_LOCATION_DEBUG
+            ${_la_lib}
+          IMPORTED_LOCATION_RELEASE
+            ${_la_lib}
+          INTERFACE_INCLUDE_DIRECTORIES
+            ${LibArchive_INCLUDE_DIRS})
+      endif ()
+    else ()
+      set (local_libarchive ON CACHE BOOL "" FORCE)
+    endif ()
+  endif ()
+endif()
+
+if (local_libarchive)
+  set (lib_post "")
+  if (MSVC)
+    set (lib_post "_static")
+  endif ()
+  ExternalProject_Add (libarchive
+    PREFIX ${nih_cache_path}
+    GIT_REPOSITORY https://github.com/libarchive/libarchive.git
+    GIT_TAG v3.4.3
+    CMAKE_ARGS
+      # passing the compiler seems to be needed for windows CI, sadly
+      -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+      -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+      $<$<BOOL:${CMAKE_VERBOSE_MAKEFILE}>:-DCMAKE_VERBOSE_MAKEFILE=ON>
+      -DCMAKE_DEBUG_POSTFIX=_d
+      $<$<NOT:$<BOOL:${is_multiconfig}>>:-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}>
+      -DENABLE_LZ4=ON
+      -ULZ4_*
+      -DLZ4_INCLUDE_DIR=$<JOIN:$<TARGET_PROPERTY:lz4_lib,INTERFACE_INCLUDE_DIRECTORIES>,::>
+      # because we are building a static lib, this lz4 library doesn't
+      # actually matter since you can't generally link static libs to other static
+      # libs. The include files are needed, but the library itself is not (until
+      # we link our application, at which point we use the lz4 we built above).
+      # nonetheless, we need to provide a library to libarchive else it will
+      # NOT include lz4 support when configuring
+      -DLZ4_LIBRARY=$<IF:$<CONFIG:Debug>,$<TARGET_PROPERTY:lz4_lib,IMPORTED_LOCATION_DEBUG>,$<TARGET_PROPERTY:lz4_lib,IMPORTED_LOCATION_RELEASE>>
+      -DENABLE_WERROR=OFF
+      -DENABLE_TAR=OFF
+      -DENABLE_TAR_SHARED=OFF
+      -DENABLE_INSTALL=ON
+      -DENABLE_NETTLE=OFF
+      -DENABLE_OPENSSL=OFF
+      -DENABLE_LZO=OFF
+      -DENABLE_LZMA=OFF
+      -DENABLE_ZLIB=OFF
+      -DENABLE_BZip2=OFF
+      -DENABLE_LIBXML2=OFF
+      -DENABLE_EXPAT=OFF
+      -DENABLE_PCREPOSIX=OFF
+      -DENABLE_LibGCC=OFF
+      -DENABLE_CNG=OFF
+      -DENABLE_CPIO=OFF
+      -DENABLE_CPIO_SHARED=OFF
+      -DENABLE_CAT=OFF
+      -DENABLE_CAT_SHARED=OFF
+      -DENABLE_XATTR=OFF
+      -DENABLE_ACL=OFF
+      -DENABLE_ICONV=OFF
+      -DENABLE_TEST=OFF
+      -DENABLE_COVERAGE=OFF
+      $<$<BOOL:${MSVC}>:
+        "-DCMAKE_C_FLAGS=-GR -Gd -fp:precise -FS -MP"
+        "-DCMAKE_C_FLAGS_DEBUG=-MTd"
+        "-DCMAKE_C_FLAGS_RELEASE=-MT"
+      >
+    LIST_SEPARATOR ::
+    LOG_BUILD ON
+    LOG_CONFIGURE ON
+    BUILD_COMMAND
+      ${CMAKE_COMMAND}
+      --build .
+      --config $<CONFIG>
+      --target archive_static
+      $<$<VERSION_GREATER_EQUAL:${CMAKE_VERSION},3.12>:--parallel ${ep_procs}>
+      $<$<BOOL:${is_multiconfig}>:
+        COMMAND
+          ${CMAKE_COMMAND} -E copy
+            <BINARY_DIR>/libarchive/$<CONFIG>/${ep_lib_prefix}archive${lib_post}$<$<CONFIG:Debug>:_d>${ep_lib_suffix}
+            <BINARY_DIR>/libarchive
+        >
+    TEST_COMMAND ""
+    INSTALL_COMMAND ""
+    DEPENDS lz4_lib
+    BUILD_BYPRODUCTS
+      <BINARY_DIR>/libarchive/${ep_lib_prefix}archive${lib_post}${ep_lib_suffix}
+      <BINARY_DIR>/libarchive/${ep_lib_prefix}archive${lib_post}_d${ep_lib_suffix}
+  )
+  ExternalProject_Get_Property (libarchive BINARY_DIR)
+  ExternalProject_Get_Property (libarchive SOURCE_DIR)
+  if (CMAKE_VERBOSE_MAKEFILE)
+    print_ep_logs (libarchive)
+  endif ()
+  file (MAKE_DIRECTORY ${SOURCE_DIR}/libarchive)
+  set_target_properties (archive_lib PROPERTIES
+    IMPORTED_LOCATION_DEBUG
+      ${BINARY_DIR}/libarchive/${ep_lib_prefix}archive${lib_post}_d${ep_lib_suffix}
+    IMPORTED_LOCATION_RELEASE
+      ${BINARY_DIR}/libarchive/${ep_lib_prefix}archive${lib_post}${ep_lib_suffix}
+    INTERFACE_INCLUDE_DIRECTORIES
+      ${SOURCE_DIR}/libarchive
+    INTERFACE_COMPILE_DEFINITIONS
+      LIBARCHIVE_STATIC)
+endif()
+
+add_dependencies (archive_lib libarchive)
+target_link_libraries (archive_lib INTERFACE lz4_lib)
+target_link_libraries (ripple_libs INTERFACE archive_lib)
+exclude_if_included (libarchive)
+exclude_if_included (archive_lib)

Builds/CMake/deps/Lz4.cmake

@@ -0,0 +1,79 @@
+#[===================================================================[
+   NIH dep: lz4
+#]===================================================================]
+
+add_library (lz4_lib STATIC IMPORTED GLOBAL)
+
+if (NOT WIN32)
+  find_package(lz4)
+endif()
+
+if(lz4)
+  set_target_properties (lz4_lib PROPERTIES
+    IMPORTED_LOCATION_DEBUG
+      ${lz4}
+    IMPORTED_LOCATION_RELEASE
+      ${lz4}
+    INTERFACE_INCLUDE_DIRECTORIES
+      ${LZ4_INCLUDE_DIR})
+
+else()
+  ExternalProject_Add (lz4
+    PREFIX ${nih_cache_path}
+    GIT_REPOSITORY https://github.com/lz4/lz4.git
+    GIT_TAG v1.9.2
+    SOURCE_SUBDIR contrib/cmake_unofficial
+    CMAKE_ARGS
+      -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+      -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+      $<$<BOOL:${CMAKE_VERBOSE_MAKEFILE}>:-DCMAKE_VERBOSE_MAKEFILE=ON>
+      -DCMAKE_DEBUG_POSTFIX=_d
+      $<$<NOT:$<BOOL:${is_multiconfig}>>:-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}>
+      -DBUILD_STATIC_LIBS=ON
+      -DBUILD_SHARED_LIBS=OFF
+      $<$<BOOL:${MSVC}>:
+        "-DCMAKE_C_FLAGS=-GR -Gd -fp:precise -FS -MP"
+        "-DCMAKE_C_FLAGS_DEBUG=-MTd"
+        "-DCMAKE_C_FLAGS_RELEASE=-MT"
+      >
+    LOG_BUILD ON
+    LOG_CONFIGURE ON
+    BUILD_COMMAND
+      ${CMAKE_COMMAND}
+      --build .
+      --config $<CONFIG>
+      --target lz4_static
+      $<$<VERSION_GREATER_EQUAL:${CMAKE_VERSION},3.12>:--parallel ${ep_procs}>
+      $<$<BOOL:${is_multiconfig}>:
+        COMMAND
+          ${CMAKE_COMMAND} -E copy
+          <BINARY_DIR>/$<CONFIG>/${ep_lib_prefix}lz4$<$<CONFIG:Debug>:_d>${ep_lib_suffix}
+          <BINARY_DIR>
+        >
+    TEST_COMMAND ""
+    INSTALL_COMMAND ""
+    BUILD_BYPRODUCTS
+      <BINARY_DIR>/${ep_lib_prefix}lz4${ep_lib_suffix}
+      <BINARY_DIR>/${ep_lib_prefix}lz4_d${ep_lib_suffix}
+  )
+  ExternalProject_Get_Property (lz4 BINARY_DIR)
+  ExternalProject_Get_Property (lz4 SOURCE_DIR)
+
+  file (MAKE_DIRECTORY ${SOURCE_DIR}/lz4)
+  set_target_properties (lz4_lib PROPERTIES
+    IMPORTED_LOCATION_DEBUG
+      ${BINARY_DIR}/${ep_lib_prefix}lz4_d${ep_lib_suffix}
+    IMPORTED_LOCATION_RELEASE
+      ${BINARY_DIR}/${ep_lib_prefix}lz4${ep_lib_suffix}
+    INTERFACE_INCLUDE_DIRECTORIES
+      ${SOURCE_DIR}/lib)
+
+  if (CMAKE_VERBOSE_MAKEFILE)
+    print_ep_logs (lz4)
+  endif ()
+  add_dependencies (lz4_lib lz4)
+  target_link_libraries (ripple_libs INTERFACE lz4_lib)
+  exclude_if_included (lz4)
+endif()
+
+exclude_if_included (lz4_lib)

Builds/CMake/deps/Nudb.cmake

@@ -0,0 +1,47 @@
+#[===================================================================[
+   NIH dep: nudb
+
+   NuDB is header-only, thus is an INTERFACE lib in CMake.
+   TODO: move the library definition into NuDB repo and add
+   proper targets and export/install
+#]===================================================================]
+
+if (is_root_project) # NuDB not needed in the case of xrpl_core inclusion build
+  add_library (nudb INTERFACE)
+  if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.11)
+    FetchContent_Declare(
+      nudb_src
+      GIT_REPOSITORY https://github.com/CPPAlliance/NuDB.git
+      GIT_TAG        2.0.5
+    )
+    FetchContent_GetProperties(nudb_src)
+    if(NOT nudb_src_POPULATED)
+      message (STATUS "Pausing to download NuDB...")
+      FetchContent_Populate(nudb_src)
+    endif()
+  else ()
+    ExternalProject_Add (nudb_src
+      PREFIX ${nih_cache_path}
+      GIT_REPOSITORY https://github.com/CPPAlliance/NuDB.git
+      GIT_TAG 2.0.5
+      CONFIGURE_COMMAND ""
+      BUILD_COMMAND ""
+      TEST_COMMAND ""
+      INSTALL_COMMAND ""
+    )
+    ExternalProject_Get_Property (nudb_src SOURCE_DIR)
+    set (nudb_src_SOURCE_DIR "${SOURCE_DIR}")
+    file (MAKE_DIRECTORY ${nudb_src_SOURCE_DIR}/include)
+    add_dependencies (nudb nudb_src)
+  endif ()
+
+  file(TO_CMAKE_PATH "${nudb_src_SOURCE_DIR}" nudb_src_SOURCE_DIR)
+# specify as system includes so as to avoid warnings
+  target_include_directories (nudb SYSTEM INTERFACE ${nudb_src_SOURCE_DIR}/include)
+  target_link_libraries (nudb
+    INTERFACE
+      Boost::thread
+      Boost::system)
+  add_library (NIH::nudb ALIAS nudb)
+  target_link_libraries (ripple_libs INTERFACE NIH::nudb)
+endif ()

Builds/CMake/deps/OpenSSL.cmake

@@ -0,0 +1,48 @@
+#[===================================================================[
+   NIH dep: openssl
+#]===================================================================]
+
+#[===============================================[
+  OPENSSL_ROOT_DIR is the only variable that
+  FindOpenSSL honors for locating, so convert any
+  OPENSSL_ROOT vars to this
+#]===============================================]
+if (NOT DEFINED OPENSSL_ROOT_DIR)
+  if (DEFINED ENV{OPENSSL_ROOT})
+    set (OPENSSL_ROOT_DIR $ENV{OPENSSL_ROOT})
+  elseif (HOMEBREW)
+    execute_process (COMMAND ${HOMEBREW} --prefix openssl
+      OUTPUT_VARIABLE OPENSSL_ROOT_DIR
+      OUTPUT_STRIP_TRAILING_WHITESPACE)
+  endif ()
+  file (TO_CMAKE_PATH "${OPENSSL_ROOT_DIR}" OPENSSL_ROOT_DIR)
+endif ()
+
+if (static)
+  set (OPENSSL_USE_STATIC_LIBS ON)
+endif ()
+set (OPENSSL_MSVC_STATIC_RT ON)
+find_package (OpenSSL 1.1.1 REQUIRED)
+target_link_libraries (ripple_libs
+  INTERFACE
+    OpenSSL::SSL
+    OpenSSL::Crypto)
+# disable SSLv2...this can also be done when building/configuring OpenSSL
+set_target_properties(OpenSSL::SSL PROPERTIES
+    INTERFACE_COMPILE_DEFINITIONS OPENSSL_NO_SSL2)
+#[=========================================================[
+   https://gitlab.kitware.com/cmake/cmake/issues/16885
+   depending on how openssl is built, it might depend
+   on zlib. In fact, the openssl find package should
+   figure this out for us, but it does not currently...
+   so let's add zlib ourselves to the lib list
+   TODO: investigate linking to static zlib for static
+   build option
+#]=========================================================]
+find_package (ZLIB)
+set (has_zlib FALSE)
+if (TARGET ZLIB::ZLIB)
+  set_target_properties(OpenSSL::Crypto PROPERTIES
+    INTERFACE_LINK_LIBRARIES ZLIB::ZLIB)
+  set (has_zlib TRUE)
+endif ()

Builds/CMake/deps/Postgres.cmake

@@ -0,0 +1,70 @@
+if(reporting)
+    find_package(PostgreSQL)
+    if(NOT PostgreSQL_FOUND)
+        message("find_package did not find postgres")
+        find_library(postgres NAMES pq libpq libpq-dev pq-dev postgresql-devel)
+        find_path(libpq-fe NAMES libpq-fe.h PATH_SUFFIXES postgresql pgsql include)
+
+        if(NOT libpq-fe_FOUND OR NOT postgres_FOUND)
+            message("No system installed Postgres found. Will build")
+            add_library(postgres SHARED IMPORTED GLOBAL)
+            add_library(pgport SHARED IMPORTED GLOBAL)
+            add_library(pgcommon SHARED IMPORTED GLOBAL)
+            ExternalProject_Add(postgres_src
+                PREFIX ${nih_cache_path}
+                GIT_REPOSITORY https://github.com/postgres/postgres.git
+                GIT_TAG master
+                CONFIGURE_COMMAND ./configure --without-readline > /dev/null
+                BUILD_COMMAND ${CMAKE_COMMAND} -E env --unset=MAKELEVEL make
+                UPDATE_COMMAND ""
+                BUILD_IN_SOURCE 1
+                INSTALL_COMMAND ""
+                BUILD_BYPRODUCTS
+                    <BINARY_DIR>/src/interfaces/libpq/${ep_lib_prefix}pq.a
+                    <BINARY_DIR>/src/common/${ep_lib_prefix}pgcommon.a
+                    <BINARY_DIR>/src/port/${ep_lib_prefix}pgport.a
+                LOG_BUILD TRUE
+                )
+            ExternalProject_Get_Property (postgres_src SOURCE_DIR)
+            ExternalProject_Get_Property (postgres_src BINARY_DIR)
+
+            set (postgres_src_SOURCE_DIR "${SOURCE_DIR}")
+            file (MAKE_DIRECTORY ${postgres_src_SOURCE_DIR})
+            list(APPEND INCLUDE_DIRS
+                ${SOURCE_DIR}/src/include
+                ${SOURCE_DIR}/src/interfaces/libpq
+                )
+            set_target_properties(postgres PROPERTIES
+                IMPORTED_LOCATION
+                    ${BINARY_DIR}/src/interfaces/libpq/${ep_lib_prefix}pq.a
+                INTERFACE_INCLUDE_DIRECTORIES
+                    "${INCLUDE_DIRS}"
+                )
+            set_target_properties(pgcommon PROPERTIES
+                IMPORTED_LOCATION
+                    ${BINARY_DIR}/src/common/${ep_lib_prefix}pgcommon.a
+                INTERFACE_INCLUDE_DIRECTORIES
+                    "${INCLUDE_DIRS}"
+                )
+            set_target_properties(pgport PROPERTIES
+                IMPORTED_LOCATION
+                    ${BINARY_DIR}/src/port/${ep_lib_prefix}pgport.a
+                INTERFACE_INCLUDE_DIRECTORIES
+                    "${INCLUDE_DIRS}"
+                )
+            add_dependencies(postgres postgres_src)
+            add_dependencies(pgcommon postgres_src)
+            add_dependencies(pgport postgres_src)
+            file(TO_CMAKE_PATH "${postgres_src_SOURCE_DIR}" postgres_src_SOURCE_DIR)
+            target_link_libraries(ripple_libs INTERFACE postgres pgcommon pgport)
+        else()
+            message("Found system installed Postgres via find_libary")
+            target_include_directories(ripple_libs INTERFACE ${libpq-fe})
+            target_link_libraries(ripple_libs INTERFACE ${postgres})
+        endif()
+    else()
+        message("Found system installed Postgres via find_package")
+        target_include_directories(ripple_libs INTERFACE ${PostgreSQL_INCLUDE_DIRS})
+        target_link_libraries(ripple_libs INTERFACE ${PostgreSQL_LIBRARIES})
+    endif()
+endif()

Builds/CMake/deps/Protobuf.cmake

@@ -1,22 +1,155 @@
-find_package(Protobuf 3.8)
+#[===================================================================[
+   import protobuf (lib and compiler) and create a lib
+   from our proto message definitions. If the system protobuf
+   is not found, fallback on EP to download and build a version
+   from official source.
+#]===================================================================]
 
-file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/proto_gen)
-set(ccbd ${CMAKE_CURRENT_BINARY_DIR})
-set(CMAKE_CURRENT_BINARY_DIR ${CMAKE_BINARY_DIR}/proto_gen)
-protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS src/ripple/proto/ripple.proto)
-set(CMAKE_CURRENT_BINARY_DIR ${ccbd})
+if (static)
+  set (Protobuf_USE_STATIC_LIBS ON)
+endif ()
+find_package (Protobuf 3.8)
+if (is_multiconfig)
+    set(protobuf_protoc_lib ${Protobuf_PROTOC_LIBRARIES})
+else ()
+    string(TOUPPER ${CMAKE_BUILD_TYPE} upper_cmake_build_type)
+    set(protobuf_protoc_lib ${Protobuf_PROTOC_LIBRARY_${upper_cmake_build_type}})
+endif ()
+if (local_protobuf OR NOT (Protobuf_FOUND AND Protobuf_PROTOC_EXECUTABLE AND protobuf_protoc_lib))
+  include (GNUInstallDirs)
+  message (STATUS "using local protobuf build.")
+  set(protobuf_reqs Protobuf_PROTOC_EXECUTABLE protobuf_protoc_lib)
+  foreach(lib ${protobuf_reqs})
+    if(NOT ${lib})
+      message(STATUS "Couldn't find ${lib}")
+    endif()
+  endforeach()
+  if (WIN32)
+    # protobuf prepends lib even on windows
+    set (pbuf_lib_pre "lib")
+  else ()
+    set (pbuf_lib_pre ${ep_lib_prefix})
+  endif ()
+  # for the external project build of protobuf, we currently ignore the
+  # static option and always build static libs here. This is consistent
+  # with our other EP builds. Dynamic libs in an EP would add complexity
+  # because we'd need to get them into the runtime path, and probably
+  # install them.
+  ExternalProject_Add (protobuf_src
+    PREFIX ${nih_cache_path}
+    GIT_REPOSITORY https://github.com/protocolbuffers/protobuf.git
+    GIT_TAG v3.8.0
+    SOURCE_SUBDIR cmake
+    CMAKE_ARGS
+      -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+      -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+      -DCMAKE_INSTALL_PREFIX=<BINARY_DIR>/_installed_
+      -Dprotobuf_BUILD_TESTS=OFF
+      -Dprotobuf_BUILD_EXAMPLES=OFF
+      -Dprotobuf_BUILD_PROTOC_BINARIES=ON
+      -Dprotobuf_MSVC_STATIC_RUNTIME=ON
+      -DBUILD_SHARED_LIBS=OFF
+      -Dprotobuf_BUILD_SHARED_LIBS=OFF
+      -DCMAKE_DEBUG_POSTFIX=_d
+      -Dprotobuf_DEBUG_POSTFIX=_d
+      -Dprotobuf_WITH_ZLIB=$<IF:$<BOOL:${has_zlib}>,ON,OFF>
+      $<$<BOOL:${CMAKE_VERBOSE_MAKEFILE}>:-DCMAKE_VERBOSE_MAKEFILE=ON>
+      $<$<BOOL:${unity}>:-DCMAKE_UNITY_BUILD=ON}>
+      $<$<NOT:$<BOOL:${is_multiconfig}>>:-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}>
+      $<$<BOOL:${MSVC}>:
+	"-DCMAKE_CXX_FLAGS=-GR -Gd -fp:precise -FS -EHa -MP"
+      >
+    LOG_BUILD ON
+    LOG_CONFIGURE ON
+    BUILD_COMMAND
+      ${CMAKE_COMMAND}
+      --build .
+      --config $<CONFIG>
+      $<$<VERSION_GREATER_EQUAL:${CMAKE_VERSION},3.12>:--parallel ${ep_procs}>
+    TEST_COMMAND ""
+    INSTALL_COMMAND
+      ${CMAKE_COMMAND} -E env --unset=DESTDIR ${CMAKE_COMMAND} --build . --config $<CONFIG> --target install
+    BUILD_BYPRODUCTS
+      <BINARY_DIR>/_installed_/${CMAKE_INSTALL_LIBDIR}/${pbuf_lib_pre}protobuf${ep_lib_suffix}
+      <BINARY_DIR>/_installed_/${CMAKE_INSTALL_LIBDIR}/${pbuf_lib_pre}protobuf_d${ep_lib_suffix}
+      <BINARY_DIR>/_installed_/${CMAKE_INSTALL_LIBDIR}/${pbuf_lib_pre}protoc${ep_lib_suffix}
+      <BINARY_DIR>/_installed_/${CMAKE_INSTALL_LIBDIR}/${pbuf_lib_pre}protoc_d${ep_lib_suffix}
+      <BINARY_DIR>/_installed_/bin/protoc${CMAKE_EXECUTABLE_SUFFIX}
+  )
+  ExternalProject_Get_Property (protobuf_src BINARY_DIR)
+  ExternalProject_Get_Property (protobuf_src SOURCE_DIR)
+  if (CMAKE_VERBOSE_MAKEFILE)
+    print_ep_logs (protobuf_src)
+  endif ()
+  exclude_if_included (protobuf_src)
 
-add_library(pbufs STATIC ${PROTO_SRCS} ${PROTO_HDRS})
-target_include_directories(pbufs SYSTEM PUBLIC
-  ${CMAKE_BINARY_DIR}/proto_gen
-  ${CMAKE_BINARY_DIR}/proto_gen/src/ripple/proto
-)
-target_link_libraries(pbufs protobuf::libprotobuf)
-target_compile_options(pbufs
+  if (NOT TARGET protobuf::libprotobuf)
+    add_library (protobuf::libprotobuf STATIC IMPORTED GLOBAL)
+  endif ()
+  file (MAKE_DIRECTORY ${BINARY_DIR}/_installed_/include)
+  set_target_properties (protobuf::libprotobuf PROPERTIES
+    IMPORTED_LOCATION_DEBUG
+      ${BINARY_DIR}/_installed_/${CMAKE_INSTALL_LIBDIR}/${pbuf_lib_pre}protobuf_d${ep_lib_suffix}
+    IMPORTED_LOCATION_RELEASE
+      ${BINARY_DIR}/_installed_/${CMAKE_INSTALL_LIBDIR}/${pbuf_lib_pre}protobuf${ep_lib_suffix}
+    INTERFACE_INCLUDE_DIRECTORIES
+      ${BINARY_DIR}/_installed_/include)
+  add_dependencies (protobuf::libprotobuf protobuf_src)
+  exclude_if_included (protobuf::libprotobuf)
+
+  if (NOT TARGET protobuf::libprotoc)
+    add_library (protobuf::libprotoc STATIC IMPORTED GLOBAL)
+  endif ()
+  set_target_properties (protobuf::libprotoc PROPERTIES
+    IMPORTED_LOCATION_DEBUG
+      ${BINARY_DIR}/_installed_/${CMAKE_INSTALL_LIBDIR}/${pbuf_lib_pre}protoc_d${ep_lib_suffix}
+    IMPORTED_LOCATION_RELEASE
+      ${BINARY_DIR}/_installed_/${CMAKE_INSTALL_LIBDIR}/${pbuf_lib_pre}protoc${ep_lib_suffix}
+    INTERFACE_INCLUDE_DIRECTORIES
+      ${BINARY_DIR}/_installed_/include)
+  add_dependencies (protobuf::libprotoc protobuf_src)
+  exclude_if_included (protobuf::libprotoc)
+
+  if (NOT TARGET protobuf::protoc)
+    add_executable (protobuf::protoc IMPORTED)
+    exclude_if_included (protobuf::protoc)
+  endif ()
+  set_target_properties (protobuf::protoc PROPERTIES
+    IMPORTED_LOCATION "${BINARY_DIR}/_installed_/bin/protoc${CMAKE_EXECUTABLE_SUFFIX}")
+  add_dependencies (protobuf::protoc protobuf_src)
+else ()
+  if (NOT TARGET protobuf::protoc)
+    if (EXISTS "${Protobuf_PROTOC_EXECUTABLE}")
+      add_executable (protobuf::protoc IMPORTED)
+      set_target_properties (protobuf::protoc PROPERTIES
+        IMPORTED_LOCATION "${Protobuf_PROTOC_EXECUTABLE}")
+    else ()
+      message (FATAL_ERROR "Protobuf import failed")
+    endif ()
+  endif ()
+endif ()
+
+file (MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/proto_gen)
+set (save_CBD ${CMAKE_CURRENT_BINARY_DIR})
+set (CMAKE_CURRENT_BINARY_DIR ${CMAKE_BINARY_DIR}/proto_gen)
+protobuf_generate_cpp (
+  PROTO_SRCS
+  PROTO_HDRS
+  src/ripple/proto/ripple.proto)
+set (CMAKE_CURRENT_BINARY_DIR ${save_CBD})
+
+add_library (pbufs STATIC ${PROTO_SRCS} ${PROTO_HDRS})
+
+target_include_directories (pbufs PRIVATE src)
+target_include_directories (pbufs
+  SYSTEM PUBLIC ${CMAKE_BINARY_DIR}/proto_gen)
+target_link_libraries (pbufs protobuf::libprotobuf)
+target_compile_options (pbufs
   PUBLIC
-    $<$<BOOL:${XCODE}>:
+    $<$<BOOL:${is_xcode}>:
       --system-header-prefix="google/protobuf"
       -Wno-deprecated-dynamic-exception-spec
-    >
-)
-add_library(Ripple::pbufs ALIAS pbufs)
+    >)
+add_library (Ripple::pbufs ALIAS pbufs)
+target_link_libraries (ripple_libs INTERFACE Ripple::pbufs)
+exclude_if_included (pbufs)

Builds/CMake/deps/Rocksdb.cmake

@@ -0,0 +1,177 @@
+#[===================================================================[
+   NIH dep: rocksdb
+#]===================================================================]
+
+add_library (rocksdb_lib UNKNOWN IMPORTED GLOBAL)
+set_target_properties (rocksdb_lib
+  PROPERTIES INTERFACE_COMPILE_DEFINITIONS RIPPLE_ROCKSDB_AVAILABLE=1)
+
+option (local_rocksdb "use local build of rocksdb." OFF)
+if (NOT local_rocksdb)
+  find_package (RocksDB 6.27 QUIET CONFIG)
+  if (TARGET RocksDB::rocksdb)
+    message (STATUS "Found RocksDB using config.")
+    get_target_property (_rockslib_l RocksDB::rocksdb IMPORTED_LOCATION_DEBUG)
+    if (_rockslib_l)
+      set_target_properties (rocksdb_lib PROPERTIES IMPORTED_LOCATION_DEBUG ${_rockslib_l})
+    endif ()
+    get_target_property (_rockslib_l RocksDB::rocksdb IMPORTED_LOCATION_RELEASE)
+    if (_rockslib_l)
+      set_target_properties (rocksdb_lib PROPERTIES IMPORTED_LOCATION_RELEASE ${_rockslib_l})
+    endif ()
+    get_target_property (_rockslib_l RocksDB::rocksdb IMPORTED_LOCATION)
+    if (_rockslib_l)
+      set_target_properties (rocksdb_lib PROPERTIES IMPORTED_LOCATION ${_rockslib_l})
+    endif ()
+    get_target_property (_rockslib_i RocksDB::rocksdb INTERFACE_INCLUDE_DIRECTORIES)
+    if (_rockslib_i)
+      set_target_properties (rocksdb_lib PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${_rockslib_i})
+    endif ()
+    target_link_libraries (ripple_libs INTERFACE RocksDB::rocksdb)
+  else ()
+    # using a find module with rocksdb is difficult because
+    # you have no idea how it was configured (transitive dependencies).
+    # the code below will generally find rocksdb using the module, but
+    # will then result in linker errors for static linkage since the
+    # transitive dependencies are unknown. force local build here for now, but leave the code as
+    # a placeholder for future investigation.
+    if (static)
+      set (local_rocksdb ON CACHE BOOL "" FORCE)
+      # TBD if there is some way to extract transitive deps..then:
+      #set (RocksDB_USE_STATIC ON)
+    else ()
+      find_package (RocksDB 6.27 MODULE)
+      if (ROCKSDB_FOUND)
+        if (RocksDB_LIBRARY_DEBUG)
+          set_target_properties (rocksdb_lib PROPERTIES IMPORTED_LOCATION_DEBUG ${RocksDB_LIBRARY_DEBUG})
+        endif ()
+        set_target_properties (rocksdb_lib PROPERTIES IMPORTED_LOCATION_RELEASE ${RocksDB_LIBRARIES})
+        set_target_properties (rocksdb_lib PROPERTIES IMPORTED_LOCATION ${RocksDB_LIBRARIES})
+        set_target_properties (rocksdb_lib PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${RocksDB_INCLUDE_DIRS})
+      else ()
+        set (local_rocksdb ON CACHE BOOL "" FORCE)
+      endif ()
+    endif ()
+  endif ()
+endif ()
+
+if (local_rocksdb)
+  message (STATUS "Using local build of RocksDB.")
+  ExternalProject_Add (rocksdb
+    PREFIX ${nih_cache_path}
+    GIT_REPOSITORY https://github.com/facebook/rocksdb.git
+    GIT_TAG v6.27.3
+    PATCH_COMMAND
+      # only used by windows build
+      ${CMAKE_COMMAND} -E copy_if_different
+      ${CMAKE_CURRENT_SOURCE_DIR}/Builds/CMake/rocks_thirdparty.inc
+      <SOURCE_DIR>/thirdparty.inc
+    COMMAND
+      # fixup their build version file to keep the values
+      # from changing always
+      ${CMAKE_COMMAND} -E copy_if_different
+      ${CMAKE_CURRENT_SOURCE_DIR}/Builds/CMake/rocksdb_build_version.cc.in
+      <SOURCE_DIR>/util/build_version.cc.in
+    CMAKE_ARGS
+      -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+      -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+      $<$<BOOL:${CMAKE_VERBOSE_MAKEFILE}>:-DCMAKE_VERBOSE_MAKEFILE=ON>
+      $<$<BOOL:${unity}>:-DCMAKE_UNITY_BUILD=ON}>
+      -DCMAKE_DEBUG_POSTFIX=_d
+      $<$<NOT:$<BOOL:${is_multiconfig}>>:-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}>
+      -DBUILD_SHARED_LIBS=OFF
+      -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+      -DWITH_JEMALLOC=$<IF:$<BOOL:${jemalloc}>,ON,OFF>
+      -DWITH_SNAPPY=ON
+      -DWITH_LZ4=ON
+      -DWITH_ZLIB=OFF
+      -DUSE_RTTI=ON
+      -DWITH_ZSTD=OFF
+      -DWITH_GFLAGS=OFF
+      -DWITH_BZ2=OFF
+      -ULZ4_*
+      -Ulz4_*
+      -Dlz4_INCLUDE_DIRS=$<JOIN:$<TARGET_PROPERTY:lz4_lib,INTERFACE_INCLUDE_DIRECTORIES>,::>
+      -Dlz4_LIBRARIES=$<IF:$<CONFIG:Debug>,$<TARGET_PROPERTY:lz4_lib,IMPORTED_LOCATION_DEBUG>,$<TARGET_PROPERTY:lz4_lib,IMPORTED_LOCATION_RELEASE>>
+      -Dlz4_FOUND=ON
+      -USNAPPY_*
+      -Usnappy_*
+      -USnappy_*
+      -Dsnappy_INCLUDE_DIRS=$<JOIN:$<TARGET_PROPERTY:snappy_lib,INTERFACE_INCLUDE_DIRECTORIES>,::>
+      -Dsnappy_LIBRARIES=$<IF:$<CONFIG:Debug>,$<TARGET_PROPERTY:snappy_lib,IMPORTED_LOCATION_DEBUG>,$<TARGET_PROPERTY:snappy_lib,IMPORTED_LOCATION_RELEASE>>
+      -Dsnappy_FOUND=ON
+      -DSnappy_INCLUDE_DIRS=$<JOIN:$<TARGET_PROPERTY:snappy_lib,INTERFACE_INCLUDE_DIRECTORIES>,::>
+      -DSnappy_LIBRARIES=$<IF:$<CONFIG:Debug>,$<TARGET_PROPERTY:snappy_lib,IMPORTED_LOCATION_DEBUG>,$<TARGET_PROPERTY:snappy_lib,IMPORTED_LOCATION_RELEASE>>
+      -DSnappy_FOUND=ON
+      -DWITH_MD_LIBRARY=OFF
+      -DWITH_RUNTIME_DEBUG=$<IF:$<CONFIG:Debug>,ON,OFF>
+      -DFAIL_ON_WARNINGS=OFF
+      -DWITH_ASAN=OFF
+      -DWITH_TSAN=OFF
+      -DWITH_UBSAN=OFF
+      -DWITH_NUMA=OFF
+      -DWITH_TBB=OFF
+      -DWITH_WINDOWS_UTF8_FILENAMES=OFF
+      -DWITH_XPRESS=OFF
+      -DPORTABLE=ON
+      -DFORCE_SSE42=OFF
+      -DDISABLE_STALL_NOTIF=OFF
+      -DOPTDBG=ON
+      -DROCKSDB_LITE=OFF
+      -DWITH_FALLOCATE=ON
+      -DWITH_LIBRADOS=OFF
+      -DWITH_JNI=OFF
+      -DROCKSDB_INSTALL_ON_WINDOWS=OFF
+      -DWITH_TESTS=OFF
+      -DWITH_TOOLS=OFF
+      $<$<BOOL:${MSVC}>:
+        "-DCMAKE_CXX_FLAGS=-GR -Gd -fp:precise -FS -MP /DNDEBUG"
+      >
+      $<$<NOT:$<BOOL:${MSVC}>>:
+        "-DCMAKE_CXX_FLAGS=-DNDEBUG"
+      >
+    LOG_BUILD ON
+    LOG_CONFIGURE ON
+    BUILD_COMMAND
+      ${CMAKE_COMMAND}
+      --build .
+      --config $<CONFIG>
+      $<$<VERSION_GREATER_EQUAL:${CMAKE_VERSION},3.12>:--parallel ${ep_procs}>
+      $<$<BOOL:${is_multiconfig}>:
+        COMMAND
+          ${CMAKE_COMMAND} -E copy
+          <BINARY_DIR>/$<CONFIG>/${ep_lib_prefix}rocksdb$<$<CONFIG:Debug>:_d>${ep_lib_suffix}
+          <BINARY_DIR>
+        >
+    LIST_SEPARATOR ::
+    TEST_COMMAND ""
+    INSTALL_COMMAND ""
+    DEPENDS snappy_lib lz4_lib
+    BUILD_BYPRODUCTS
+      <BINARY_DIR>/${ep_lib_prefix}rocksdb${ep_lib_suffix}
+      <BINARY_DIR>/${ep_lib_prefix}rocksdb_d${ep_lib_suffix}
+  )
+  ExternalProject_Get_Property (rocksdb BINARY_DIR)
+  ExternalProject_Get_Property (rocksdb SOURCE_DIR)
+  if (CMAKE_VERBOSE_MAKEFILE)
+    print_ep_logs (rocksdb)
+  endif ()
+  file (MAKE_DIRECTORY ${SOURCE_DIR}/include)
+  set_target_properties (rocksdb_lib PROPERTIES
+    IMPORTED_LOCATION_DEBUG
+      ${BINARY_DIR}/${ep_lib_prefix}rocksdb_d${ep_lib_suffix}
+    IMPORTED_LOCATION_RELEASE
+      ${BINARY_DIR}/${ep_lib_prefix}rocksdb${ep_lib_suffix}
+    INTERFACE_INCLUDE_DIRECTORIES
+      ${SOURCE_DIR}/include)
+  add_dependencies (rocksdb_lib rocksdb)
+  exclude_if_included (rocksdb)
+endif ()
+
+target_link_libraries (rocksdb_lib
+  INTERFACE
+    snappy_lib
+    lz4_lib
+    $<$<BOOL:${MSVC}>:rpcrt4>)
+exclude_if_included (rocksdb_lib)
+target_link_libraries (ripple_libs INTERFACE rocksdb_lib)

Builds/CMake/deps/Secp256k1.cmake

@@ -0,0 +1,58 @@
+#[===================================================================[
+   NIH dep: secp256k1
+#]===================================================================]
+
+add_library (secp256k1_lib STATIC IMPORTED GLOBAL)
+
+if (NOT WIN32)
+  find_package(secp256k1)
+endif()
+
+if(secp256k1)
+  set_target_properties (secp256k1_lib PROPERTIES
+    IMPORTED_LOCATION_DEBUG
+      ${secp256k1}
+    IMPORTED_LOCATION_RELEASE
+      ${secp256k1}
+    INTERFACE_INCLUDE_DIRECTORIES
+      ${SECP256K1_INCLUDE_DIR})
+
+  add_library (secp256k1 ALIAS secp256k1_lib)
+  add_library (NIH::secp256k1 ALIAS secp256k1_lib)
+
+else()
+  set(INSTALL_SECP256K1 true)
+
+  add_library (secp256k1 STATIC
+    src/secp256k1/src/secp256k1.c)
+  target_compile_definitions (secp256k1
+    PRIVATE
+      USE_NUM_NONE
+      USE_FIELD_10X26
+      USE_FIELD_INV_BUILTIN
+      USE_SCALAR_8X32
+      USE_SCALAR_INV_BUILTIN)
+  target_include_directories (secp256k1
+    PUBLIC
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src>
+      $<INSTALL_INTERFACE:include>
+    PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/secp256k1)
+  target_compile_options (secp256k1
+    PRIVATE
+      $<$<BOOL:${MSVC}>:-wd4319>
+      $<$<NOT:$<BOOL:${MSVC}>>:
+        -Wno-deprecated-declarations
+        -Wno-unused-function
+      >
+      $<$<BOOL:${is_gcc}>:-Wno-nonnull-compare>)
+  target_link_libraries (ripple_libs INTERFACE NIH::secp256k1)
+#[===========================[
+     headers installation
+#]===========================]
+  install (
+    FILES
+      src/secp256k1/include/secp256k1.h
+  DESTINATION include/secp256k1/include)
+
+  add_library (NIH::secp256k1 ALIAS secp256k1)
+endif()

Builds/CMake/deps/Snappy.cmake

@@ -0,0 +1,77 @@
+#[===================================================================[
+   NIH dep: snappy
+#]===================================================================]
+
+add_library (snappy_lib STATIC IMPORTED GLOBAL)
+
+if (NOT WIN32)
+  find_package(snappy)
+endif()
+
+if(snappy)
+  set_target_properties (snappy_lib PROPERTIES
+    IMPORTED_LOCATION_DEBUG
+      ${snappy}
+    IMPORTED_LOCATION_RELEASE
+      ${snappy}
+    INTERFACE_INCLUDE_DIRECTORIES
+    ${SNAPPY_INCLUDE_DIR})
+
+else()
+  ExternalProject_Add (snappy
+    PREFIX ${nih_cache_path}
+    GIT_REPOSITORY https://github.com/google/snappy.git
+    GIT_TAG 1.1.7
+    CMAKE_ARGS
+      -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+      -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+      $<$<BOOL:${CMAKE_VERBOSE_MAKEFILE}>:-DCMAKE_VERBOSE_MAKEFILE=ON>
+      -DCMAKE_DEBUG_POSTFIX=_d
+      $<$<NOT:$<BOOL:${is_multiconfig}>>:-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}>
+      -DBUILD_SHARED_LIBS=OFF
+      -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+      -DSNAPPY_BUILD_TESTS=OFF
+      $<$<BOOL:${MSVC}>:
+        "-DCMAKE_CXX_FLAGS=-GR -Gd -fp:precise -FS -EHa -MP"
+        "-DCMAKE_CXX_FLAGS_DEBUG=-MTd"
+        "-DCMAKE_CXX_FLAGS_RELEASE=-MT"
+      >
+    LOG_BUILD ON
+    LOG_CONFIGURE ON
+    BUILD_COMMAND
+      ${CMAKE_COMMAND}
+      --build .
+      --config $<CONFIG>
+      $<$<VERSION_GREATER_EQUAL:${CMAKE_VERSION},3.12>:--parallel ${ep_procs}>
+      $<$<BOOL:${is_multiconfig}>:
+        COMMAND
+          ${CMAKE_COMMAND} -E copy
+          <BINARY_DIR>/$<CONFIG>/${ep_lib_prefix}snappy$<$<CONFIG:Debug>:_d>${ep_lib_suffix}
+          <BINARY_DIR>
+        >
+    TEST_COMMAND ""
+    INSTALL_COMMAND
+      ${CMAKE_COMMAND} -E copy_if_different <BINARY_DIR>/config.h <BINARY_DIR>/snappy-stubs-public.h <SOURCE_DIR>
+    BUILD_BYPRODUCTS
+      <BINARY_DIR>/${ep_lib_prefix}snappy${ep_lib_suffix}
+      <BINARY_DIR>/${ep_lib_prefix}snappy_d${ep_lib_suffix}
+  )
+  ExternalProject_Get_Property (snappy BINARY_DIR)
+  ExternalProject_Get_Property (snappy SOURCE_DIR)
+  if (CMAKE_VERBOSE_MAKEFILE)
+    print_ep_logs (snappy)
+  endif ()
+  file (MAKE_DIRECTORY ${SOURCE_DIR}/snappy)
+  set_target_properties (snappy_lib PROPERTIES
+    IMPORTED_LOCATION_DEBUG
+      ${BINARY_DIR}/${ep_lib_prefix}snappy_d${ep_lib_suffix}
+    IMPORTED_LOCATION_RELEASE
+      ${BINARY_DIR}/${ep_lib_prefix}snappy${ep_lib_suffix}
+    INTERFACE_INCLUDE_DIRECTORIES
+      ${SOURCE_DIR})
+endif()
+
+add_dependencies (snappy_lib snappy)
+target_link_libraries (ripple_libs INTERFACE snappy_lib)
+exclude_if_included (snappy)
+exclude_if_included (snappy_lib)

Builds/CMake/deps/Soci.cmake

@@ -0,0 +1,164 @@
+#[===================================================================[
+   NIH dep: soci
+#]===================================================================]
+
+foreach (_comp core empty sqlite3)
+  add_library ("soci_${_comp}" STATIC IMPORTED GLOBAL)
+endforeach ()
+
+if (NOT WIN32)
+  find_package(soci)
+endif()
+
+if (soci)
+  foreach (_comp core empty sqlite3)
+    set_target_properties ("soci_${_comp}" PROPERTIES
+      IMPORTED_LOCATION_DEBUG
+        ${soci}
+      IMPORTED_LOCATION_RELEASE
+        ${soci}
+      INTERFACE_INCLUDE_DIRECTORIES
+        ${SOCI_INCLUDE_DIR})
+  endforeach ()
+
+else()
+  set (soci_lib_pre ${ep_lib_prefix})
+  set (soci_lib_post "")
+  if (WIN32)
+    # for some reason soci on windows still prepends lib (non-standard)
+    set (soci_lib_pre lib)
+    # this version in the name might change if/when we change versions of soci
+    set (soci_lib_post "_4_0")
+  endif ()
+  get_target_property (_boost_incs Boost::date_time INTERFACE_INCLUDE_DIRECTORIES)
+  get_target_property (_boost_dt Boost::date_time IMPORTED_LOCATION)
+  if (NOT _boost_dt)
+    get_target_property (_boost_dt Boost::date_time IMPORTED_LOCATION_RELEASE)
+  endif ()
+  if (NOT _boost_dt)
+    get_target_property (_boost_dt Boost::date_time IMPORTED_LOCATION_DEBUG)
+  endif ()
+
+  ExternalProject_Add (soci
+    PREFIX ${nih_cache_path}
+    GIT_REPOSITORY https://github.com/SOCI/soci.git
+    GIT_TAG 04e1870294918d20761736743bb6136314c42dd5
+    # We had an issue with soci integer range checking for boost::optional
+    # and needed to remove the exception that SOCI throws in this case.
+    # This is *probably* a bug in SOCI, but has never been investigated more
+    # nor reported to the maintainers.
+    # This cmake script comments out the lines in question.
+    # This patch process is likely fragile and should be reviewed carefully
+    # whenever we update the GIT_TAG above.
+    PATCH_COMMAND
+      ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_SOURCE_DIR}/Builds/CMake/soci_patch.cmake
+    CMAKE_ARGS
+      -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+      -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+      $<$<BOOL:${CMAKE_VERBOSE_MAKEFILE}>:-DCMAKE_VERBOSE_MAKEFILE=ON>
+      $<$<BOOL:${CMAKE_TOOLCHAIN_FILE}>:-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}>
+      $<$<BOOL:${VCPKG_TARGET_TRIPLET}>:-DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET}>
+      $<$<BOOL:${unity}>:-DCMAKE_UNITY_BUILD=ON}>
+      -DCMAKE_PREFIX_PATH=${CMAKE_BINARY_DIR}/sqlite3
+      -DCMAKE_MODULE_PATH=${CMAKE_CURRENT_SOURCE_DIR}/Builds/CMake
+      -DCMAKE_INCLUDE_PATH=$<JOIN:$<TARGET_PROPERTY:sqlite,INTERFACE_INCLUDE_DIRECTORIES>,::>
+      -DCMAKE_LIBRARY_PATH=${sqlite_BINARY_DIR}
+      -DCMAKE_DEBUG_POSTFIX=_d
+      $<$<NOT:$<BOOL:${is_multiconfig}>>:-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}>
+      -DSOCI_CXX_C11=ON
+      -DSOCI_STATIC=ON
+      -DSOCI_LIBDIR=lib
+      -DSOCI_SHARED=OFF
+      -DSOCI_TESTS=OFF
+      # hacks to workaround the fact that soci doesn't currently use
+      # boost imported targets in its cmake. If they switch to
+      # proper imported targets, this next line can be removed
+      # (as well as the get_property above that sets _boost_incs)
+      -DBoost_INCLUDE_DIRS=$<JOIN:${_boost_incs},::>
+      -DBoost_INCLUDE_DIR=$<JOIN:${_boost_incs},::>
+      -DBOOST_ROOT=${BOOST_ROOT}
+      -DWITH_BOOST=ON
+      -DBoost_FOUND=ON
+      -DBoost_NO_BOOST_CMAKE=ON
+      -DBoost_DATE_TIME_FOUND=ON
+      -DSOCI_HAVE_BOOST=ON
+      -DSOCI_HAVE_BOOST_DATE_TIME=ON
+      -DBoost_DATE_TIME_LIBRARY=${_boost_dt}
+      -DSOCI_DB2=OFF
+      -DSOCI_FIREBIRD=OFF
+      -DSOCI_MYSQL=OFF
+      -DSOCI_ODBC=OFF
+      -DSOCI_ORACLE=OFF
+      -DSOCI_POSTGRESQL=OFF
+      -DSOCI_SQLITE3=ON
+      -DSQLITE3_INCLUDE_DIR=$<JOIN:$<TARGET_PROPERTY:sqlite,INTERFACE_INCLUDE_DIRECTORIES>,::>
+      -DSQLITE3_LIBRARY=$<IF:$<CONFIG:Debug>,$<TARGET_PROPERTY:sqlite,IMPORTED_LOCATION_DEBUG>,$<TARGET_PROPERTY:sqlite,IMPORTED_LOCATION_RELEASE>>
+      $<$<BOOL:${APPLE}>:-DCMAKE_FIND_FRAMEWORK=LAST>
+      $<$<BOOL:${MSVC}>:
+        "-DCMAKE_CXX_FLAGS=-GR -Gd -fp:precise -FS -EHa -MP"
+        "-DCMAKE_CXX_FLAGS_DEBUG=-MTd"
+        "-DCMAKE_CXX_FLAGS_RELEASE=-MT"
+      >
+      $<$<NOT:$<BOOL:${MSVC}>>:
+        "-DCMAKE_CXX_FLAGS=-Wno-deprecated-declarations"
+      >
+      # SEE: https://github.com/SOCI/soci/issues/640
+      $<$<AND:$<BOOL:${is_gcc}>,$<VERSION_GREATER_EQUAL:${CMAKE_CXX_COMPILER_VERSION},8>>:
+        "-DCMAKE_CXX_FLAGS=-Wno-deprecated-declarations -Wno-error=format-overflow -Wno-format-overflow -Wno-error=format-truncation"
+      >
+    LIST_SEPARATOR ::
+    LOG_BUILD ON
+    LOG_CONFIGURE ON
+    BUILD_COMMAND
+      ${CMAKE_COMMAND}
+      --build .
+      --config $<CONFIG>
+      $<$<VERSION_GREATER_EQUAL:${CMAKE_VERSION},3.12>:--parallel ${ep_procs}>
+      $<$<BOOL:${is_multiconfig}>:
+        COMMAND
+          ${CMAKE_COMMAND} -E copy
+          <BINARY_DIR>/lib/$<CONFIG>/${soci_lib_pre}soci_core${soci_lib_post}$<$<CONFIG:Debug>:_d>${ep_lib_suffix}
+          <BINARY_DIR>/lib/$<CONFIG>/${soci_lib_pre}soci_empty${soci_lib_post}$<$<CONFIG:Debug>:_d>${ep_lib_suffix}
+          <BINARY_DIR>/lib/$<CONFIG>/${soci_lib_pre}soci_sqlite3${soci_lib_post}$<$<CONFIG:Debug>:_d>${ep_lib_suffix}
+          <BINARY_DIR>/lib
+        >
+    TEST_COMMAND ""
+    INSTALL_COMMAND ""
+    DEPENDS sqlite
+    BUILD_BYPRODUCTS
+      <BINARY_DIR>/lib/${soci_lib_pre}soci_core${soci_lib_post}${ep_lib_suffix}
+      <BINARY_DIR>/lib/${soci_lib_pre}soci_core${soci_lib_post}_d${ep_lib_suffix}
+      <BINARY_DIR>/lib/${soci_lib_pre}soci_empty${soci_lib_post}${ep_lib_suffix}
+      <BINARY_DIR>/lib/${soci_lib_pre}soci_empty${soci_lib_post}_d${ep_lib_suffix}
+      <BINARY_DIR>/lib/${soci_lib_pre}soci_sqlite3${soci_lib_post}${ep_lib_suffix}
+      <BINARY_DIR>/lib/${soci_lib_pre}soci_sqlite3${soci_lib_post}_d${ep_lib_suffix}
+  )
+  ExternalProject_Get_Property (soci BINARY_DIR)
+  ExternalProject_Get_Property (soci SOURCE_DIR)
+  if (CMAKE_VERBOSE_MAKEFILE)
+    print_ep_logs (soci)
+  endif ()
+  file (MAKE_DIRECTORY ${SOURCE_DIR}/include)
+  file (MAKE_DIRECTORY ${BINARY_DIR}/include)
+  foreach (_comp core empty sqlite3)
+    set_target_properties ("soci_${_comp}" PROPERTIES
+      IMPORTED_LOCATION_DEBUG
+        ${BINARY_DIR}/lib/${soci_lib_pre}soci_${_comp}${soci_lib_post}_d${ep_lib_suffix}
+      IMPORTED_LOCATION_RELEASE
+        ${BINARY_DIR}/lib/${soci_lib_pre}soci_${_comp}${soci_lib_post}${ep_lib_suffix}
+      INTERFACE_INCLUDE_DIRECTORIES
+        "${SOURCE_DIR}/include;${BINARY_DIR}/include")
+    add_dependencies ("soci_${_comp}" soci) # something has to depend on the ExternalProject to trigger it
+    target_link_libraries (ripple_libs INTERFACE "soci_${_comp}")
+    if (NOT _comp STREQUAL "core")
+      target_link_libraries ("soci_${_comp}" INTERFACE soci_core)
+    endif ()
+  endforeach ()
+endif()
+
+foreach (_comp core empty sqlite3)
+  exclude_if_included ("soci_${_comp}")
+endforeach ()
+
+
+exclude_if_included (soci)

Builds/CMake/deps/Sqlite.cmake

@@ -0,0 +1,93 @@
+#[===================================================================[
+   NIH dep: sqlite
+#]===================================================================]
+
+add_library (sqlite STATIC IMPORTED GLOBAL)
+
+if (NOT WIN32)
+  find_package(sqlite)
+endif()
+
+
+if(sqlite3)
+  set_target_properties (sqlite PROPERTIES
+    IMPORTED_LOCATION_DEBUG
+      ${sqlite3}
+    IMPORTED_LOCATION_RELEASE
+      ${sqlite3}
+    INTERFACE_INCLUDE_DIRECTORIES
+    ${SQLITE_INCLUDE_DIR})
+
+else()
+  ExternalProject_Add (sqlite3
+    PREFIX ${nih_cache_path}
+    # sqlite doesn't use git, but it provides versioned tarballs
+    URL https://www.sqlite.org/2018/sqlite-amalgamation-3260000.zip
+      http://www.sqlite.org/2018/sqlite-amalgamation-3260000.zip
+      https://www2.sqlite.org/2018/sqlite-amalgamation-3260000.zip
+      http://www2.sqlite.org/2018/sqlite-amalgamation-3260000.zip
+    # ^^^ version is apparent in the URL:  3260000 => 3.26.0
+    URL_HASH SHA256=de5dcab133aa339a4cf9e97c40aa6062570086d6085d8f9ad7bc6ddf8a52096e
+    # Don't need to worry about MITM attacks too much because the download
+    # is checked against a strong hash
+    TLS_VERIFY false
+    # we wrote a very simple CMake file to build sqlite
+    # so that's what we copy here so that we can build with
+    # CMake. sqlite doesn't generally provided a build system
+    # for the single amalgamation source file.
+    PATCH_COMMAND
+      ${CMAKE_COMMAND} -E copy_if_different
+      ${CMAKE_CURRENT_SOURCE_DIR}/Builds/CMake/CMake_sqlite3.txt
+      <SOURCE_DIR>/CMakeLists.txt
+    CMAKE_ARGS
+      -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+      -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+      $<$<BOOL:${CMAKE_VERBOSE_MAKEFILE}>:-DCMAKE_VERBOSE_MAKEFILE=ON>
+      -DCMAKE_DEBUG_POSTFIX=_d
+      $<$<NOT:$<BOOL:${is_multiconfig}>>:-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}>
+      $<$<BOOL:${MSVC}>:
+        "-DCMAKE_C_FLAGS=-GR -Gd -fp:precise -FS -MP"
+        "-DCMAKE_C_FLAGS_DEBUG=-MTd"
+        "-DCMAKE_C_FLAGS_RELEASE=-MT"
+      >
+    LOG_BUILD ON
+    LOG_CONFIGURE ON
+    BUILD_COMMAND
+      ${CMAKE_COMMAND}
+      --build .
+      --config $<CONFIG>
+      $<$<VERSION_GREATER_EQUAL:${CMAKE_VERSION},3.12>:--parallel ${ep_procs}>
+      $<$<BOOL:${is_multiconfig}>:
+        COMMAND
+          ${CMAKE_COMMAND} -E copy
+            <BINARY_DIR>/$<CONFIG>/${ep_lib_prefix}sqlite3$<$<CONFIG:Debug>:_d>${ep_lib_suffix}
+            <BINARY_DIR>
+        >
+    TEST_COMMAND ""
+    INSTALL_COMMAND ""
+    BUILD_BYPRODUCTS
+      <BINARY_DIR>/${ep_lib_prefix}sqlite3${ep_lib_suffix}
+      <BINARY_DIR>/${ep_lib_prefix}sqlite3_d${ep_lib_suffix}
+  )
+  ExternalProject_Get_Property (sqlite3 BINARY_DIR)
+  ExternalProject_Get_Property (sqlite3 SOURCE_DIR)
+  if (CMAKE_VERBOSE_MAKEFILE)
+    print_ep_logs (sqlite3)
+  endif ()
+
+  set_target_properties (sqlite PROPERTIES
+    IMPORTED_LOCATION_DEBUG
+      ${BINARY_DIR}/${ep_lib_prefix}sqlite3_d${ep_lib_suffix}
+    IMPORTED_LOCATION_RELEASE
+      ${BINARY_DIR}/${ep_lib_prefix}sqlite3${ep_lib_suffix}
+    INTERFACE_INCLUDE_DIRECTORIES
+      ${SOURCE_DIR})
+
+  add_dependencies (sqlite sqlite3)
+  exclude_if_included (sqlite3)
+endif()
+
+target_link_libraries (sqlite INTERFACE $<$<NOT:$<BOOL:${MSVC}>>:dl>)
+target_link_libraries (ripple_libs INTERFACE sqlite)
+exclude_if_included (sqlite)
+set(sqlite_BINARY_DIR ${BINARY_DIR})

Builds/CMake/deps/cassandra.cmake

@@ -0,0 +1,165 @@
+if(reporting)
+    find_library(cassandra NAMES cassandra)
+    if(NOT cassandra)
+
+        message("System installed Cassandra cpp driver not found. Will build")
+
+        find_library(zlib NAMES zlib1g-dev zlib-devel zlib z)
+        if(NOT zlib)
+            message("zlib not found. will build")
+            add_library(zlib STATIC IMPORTED GLOBAL)
+            ExternalProject_Add(zlib_src
+                PREFIX ${nih_cache_path}
+                GIT_REPOSITORY https://github.com/madler/zlib.git
+                GIT_TAG master
+                INSTALL_COMMAND ""
+                BUILD_BYPRODUCTS <BINARY_DIR>/${ep_lib_prefix}z.a
+                LOG_BUILD TRUE
+                LOG_CONFIGURE TRUE
+                )
+
+
+            ExternalProject_Get_Property (zlib_src SOURCE_DIR)
+            ExternalProject_Get_Property (zlib_src BINARY_DIR)
+            set (zlib_src_SOURCE_DIR "${SOURCE_DIR}")
+            file (MAKE_DIRECTORY ${zlib_src_SOURCE_DIR}/include)
+
+            set_target_properties (zlib PROPERTIES
+                IMPORTED_LOCATION
+                ${BINARY_DIR}/${ep_lib_prefix}z.a
+                INTERFACE_INCLUDE_DIRECTORIES
+                ${SOURCE_DIR}/include)
+            add_dependencies(zlib zlib_src)
+
+            file(TO_CMAKE_PATH "${zlib_src_SOURCE_DIR}" zlib_src_SOURCE_DIR)
+        endif()
+
+
+
+
+        find_library(krb5 NAMES krb5-dev libkrb5-dev)
+
+        if(NOT krb5)
+            message("krb5 not found. will build")
+            add_library(krb5 STATIC IMPORTED GLOBAL)
+            ExternalProject_Add(krb5_src
+                PREFIX ${nih_cache_path}
+                GIT_REPOSITORY https://github.com/krb5/krb5.git
+                GIT_TAG master
+                UPDATE_COMMAND ""
+                CONFIGURE_COMMAND autoreconf src && CFLAGS=-fcommon ./src/configure --enable-static --disable-shared > /dev/null
+                BUILD_IN_SOURCE 1
+                BUILD_COMMAND make
+                INSTALL_COMMAND ""
+                BUILD_BYPRODUCTS <SOURCE_DIR>/lib/${ep_lib_prefix}krb5.a
+                LOG_BUILD TRUE
+                )
+
+            ExternalProject_Get_Property (krb5_src SOURCE_DIR)
+            ExternalProject_Get_Property (krb5_src BINARY_DIR)
+            set (krb5_src_SOURCE_DIR "${SOURCE_DIR}")
+            file (MAKE_DIRECTORY ${krb5_src_SOURCE_DIR}/include)
+
+            set_target_properties (krb5 PROPERTIES
+                IMPORTED_LOCATION
+                ${BINARY_DIR}/lib/${ep_lib_prefix}krb5.a
+                INTERFACE_INCLUDE_DIRECTORIES
+                ${SOURCE_DIR}/include)
+            add_dependencies(krb5 krb5_src)
+
+            file(TO_CMAKE_PATH "${krb5_src_SOURCE_DIR}" krb5_src_SOURCE_DIR)
+        endif()
+
+
+        find_library(libuv1 NAMES uv1 libuv1 liubuv1-dev libuv1:amd64)
+
+
+        if(NOT libuv1)
+            message("libuv1 not found, will build")
+            add_library(libuv1 STATIC IMPORTED GLOBAL)
+            ExternalProject_Add(libuv_src
+                PREFIX ${nih_cache_path}
+                GIT_REPOSITORY https://github.com/libuv/libuv.git
+                GIT_TAG v1.x
+                INSTALL_COMMAND ""
+                BUILD_BYPRODUCTS <BINARY_DIR>/${ep_lib_prefix}uv_a.a
+                LOG_BUILD TRUE
+                LOG_CONFIGURE TRUE
+                )
+
+            ExternalProject_Get_Property (libuv_src SOURCE_DIR)
+            ExternalProject_Get_Property (libuv_src BINARY_DIR)
+            set (libuv_src_SOURCE_DIR "${SOURCE_DIR}")
+            file (MAKE_DIRECTORY ${libuv_src_SOURCE_DIR}/include)
+
+            set_target_properties (libuv1 PROPERTIES
+                IMPORTED_LOCATION
+                ${BINARY_DIR}/${ep_lib_prefix}uv_a.a
+                INTERFACE_INCLUDE_DIRECTORIES
+                ${SOURCE_DIR}/include)
+            add_dependencies(libuv1 libuv_src)
+
+            file(TO_CMAKE_PATH "${libuv_src_SOURCE_DIR}" libuv_src_SOURCE_DIR)
+        endif()
+
+        add_library (cassandra STATIC IMPORTED GLOBAL)
+        ExternalProject_Add(cassandra_src
+            PREFIX ${nih_cache_path}
+            GIT_REPOSITORY https://github.com/datastax/cpp-driver.git
+            GIT_TAG master
+            CMAKE_ARGS
+            -DLIBUV_ROOT_DIR=${BINARY_DIR}
+            -DLIBUV_LIBARY=${BINARY_DIR}/libuv_a.a
+            -DLIBUV_INCLUDE_DIR=${SOURCE_DIR}/include
+            -DCASS_BUILD_STATIC=ON
+            INSTALL_COMMAND ""
+            BUILD_BYPRODUCTS <BINARY_DIR>/${ep_lib_prefix}cassandra_static.a
+            LOG_BUILD TRUE
+            LOG_CONFIGURE TRUE
+            )
+
+        ExternalProject_Get_Property (cassandra_src SOURCE_DIR)
+        ExternalProject_Get_Property (cassandra_src BINARY_DIR)
+        set (cassandra_src_SOURCE_DIR "${SOURCE_DIR}")
+        file (MAKE_DIRECTORY ${cassandra_src_SOURCE_DIR}/include)
+
+        set_target_properties (cassandra PROPERTIES
+            IMPORTED_LOCATION
+            ${BINARY_DIR}/${ep_lib_prefix}cassandra_static.a
+            INTERFACE_INCLUDE_DIRECTORIES
+            ${SOURCE_DIR}/include)
+        add_dependencies(cassandra cassandra_src)
+
+        if(NOT libuv1)
+            ExternalProject_Add_StepDependencies(cassandra_src build libuv1)
+            target_link_libraries(cassandra INTERFACE libuv1)
+        else()
+            target_link_libraries(cassandra INTERFACE ${libuv1})
+        endif()
+        if(NOT krb5)
+
+            ExternalProject_Add_StepDependencies(cassandra_src build krb5)
+            target_link_libraries(cassandra INTERFACE krb5)
+        else()
+            target_link_libraries(cassandra INTERFACE ${krb5})
+        endif()
+
+        if(NOT zlib)
+            ExternalProject_Add_StepDependencies(cassandra_src build zlib)
+            target_link_libraries(cassandra INTERFACE zlib)
+        else()
+            target_link_libraries(cassandra INTERFACE ${zlib})
+        endif()
+
+        file(TO_CMAKE_PATH "${cassandra_src_SOURCE_DIR}" cassandra_src_SOURCE_DIR)
+        target_link_libraries(ripple_libs INTERFACE cassandra)
+    else()
+        message("Found system installed cassandra cpp driver")
+
+        find_path(cassandra_includes NAMES cassandra.h REQUIRED)
+        target_link_libraries (ripple_libs INTERFACE ${cassandra})
+        target_include_directories(ripple_libs INTERFACE ${cassandra_includes})
+    endif()
+
+    exclude_if_included (cassandra)
+endif()

Builds/CMake/deps/date.cmake

@@ -0,0 +1,49 @@
+#[===================================================================[
+   NIH dep: date
+
+   the main library is header-only, thus is an INTERFACE lib in CMake.
+
+   NOTE: this has been accepted into c++20 so can likely be replaced
+   when we update to that standard
+#]===================================================================]
+
+find_package (date QUIET)
+if (NOT TARGET date::date)
+  if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.14)
+    FetchContent_Declare(
+      hh_date_src
+      GIT_REPOSITORY https://github.com/HowardHinnant/date.git
+      GIT_TAG        fc4cf092f9674f2670fb9177edcdee870399b829
+    )
+    FetchContent_MakeAvailable(hh_date_src)
+  else ()
+    ExternalProject_Add (hh_date_src
+      PREFIX ${nih_cache_path}
+      GIT_REPOSITORY https://github.com/HowardHinnant/date.git
+      GIT_TAG        fc4cf092f9674f2670fb9177edcdee870399b829
+      CONFIGURE_COMMAND ""
+      BUILD_COMMAND ""
+      TEST_COMMAND ""
+      INSTALL_COMMAND ""
+    )
+    ExternalProject_Get_Property (hh_date_src SOURCE_DIR)
+    set (hh_date_src_SOURCE_DIR "${SOURCE_DIR}")
+    file (MAKE_DIRECTORY ${hh_date_src_SOURCE_DIR}/include)
+    add_library (date_interface INTERFACE)
+    add_library (date::date ALIAS date_interface)
+    add_dependencies (date_interface hh_date_src)
+    file (TO_CMAKE_PATH "${hh_date_src_SOURCE_DIR}" hh_date_src_SOURCE_DIR)
+    target_include_directories (date_interface
+      SYSTEM INTERFACE
+          $<BUILD_INTERFACE:${hh_date_src_SOURCE_DIR}/include>
+          $<INSTALL_INTERFACE:include>)
+    install (
+      FILES
+        ${hh_date_src_SOURCE_DIR}/include/date/date.h
+      DESTINATION include/date)
+    install (TARGETS date_interface
+      EXPORT RippleExports
+      INCLUDES DESTINATION include)
+  endif ()
+endif ()
+

Builds/CMake/deps/gRPC.cmake

@@ -1,15 +1,319 @@
-find_package(gRPC 1.23)
+
+# currently linking to unsecure versions...if we switch, we'll
+# need to add ssl as a link dependency to the grpc targets
+option (use_secure_grpc "use TLS version of grpc libs." OFF)
+if (use_secure_grpc)
+  set (grpc_suffix "")
+else ()
+  set (grpc_suffix "_unsecure")
+endif ()
+
+find_package (gRPC 1.23 CONFIG QUIET)
+if (TARGET gRPC::gpr AND NOT local_grpc)
+  get_target_property (_grpc_l gRPC::gpr IMPORTED_LOCATION_DEBUG)
+  if (NOT _grpc_l)
+    get_target_property (_grpc_l gRPC::gpr IMPORTED_LOCATION_RELEASE)
+  endif ()
+  if (NOT _grpc_l)
+    get_target_property (_grpc_l gRPC::gpr IMPORTED_LOCATION)
+  endif ()
+  message (STATUS "Found cmake config for gRPC. Using ${_grpc_l}.")
+else ()
+  find_package (PkgConfig QUIET)
+  if (PKG_CONFIG_FOUND)
+    pkg_check_modules (grpc QUIET "grpc${grpc_suffix}>=1.25" "grpc++${grpc_suffix}" gpr)
+  endif ()
+
+  if (grpc_FOUND)
+    message (STATUS "Found gRPC using pkg-config. Using ${grpc_gpr_PREFIX}.")
+  endif ()
+
+  add_executable (gRPC::grpc_cpp_plugin IMPORTED)
+  exclude_if_included (gRPC::grpc_cpp_plugin)
+
+  if (grpc_FOUND AND NOT local_grpc)
+    # use installed grpc (via pkg-config)
+    macro (add_imported_grpc libname_)
+      if (static)
+        set (_search "${CMAKE_STATIC_LIBRARY_PREFIX}${libname_}${CMAKE_STATIC_LIBRARY_SUFFIX}")
+      else ()
+        set (_search "${CMAKE_SHARED_LIBRARY_PREFIX}${libname_}${CMAKE_SHARED_LIBRARY_SUFFIX}")
+      endif()
+      find_library(_found_${libname_}
+        NAMES ${_search}
+        HINTS ${grpc_LIBRARY_DIRS})
+      if (_found_${libname_})
+        message (STATUS "importing ${libname_} as ${_found_${libname_}}")
+      else ()
+        message (FATAL_ERROR "using pkg-config for grpc, can't find ${_search}")
+      endif ()
+      add_library ("gRPC::${libname_}" STATIC IMPORTED GLOBAL)
+      set_target_properties ("gRPC::${libname_}" PROPERTIES IMPORTED_LOCATION ${_found_${libname_}})
+      if (grpc_INCLUDE_DIRS)
+        set_target_properties ("gRPC::${libname_}" PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${grpc_INCLUDE_DIRS})
+      endif ()
+      target_link_libraries (ripple_libs INTERFACE "gRPC::${libname_}")
+      exclude_if_included ("gRPC::${libname_}")
+    endmacro ()
+
+    set_target_properties (gRPC::grpc_cpp_plugin PROPERTIES
+        IMPORTED_LOCATION "${grpc_gpr_PREFIX}/bin/grpc_cpp_plugin${CMAKE_EXECUTABLE_SUFFIX}")
+
+    pkg_check_modules (cares QUIET libcares)
+    if (cares_FOUND)
+      if (static)
+        set (_search "${CMAKE_STATIC_LIBRARY_PREFIX}cares${CMAKE_STATIC_LIBRARY_SUFFIX}")
+        set (_prefix cares_STATIC)
+        set (_static STATIC)
+      else ()
+        set (_search "${CMAKE_SHARED_LIBRARY_PREFIX}cares${CMAKE_SHARED_LIBRARY_SUFFIX}")
+        set (_prefix cares)
+        set (_static)
+      endif()
+      find_library(_location NAMES ${_search} HINTS ${cares_LIBRARY_DIRS})
+      if (NOT _location)
+        message (FATAL_ERROR "using pkg-config for grpc, can't find c-ares")
+      endif ()
+      add_library (c-ares::cares ${_static} IMPORTED GLOBAL)
+      set_target_properties (c-ares::cares PROPERTIES
+        IMPORTED_LOCATION ${_location}
+        INTERFACE_INCLUDE_DIRECTORIES "${${_prefix}_INCLUDE_DIRS}"
+        INTERFACE_LINK_OPTIONS "${${_prefix}_LDFLAGS}"
+      )
+      exclude_if_included (c-ares::cares)
+    else ()
+      message (FATAL_ERROR "using pkg-config for grpc, can't find c-ares")
+    endif ()
+  else ()
+    #[===========================[
+       c-ares (grpc requires)
+    #]===========================]
+    ExternalProject_Add (c-ares_src
+      PREFIX ${nih_cache_path}
+      GIT_REPOSITORY https://github.com/c-ares/c-ares.git
+      GIT_TAG cares-1_15_0
+      CMAKE_ARGS
+        -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+        $<$<BOOL:${CMAKE_VERBOSE_MAKEFILE}>:-DCMAKE_VERBOSE_MAKEFILE=ON>
+        -DCMAKE_DEBUG_POSTFIX=_d
+        $<$<NOT:$<BOOL:${is_multiconfig}>>:-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}>
+        -DCMAKE_INSTALL_PREFIX=<BINARY_DIR>/_installed_
+        -DCARES_SHARED=OFF
+        -DCARES_STATIC=ON
+        -DCARES_STATIC_PIC=ON
+        -DCARES_INSTALL=ON
+        -DCARES_MSVC_STATIC_RUNTIME=ON
+        $<$<BOOL:${MSVC}>:
+          "-DCMAKE_C_FLAGS=-GR -Gd -fp:precise -FS -MP"
+        >
+      LOG_BUILD ON
+      LOG_CONFIGURE ON
+      BUILD_COMMAND
+        ${CMAKE_COMMAND}
+          --build .
+          --config $<CONFIG>
+          $<$<VERSION_GREATER_EQUAL:${CMAKE_VERSION},3.12>:--parallel ${ep_procs}>
+      TEST_COMMAND ""
+      INSTALL_COMMAND
+        ${CMAKE_COMMAND} -E env --unset=DESTDIR ${CMAKE_COMMAND} --build . --config $<CONFIG> --target install
+      BUILD_BYPRODUCTS
+        <BINARY_DIR>/_installed_/lib/${ep_lib_prefix}cares${ep_lib_suffix}
+        <BINARY_DIR>/_installed_/lib/${ep_lib_prefix}cares_d${ep_lib_suffix}
+    )
+    exclude_if_included (c-ares_src)
+    ExternalProject_Get_Property (c-ares_src BINARY_DIR)
+    set (cares_binary_dir "${BINARY_DIR}")
+
+    add_library (c-ares::cares STATIC IMPORTED GLOBAL)
+    file (MAKE_DIRECTORY ${BINARY_DIR}/_installed_/include)
+    set_target_properties (c-ares::cares PROPERTIES
+      IMPORTED_LOCATION_DEBUG
+        ${BINARY_DIR}/_installed_/lib/${ep_lib_prefix}cares_d${ep_lib_suffix}
+      IMPORTED_LOCATION_RELEASE
+        ${BINARY_DIR}/_installed_/lib/${ep_lib_prefix}cares${ep_lib_suffix}
+      INTERFACE_INCLUDE_DIRECTORIES
+        ${BINARY_DIR}/_installed_/include)
+    add_dependencies (c-ares::cares c-ares_src)
+    exclude_if_included (c-ares::cares)
+
+    if (NOT has_zlib)
+      #[===========================[
+         zlib (grpc requires)
+      #]===========================]
+      if (MSVC)
+        set (zlib_debug_postfix "d") # zlib cmake sets this internally for MSVC, so we really don't have a choice
+        set (zlib_base "zlibstatic")
+      else ()
+        set (zlib_debug_postfix "_d")
+        set (zlib_base "z")
+      endif ()
+      ExternalProject_Add (zlib_src
+        PREFIX ${nih_cache_path}
+        GIT_REPOSITORY https://github.com/madler/zlib.git
+        GIT_TAG v1.2.11
+        CMAKE_ARGS
+          -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+          $<$<BOOL:${CMAKE_VERBOSE_MAKEFILE}>:-DCMAKE_VERBOSE_MAKEFILE=ON>
+          -DCMAKE_DEBUG_POSTFIX=${zlib_debug_postfix}
+          $<$<NOT:$<BOOL:${is_multiconfig}>>:-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}>
+          -DCMAKE_INSTALL_PREFIX=<BINARY_DIR>/_installed_
+          -DBUILD_SHARED_LIBS=OFF
+          $<$<BOOL:${MSVC}>:
+            "-DCMAKE_C_FLAGS=-GR -Gd -fp:precise -FS -MP"
+            "-DCMAKE_C_FLAGS_DEBUG=-MTd"
+            "-DCMAKE_C_FLAGS_RELEASE=-MT"
+          >
+        LOG_BUILD ON
+        LOG_CONFIGURE ON
+        BUILD_COMMAND
+          ${CMAKE_COMMAND}
+            --build .
+            --config $<CONFIG>
+            $<$<VERSION_GREATER_EQUAL:${CMAKE_VERSION},3.12>:--parallel ${ep_procs}>
+        TEST_COMMAND ""
+        INSTALL_COMMAND
+          ${CMAKE_COMMAND} -E env --unset=DESTDIR ${CMAKE_COMMAND} --build . --config $<CONFIG> --target install
+        BUILD_BYPRODUCTS
+          <BINARY_DIR>/_installed_/lib/${ep_lib_prefix}${zlib_base}${ep_lib_suffix}
+          <BINARY_DIR>/_installed_/lib/${ep_lib_prefix}${zlib_base}${zlib_debug_postfix}${ep_lib_suffix}
+      )
+      exclude_if_included (zlib_src)
+      ExternalProject_Get_Property (zlib_src BINARY_DIR)
+      set (zlib_binary_dir "${BINARY_DIR}")
+
+      add_library (ZLIB::ZLIB STATIC IMPORTED GLOBAL)
+      file (MAKE_DIRECTORY ${BINARY_DIR}/_installed_/include)
+      set_target_properties (ZLIB::ZLIB PROPERTIES
+        IMPORTED_LOCATION_DEBUG
+          ${BINARY_DIR}/_installed_/lib/${ep_lib_prefix}${zlib_base}${zlib_debug_postfix}${ep_lib_suffix}
+        IMPORTED_LOCATION_RELEASE
+          ${BINARY_DIR}/_installed_/lib/${ep_lib_prefix}${zlib_base}${ep_lib_suffix}
+        INTERFACE_INCLUDE_DIRECTORIES
+          ${BINARY_DIR}/_installed_/include)
+      add_dependencies (ZLIB::ZLIB zlib_src)
+      exclude_if_included (ZLIB::ZLIB)
+    endif ()
+
+    #[===========================[
+       grpc
+    #]===========================]
+    ExternalProject_Add (grpc_src
+      PREFIX ${nih_cache_path}
+      GIT_REPOSITORY https://github.com/grpc/grpc.git
+      GIT_TAG v1.25.0
+      CMAKE_ARGS
+        -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+        -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+        $<$<BOOL:${CMAKE_VERBOSE_MAKEFILE}>:-DCMAKE_VERBOSE_MAKEFILE=ON>
+        $<$<BOOL:${CMAKE_TOOLCHAIN_FILE}>:-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}>
+        $<$<BOOL:${VCPKG_TARGET_TRIPLET}>:-DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET}>
+        $<$<BOOL:${unity}>:-DCMAKE_UNITY_BUILD=ON}>
+        -DCMAKE_DEBUG_POSTFIX=_d
+        $<$<NOT:$<BOOL:${is_multiconfig}>>:-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}>
+        -DgRPC_BUILD_TESTS=OFF
+        -DgRPC_BENCHMARK_PROVIDER=""
+        -DgRPC_BUILD_CSHARP_EXT=OFF
+        -DgRPC_MSVC_STATIC_RUNTIME=ON
+        -DgRPC_INSTALL=OFF
+        -DgRPC_CARES_PROVIDER=package
+        -Dc-ares_DIR=${cares_binary_dir}/_installed_/lib/cmake/c-ares
+        -DgRPC_SSL_PROVIDER=package
+        -DOPENSSL_ROOT_DIR=${OPENSSL_ROOT_DIR}
+        -DgRPC_PROTOBUF_PROVIDER=package
+        -DProtobuf_USE_STATIC_LIBS=$<IF:$<AND:$<BOOL:${Protobuf_FOUND}>,$<NOT:$<BOOL:${static}>>>,OFF,ON>
+        -DProtobuf_INCLUDE_DIR=$<JOIN:$<TARGET_PROPERTY:protobuf::libprotobuf,INTERFACE_INCLUDE_DIRECTORIES>,:_:>
+        -DProtobuf_LIBRARY=$<IF:$<CONFIG:Debug>,$<TARGET_PROPERTY:protobuf::libprotobuf,IMPORTED_LOCATION_DEBUG>,$<TARGET_PROPERTY:protobuf::libprotobuf,IMPORTED_LOCATION_RELEASE>>
+        -DProtobuf_PROTOC_LIBRARY=$<IF:$<CONFIG:Debug>,$<TARGET_PROPERTY:protobuf::libprotoc,IMPORTED_LOCATION_DEBUG>,$<TARGET_PROPERTY:protobuf::libprotoc,IMPORTED_LOCATION_RELEASE>>
+        -DProtobuf_PROTOC_EXECUTABLE=$<TARGET_PROPERTY:protobuf::protoc,IMPORTED_LOCATION>
+        -DgRPC_ZLIB_PROVIDER=package
+        $<$<NOT:$<BOOL:${has_zlib}>>:-DZLIB_ROOT=${zlib_binary_dir}/_installed_>
+        $<$<BOOL:${MSVC}>:
+          "-DCMAKE_CXX_FLAGS=-GR -Gd -fp:precise -FS -EHa -MP"
+          "-DCMAKE_C_FLAGS=-GR -Gd -fp:precise -FS -MP"
+        >
+      LOG_BUILD ON
+      LOG_CONFIGURE ON
+      BUILD_COMMAND
+        ${CMAKE_COMMAND}
+        --build .
+        --config $<CONFIG>
+        $<$<VERSION_GREATER_EQUAL:${CMAKE_VERSION},3.12>:--parallel ${ep_procs}>
+        $<$<BOOL:${is_multiconfig}>:
+          COMMAND
+            ${CMAKE_COMMAND} -E copy
+            <BINARY_DIR>/$<CONFIG>/${ep_lib_prefix}grpc${grpc_suffix}$<$<CONFIG:Debug>:_d>${ep_lib_suffix}
+            <BINARY_DIR>/$<CONFIG>/${ep_lib_prefix}grpc++${grpc_suffix}$<$<CONFIG:Debug>:_d>${ep_lib_suffix}
+            <BINARY_DIR>/$<CONFIG>/${ep_lib_prefix}address_sorting$<$<CONFIG:Debug>:_d>${ep_lib_suffix}
+            <BINARY_DIR>/$<CONFIG>/${ep_lib_prefix}gpr$<$<CONFIG:Debug>:_d>${ep_lib_suffix}
+            <BINARY_DIR>/$<CONFIG>/grpc_cpp_plugin${CMAKE_EXECUTABLE_SUFFIX}
+            <BINARY_DIR>
+          >
+      LIST_SEPARATOR :_:
+      TEST_COMMAND ""
+      INSTALL_COMMAND ""
+      DEPENDS c-ares_src
+      BUILD_BYPRODUCTS
+        <BINARY_DIR>/${ep_lib_prefix}grpc${grpc_suffix}${ep_lib_suffix}
+        <BINARY_DIR>/${ep_lib_prefix}grpc${grpc_suffix}_d${ep_lib_suffix}
+        <BINARY_DIR>/${ep_lib_prefix}grpc++${grpc_suffix}${ep_lib_suffix}
+        <BINARY_DIR>/${ep_lib_prefix}grpc++${grpc_suffix}_d${ep_lib_suffix}
+        <BINARY_DIR>/${ep_lib_prefix}address_sorting${ep_lib_suffix}
+        <BINARY_DIR>/${ep_lib_prefix}address_sorting_d${ep_lib_suffix}
+        <BINARY_DIR>/${ep_lib_prefix}gpr${ep_lib_suffix}
+        <BINARY_DIR>/${ep_lib_prefix}gpr_d${ep_lib_suffix}
+        <BINARY_DIR>/grpc_cpp_plugin${CMAKE_EXECUTABLE_SUFFIX}
+    )
+    if (TARGET protobuf_src)
+      ExternalProject_Add_StepDependencies(grpc_src build protobuf_src)
+    endif ()
+    exclude_if_included (grpc_src)
+    ExternalProject_Get_Property (grpc_src BINARY_DIR)
+    ExternalProject_Get_Property (grpc_src SOURCE_DIR)
+    set (grpc_binary_dir "${BINARY_DIR}")
+    set (grpc_source_dir "${SOURCE_DIR}")
+    if (CMAKE_VERBOSE_MAKEFILE)
+      print_ep_logs (grpc_src)
+    endif ()
+    file (MAKE_DIRECTORY ${SOURCE_DIR}/include)
+
+    macro (add_imported_grpc libname_)
+      add_library ("gRPC::${libname_}" STATIC IMPORTED GLOBAL)
+      set_target_properties ("gRPC::${libname_}" PROPERTIES
+        IMPORTED_LOCATION_DEBUG
+          ${grpc_binary_dir}/${ep_lib_prefix}${libname_}_d${ep_lib_suffix}
+        IMPORTED_LOCATION_RELEASE
+          ${grpc_binary_dir}/${ep_lib_prefix}${libname_}${ep_lib_suffix}
+        INTERFACE_INCLUDE_DIRECTORIES
+          ${grpc_source_dir}/include)
+      add_dependencies ("gRPC::${libname_}" grpc_src)
+      target_link_libraries (ripple_libs INTERFACE "gRPC::${libname_}")
+      exclude_if_included ("gRPC::${libname_}")
+    endmacro ()
+
+    set_target_properties (gRPC::grpc_cpp_plugin PROPERTIES
+        IMPORTED_LOCATION "${grpc_binary_dir}/grpc_cpp_plugin${CMAKE_EXECUTABLE_SUFFIX}")
+    add_dependencies (gRPC::grpc_cpp_plugin grpc_src)
+  endif ()
+
+  add_imported_grpc (gpr)
+  add_imported_grpc ("grpc${grpc_suffix}")
+  add_imported_grpc ("grpc++${grpc_suffix}")
+  add_imported_grpc (address_sorting)
+
+  target_link_libraries ("gRPC::grpc${grpc_suffix}" INTERFACE c-ares::cares gRPC::gpr gRPC::address_sorting ZLIB::ZLIB)
+  target_link_libraries ("gRPC::grpc++${grpc_suffix}" INTERFACE "gRPC::grpc${grpc_suffix}" gRPC::gpr)
+endif ()
 
 #[=================================[
    generate protobuf sources for
    grpc defs and bundle into a
    static lib
 #]=================================]
-set(GRPC_GEN_DIR "${CMAKE_BINARY_DIR}/proto_gen_grpc")
-file(MAKE_DIRECTORY ${GRPC_GEN_DIR})
-set(GRPC_PROTO_SRCS)
-set(GRPC_PROTO_HDRS)
-set(GRPC_PROTO_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/src/ripple/proto/org")
+set (GRPC_GEN_DIR "${CMAKE_BINARY_DIR}/proto_gen_grpc")
+file (MAKE_DIRECTORY ${GRPC_GEN_DIR})
+set (GRPC_PROTO_SRCS)
+set (GRPC_PROTO_HDRS)
+set (GRPC_PROTO_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/src/ripple/proto/org")
 file(GLOB_RECURSE GRPC_DEFINITION_FILES LIST_DIRECTORIES false "${GRPC_PROTO_ROOT}/*.proto")
 foreach(file ${GRPC_DEFINITION_FILES})
   get_filename_component(_abs_file ${file} ABSOLUTE)
@@ -20,10 +324,10 @@ foreach(file ${GRPC_DEFINITION_FILES})
   get_filename_component(_rel_root_dir ${_rel_root_file} DIRECTORY)
   file(RELATIVE_PATH _rel_dir ${CMAKE_CURRENT_SOURCE_DIR} ${_abs_dir})
 
-  set(src_1 "${GRPC_GEN_DIR}/${_rel_root_dir}/${_basename}.grpc.pb.cc")
-  set(src_2 "${GRPC_GEN_DIR}/${_rel_root_dir}/${_basename}.pb.cc")
-  set(hdr_1 "${GRPC_GEN_DIR}/${_rel_root_dir}/${_basename}.grpc.pb.h")
-  set(hdr_2 "${GRPC_GEN_DIR}/${_rel_root_dir}/${_basename}.pb.h")
+  set (src_1 "${GRPC_GEN_DIR}/${_rel_root_dir}/${_basename}.grpc.pb.cc")
+  set (src_2 "${GRPC_GEN_DIR}/${_rel_root_dir}/${_basename}.pb.cc")
+  set (hdr_1 "${GRPC_GEN_DIR}/${_rel_root_dir}/${_basename}.grpc.pb.h")
+  set (hdr_2 "${GRPC_GEN_DIR}/${_rel_root_dir}/${_basename}.pb.h")
   add_custom_command(
     OUTPUT ${src_1} ${src_2} ${hdr_1} ${hdr_2}
     COMMAND protobuf::protoc
@@ -41,22 +345,20 @@ foreach(file ${GRPC_DEFINITION_FILES})
     list(APPEND GRPC_PROTO_HDRS ${hdr_1} ${hdr_2})
 endforeach()
 
-add_library(grpc_pbufs STATIC ${GRPC_PROTO_SRCS} ${GRPC_PROTO_HDRS})
-#target_include_directories(grpc_pbufs PRIVATE src)
-target_include_directories(grpc_pbufs SYSTEM PUBLIC ${GRPC_GEN_DIR})
-target_link_libraries(grpc_pbufs
-  "gRPC::grpc++"
-  # libgrpc is missing references.
-  absl::random_random
-)
-target_compile_options(grpc_pbufs
+add_library (grpc_pbufs STATIC ${GRPC_PROTO_SRCS} ${GRPC_PROTO_HDRS})
+#target_include_directories (grpc_pbufs PRIVATE src)
+target_include_directories (grpc_pbufs SYSTEM PUBLIC ${GRPC_GEN_DIR})
+target_link_libraries (grpc_pbufs protobuf::libprotobuf "gRPC::grpc++${grpc_suffix}")
+target_compile_options (grpc_pbufs
   PRIVATE
     $<$<BOOL:${MSVC}>:-wd4065>
     $<$<NOT:$<BOOL:${MSVC}>>:-Wno-deprecated-declarations>
   PUBLIC
     $<$<BOOL:${MSVC}>:-wd4996>
-    $<$<BOOL:${XCODE}>:
+    $<$<BOOL:${is_xcode}>:
       --system-header-prefix="google/protobuf"
       -Wno-deprecated-dynamic-exception-spec
     >)
-add_library(Ripple::grpc_pbufs ALIAS grpc_pbufs)
+add_library (Ripple::grpc_pbufs ALIAS grpc_pbufs)
+target_link_libraries (ripple_libs INTERFACE Ripple::grpc_pbufs)
+exclude_if_included (grpc_pbufs)

Builds/CMake/rocks_thirdparty.inc

@@ -0,0 +1,15 @@
+set (THIRDPARTY_LIBS "")
+
+if(WITH_SNAPPY)
+  add_definitions(-DSNAPPY)
+  include_directories(${snappy_INCLUDE_DIRS})
+  set (THIRDPARTY_LIBS ${THIRDPARTY_LIBS} ${snappy_LIBRARIES})
+endif()
+
+if(WITH_LZ4)
+  add_definitions(-DLZ4)
+  include_directories(${lz4_INCLUDE_DIRS})
+  set (THIRDPARTY_LIBS ${THIRDPARTY_LIBS} ${lz4_LIBRARIES})
+endif()
+
+

Builds/CMake/rocksdb_build_version.cc.in

@@ -0,0 +1,71 @@
+// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+
+#include <memory>
+
+#include "rocksdb/version.h"
+#include "util/string_util.h"
+
+// The build script may replace these values with real values based
+// on whether or not GIT is available and the platform settings
+static const std::string rocksdb_build_git_sha  = "rocksdb_build_git_sha:@GIT_SHA@";
+static const std::string rocksdb_build_git_tag = "rocksdb_build_git_tag:@GIT_TAG@";
+#define HAS_GIT_CHANGES @GIT_MOD@
+#if HAS_GIT_CHANGES == 0
+// If HAS_GIT_CHANGES is 0, the GIT date is used.
+// Use the time the branch/tag was last modified
+static const std::string rocksdb_build_date = "rocksdb_build_date:@GIT_DATE@";
+#else
+// If HAS_GIT_CHANGES is > 0, the branch/tag has modifications.
+// Use the time the build was created.
+static const std::string rocksdb_build_date = "rocksdb_build_date:@BUILD_DATE@";
+#endif
+
+namespace ROCKSDB_NAMESPACE {
+static void AddProperty(std::unordered_map<std::string, std::string> *props, const std::string& name) {
+  size_t colon = name.find(":");
+  if (colon != std::string::npos && colon > 0 && colon < name.length() - 1) {
+    // If we found a "@:", then this property was a build-time substitution that failed.  Skip it
+    size_t at = name.find("@", colon);
+    if (at != colon + 1) {
+      // Everything before the colon is the name, after is the value
+      (*props)[name.substr(0, colon)] = name.substr(colon + 1);
+    }
+  }
+}
+  
+static std::unordered_map<std::string, std::string>* LoadPropertiesSet() {
+  auto * properties = new std::unordered_map<std::string, std::string>();
+  AddProperty(properties, rocksdb_build_git_sha);
+  AddProperty(properties, rocksdb_build_git_tag);
+  AddProperty(properties, rocksdb_build_date);
+  return properties;
+}
+
+const std::unordered_map<std::string, std::string>& GetRocksBuildProperties() {
+  static std::unique_ptr<std::unordered_map<std::string, std::string>> props(LoadPropertiesSet());
+  return *props;
+}
+
+std::string GetRocksVersionAsString(bool with_patch) {
+  std::string version = ToString(ROCKSDB_MAJOR) + "." + ToString(ROCKSDB_MINOR);
+  if (with_patch) {
+    return version + "." + ToString(ROCKSDB_PATCH);
+  } else {
+    return version;
+  }
+}
+  
+std::string GetRocksBuildInfoAsString(const std::string& program, bool verbose) {
+  std::string info = program + " (RocksDB) " + GetRocksVersionAsString(true);
+  if (verbose) {
+    for (const auto& it : GetRocksBuildProperties()) {
+      info.append("\n    ");
+      info.append(it.first);
+      info.append(": ");
+      info.append(it.second);
+    }
+  }
+  return info;
+}
+} // namespace ROCKSDB_NAMESPACE
+

Builds/CMake/soci_patch.cmake

@@ -0,0 +1,39 @@
+# This patches unsigned-types.h in the soci official sources
+# so as to remove type range check exceptions that cause
+# us trouble when using boost::optional to select int values
+
+# Some versions of CMake erroneously patch external projects on every build.
+# If the patch makes no changes, skip it. This workaround can be
+# removed once we stop supporting vulnerable versions of CMake.
+# https://gitlab.kitware.com/cmake/cmake/-/issues/21086
+file (STRINGS include/soci/unsigned-types.h sourcecode)
+# Delete the .patched file if it exists, so it doesn't end up duplicated.
+# Trying to remove a file that does not exist is not a problem.
+file (REMOVE include/soci/unsigned-types.h.patched)
+foreach (line_ ${sourcecode})
+  if (line_ MATCHES "^[ \\t]+throw[ ]+soci_error[ ]*\\([ ]*\"Value outside of allowed.+$")
+    set (line_ "//${CMAKE_MATCH_0}")
+  endif ()
+  file (APPEND include/soci/unsigned-types.h.patched "${line_}\n")
+endforeach ()
+execute_process( COMMAND ${CMAKE_COMMAND} -E compare_files
+                 include/soci/unsigned-types.h include/soci/unsigned-types.h.patched
+                 RESULT_VARIABLE compare_result
+)
+if( compare_result EQUAL 0)
+  message(DEBUG "The soci source and patch files are identical. Make no changes.")
+  file (REMOVE include/soci/unsigned-types.h.patched)
+  return()
+endif()
+file (RENAME include/soci/unsigned-types.h include/soci/unsigned-types.h.orig)
+file (RENAME include/soci/unsigned-types.h.patched include/soci/unsigned-types.h)
+# also fix Boost.cmake so that it just returns when we override the Boost_FOUND var
+file (APPEND cmake/dependencies/Boost.cmake.patched "if (Boost_FOUND)\n")
+file (APPEND cmake/dependencies/Boost.cmake.patched "  return ()\n")
+file (APPEND cmake/dependencies/Boost.cmake.patched "endif ()\n")
+file (STRINGS cmake/dependencies/Boost.cmake sourcecode)
+foreach (line_ ${sourcecode})
+  file (APPEND cmake/dependencies/Boost.cmake.patched "${line_}\n")
+endforeach ()
+file (RENAME cmake/dependencies/Boost.cmake.patched cmake/dependencies/Boost.cmake)
+

Builds/VisualStudio2017/CMakeSettings-example.json

@@ -3,7 +3,7 @@
   "configurations": [
     {
       "name": "x64-Debug",
-      "generator": "Visual Studio 16 2019",
+      "generator": "Visual Studio 15 2017 Win64",
       "configurationType": "Debug",
       "inheritEnvironments": [ "msvc_x64_x64" ],
       "buildRoot": "${thisFileDir}\\build\\${name}",
@@ -23,7 +23,7 @@
     },
     {
       "name": "x64-Release",
-      "generator": "Visual Studio 16 2019",
+      "generator": "Visual Studio 15 2017 Win64",
       "configurationType": "Release",
       "inheritEnvironments": [ "msvc_x64_x64" ],
       "buildRoot": "${thisFileDir}\\build\\${name}",

Builds/VisualStudio2017/README.md

@@ -1 +1,263 @@
-[Build instructions are currently located in `BUILD.md`](../../BUILD.md)
+# Visual Studio 2017 Build Instructions
+
+## Important
+
+We do not recommend Windows for rippled production use at this time. Currently,
+the Ubuntu platform has received the highest level of quality assurance,
+testing, and support. Additionally, 32-bit Windows versions are not supported.
+
+## Prerequisites
+
+To clone the source code repository, create branches for inspection or
+modification, build rippled under Visual Studio, and run the unit tests you will
+need these software components
+
+| Component | Minimum Recommended Version |
+|-----------|-----------------------|
+| [Visual Studio 2017](README.md#install-visual-studio-2017)| 15.5.4 |
+| [Git for Windows](README.md#install-git-for-windows)| 2.16.1 |
+| [OpenSSL Library](README.md#install-openssl) | 1.1.1L |
+| [Boost library](README.md#build-boost) | 1.70.0 |
+| [CMake for Windows](README.md#optional-install-cmake-for-windows)* | 3.12 |
+
+\* Only needed if not using the integrated CMake in VS 2017 and prefer generating dedicated project/solution files.
+
+## Install Software
+
+### Install Visual Studio 2017
+
+If not already installed on your system, download your choice of installer from
+the [Visual Studio 2017
+Download](https://www.visualstudio.com/downloads/download-visual-studio-vs)
+page, run the installer, and follow the directions. **You may need to choose the
+`Desktop development with C++` workload to install all necessary C++ features.**
+
+Any version of Visual Studio 2017 may be used to build rippled. The **Visual
+Studio 2017 Community** edition is available free of charge (see [the product
+page](https://www.visualstudio.com/products/visual-studio-community-vs) for
+licensing details), while paid editions may be used for an initial free-trial
+period.
+
+### Install Git for Windows
+
+Git is a distributed revision control system. The Windows version also provides
+the bash shell and many Windows versions of Unix commands. While there are other
+varieties of Git (such as TortoiseGit, which has a native Windows interface and
+integrates with the Explorer shell), we recommend installing [Git for
+Windows](https://git-scm.com/) since it provides a Unix-like command line
+environment useful for running shell scripts. Use of the bash shell under
+Windows is mandatory for running the unit tests.
+
+### Install OpenSSL
+
+[Download the latest version of
+OpenSSL.](http://slproweb.com/products/Win32OpenSSL.html) There will
+several `Win64` bit variants available, you want the non-light
+`v1.1` line. As of this writing, you **should** select
+
+* Win64 OpenSSL v1.1.1L
+
+and should **not** select
+
+* Anything with "Win32" in the name
+* Anything with "light" in the name
+* Anything with "EXPERIMENTAL" in the name
+* Anything in the 3.0 line - rippled won't currently build with this version.
+
+Run the installer, and choose an appropriate location for your OpenSSL
+installation. In this guide we use `C:\lib\OpenSSL-Win64` as the destination
+location.
+
+You may be informed on running the installer that "Visual C++ 2008
+Redistributables" must first be installed first. If so, download it from the
+[same page](http://slproweb.com/products/Win32OpenSSL.html), again making sure
+to get the correct 32-/64-bit variant.
+
+* NOTE: Since rippled links statically to OpenSSL, it does not matter where the
+  OpenSSL .DLL files are placed, or what version they are. rippled does not use
+  or require any external .DLL files to run other than the standard operating
+  system ones.
+
+### Build Boost
+
+Boost 1.70 or later is required.
+
+After [downloading boost](http://www.boost.org/users/download/) and unpacking it
+to `c:\lib`. As of this writing, the most recent version of boost is 1.70.0,
+which will unpack into a directory named `boost_1_70_0`. We recommended either
+renaming this directory to `boost`, or creating a junction link `mklink /J boost
+boost_1_70_0`, so that you can more easily switch between versions.
+
+Next, open **Developer Command Prompt** and type the following commands
+
+```powershell
+cd C:\lib\boost
+bootstrap
+```
+
+The rippled application is linked statically to the standard runtimes and
+external dependencies on Windows, to ensure that the behavior of the executable
+is not affected by changes in outside files. Therefore, it is necessary to build
+the required boost static libraries using this command:
+
+```powershell
+bjam -j<Num Parallel> --toolset=msvc-14.1 address-model=64 architecture=x86 link=static threading=multi runtime-link=shared,static stage
+```
+
+where you should replace `<Num Parallel>` with the number of parallel
+invocations to use build, e.g. `bjam -j4 ...` would use up to 4 concurrent build
+shell commands for the build.
+
+Building the boost libraries may take considerable time. When the build process
+is completed, take note of both the reported compiler include paths and linker
+library paths as they will be required later.
+
+### (Optional) Install CMake for Windows
+
+[CMake](http://cmake.org) is a cross platform build system generator. Visual
+Studio 2017 includes an integrated version of CMake that avoids having to
+manually run CMake, but it is undergoing continuous improvement. Users that
+prefer to use standard Visual Studio project and solution files need to install
+a dedicated version of CMake to generate them.  The latest version can be found
+at the [CMake download site](https://cmake.org/download/). It is recommended you
+select the install option to add CMake to your path.
+
+## Clone the rippled repository
+
+If you are familiar with cloning github repositories, just follow your normal
+process and clone `git@github.com:ripple/rippled.git`. Otherwise follow this
+section for instructions.
+
+1. If you don't have a github account, sign up for one at
+   [github.com](https://github.com/).
+2. Make sure you have Github ssh keys. For help see
+   [generating-ssh-keys](https://help.github.com/articles/generating-ssh-keys).
+
+Open the "Git Bash" shell that was installed with "Git for Windows" in the step
+above. Navigate to the directory where you want to clone rippled (git bash uses
+`/c` for windows's `C:` and forward slash where windows uses backslash, so
+`C:\Users\joe\projs` would be `/c/Users/joe/projs` in git bash). Now clone the
+repository and optionally switch to the *master* branch. Type the following at
+the bash prompt:
+
+```powershell
+git clone git@github.com:ripple/rippled.git
+cd rippled
+```
+If you receive an error about not having the "correct access rights" make sure
+you have Github ssh keys, as described above.
+
+For a stable release, choose the `master` branch or one of the tagged releases
+listed on [rippled's GitHub page](https://github.com/ripple/rippled/releases).
+
+```
+git checkout master
+```
+
+To test the latest release candidate, choose the `release` branch.
+
+```
+git checkout release
+```
+
+If you are doing development work and want the latest set of untested features,
+you can consider using the `develop` branch instead.
+
+```
+git checkout develop
+```
+
+# Build using Visual Studio integrated CMake
+
+In Visual Studio 2017, Microsoft added [integrated IDE support for
+cmake](https://blogs.msdn.microsoft.com/vcblog/2016/10/05/cmake-support-in-visual-studio/).
+To begin, simply:
+
+1. Launch Visual Studio and choose **File | Open | Folder**, navigating to the
+   cloned rippled folder.
+2. Right-click on `CMakeLists.txt` in the **Solution Explorer - Folder View** to
+   generate a `CMakeSettings.json` file. A sample settings file is provided
+   [here](/Builds/VisualStudio2017/CMakeSettings-example.json). Customize the
+   settings for `BOOST_ROOT`, `OPENSSL_ROOT` to match the install paths if they
+   differ from those in the file.
+4. Select either the `x64-Release` or `x64-Debug` configuration from the
+   **Project Setings** drop-down. This should invoke the built-in CMake project
+   generator. If not, you can right-click on the `CMakeLists.txt` file and
+   choose **Cache | Generate Cache**.
+5. Select either the `rippled.exe` (unity) or `rippled_classic.exe` (non-unity)
+   option in the **Select Startup Item** drop-down. This will be the target
+   built when you press F7. Alternatively, you can choose a target to build from
+   the top-level **CMake | Build** menu. Note that at this time, there are other
+   targets listed that come from third party visual studio files embedded in the
+   rippled repo, e.g. `datagen.vcxproj`. Please ignore them.
+
+For details on configuring debugging sessions or further customization of CMake,
+please refer to the [CMake tools for VS
+documentation](https://docs.microsoft.com/en-us/cpp/ide/cmake-tools-for-visual-cpp).
+
+If using the provided `CMakeSettings.json` file, the executable will be in
+```
+.\build\x64-Release\Release\rippled.exe
+```
+or
+```
+.\build\x64-Debug\Debug\rippled.exe
+```
+These paths are relative to your cloned git repository.
+
+# Build using stand-alone CMake
+
+This requires having installed [CMake for
+Windows](README.md#optional-install-cmake-for-windows). We do not recommend
+mixing this method with the integrated CMake method for the same repository
+clone. Assuming you included the cmake executable folder in your path,
+execute the following commands within your `rippled` cloned repository:
+
+```
+mkdir build\cmake
+cd build\cmake
+cmake ..\.. -G"Visual Studio 15 2017 Win64" -DBOOST_ROOT="C:\lib\boost_1_70_0" -DOPENSSL_ROOT="C:\lib\OpenSSL-Win64" -DCMAKE_GENERATOR_TOOLSET=host=x64
+```
+Now launch Visual Studio 2017 and select **File | Open | Project/Solution**.
+Navigate to the `build\cmake` folder created above and select the `rippled.sln`
+file. You can then choose whether to build the `Debug` or `Release` solution
+configuration.
+
+The executable will be in
+```
+.\build\cmake\Release\rippled.exe
+```
+or
+```
+.\build\cmake\Debug\rippled.exe
+```
+These paths are relative to your cloned git repository.
+
+# Unity/No-Unity Builds
+
+The rippled build system defaults to using
+[unity source files](http://onqtam.com/programming/2018-07-07-unity-builds/)
+to improve build times. In some cases it might be desirable to disable the
+unity build and compile individual translation units. Here is how you can
+switch to a "no-unity" build configuration:
+
+## Visual Studio Integrated CMake
+
+Edit your `CmakeSettings.json` (described above) by adding `-Dunity=OFF`
+to the `cmakeCommandArgs` entry for each build configuration.
+
+## Standalone CMake Builds
+
+When running cmake to generate the Visual Studio project files, add
+`-Dunity=OFF` to the command line options passed to cmake.
+
+**Note:** you will need to re-run the cmake configuration step anytime you
+want to switch between unity/no-unity builds.
+
+# Unit Test (Recommended)
+
+`rippled` builds a set of unit tests into the server executable. To run these
+unit tests after building, pass the `--unittest` option to the compiled
+`rippled` executable. The executable will exit with summary info after running
+the unit tests.
+

Builds/VisualStudio2019/README.md

@@ -1,263 +0,0 @@
-# Visual Studio 2019 Build Instructions
-
-## Important
-
-We do not recommend Windows for rippled production use at this time. Currently,
-the Ubuntu platform has received the highest level of quality assurance,
-testing, and support. Additionally, 32-bit Windows versions are not supported.
-
-## Prerequisites
-
-To clone the source code repository, create branches for inspection or
-modification, build rippled under Visual Studio, and run the unit tests you will
-need these software components
-
-| Component | Minimum Recommended Version |
-|-----------|-----------------------|
-| [Visual Studio 2019](README.md#install-visual-studio-2019)| 15.5.4 |
-| [Git for Windows](README.md#install-git-for-windows)| 2.16.1 |
-| [OpenSSL Library](README.md#install-openssl) | 1.1.1L |
-| [Boost library](README.md#build-boost) | 1.70.0 |
-| [CMake for Windows](README.md#optional-install-cmake-for-windows)* | 3.12 |
-
-\* Only needed if not using the integrated CMake in VS 2019 and prefer generating dedicated project/solution files.
-
-## Install Software
-
-### Install Visual Studio 2019
-
-If not already installed on your system, download your choice of installer from
-the [Visual Studio 2019
-Download](https://www.visualstudio.com/downloads/download-visual-studio-vs)
-page, run the installer, and follow the directions. **You may need to choose the
-`Desktop development with C++` workload to install all necessary C++ features.**
-
-Any version of Visual Studio 2019 may be used to build rippled. The **Visual
-Studio 2019 Community** edition is available free of charge (see [the product
-page](https://www.visualstudio.com/products/visual-studio-community-vs) for
-licensing details), while paid editions may be used for an initial free-trial
-period.
-
-### Install Git for Windows
-
-Git is a distributed revision control system. The Windows version also provides
-the bash shell and many Windows versions of Unix commands. While there are other
-varieties of Git (such as TortoiseGit, which has a native Windows interface and
-integrates with the Explorer shell), we recommend installing [Git for
-Windows](https://git-scm.com/) since it provides a Unix-like command line
-environment useful for running shell scripts. Use of the bash shell under
-Windows is mandatory for running the unit tests.
-
-### Install OpenSSL
-
-[Download the latest version of
-OpenSSL.](http://slproweb.com/products/Win32OpenSSL.html) There will
-several `Win64` bit variants available, you want the non-light
-`v1.1` line. As of this writing, you **should** select
-
-* Win64 OpenSSL v1.1.1q
-
-and should **not** select
-
-* Anything with "Win32" in the name
-* Anything with "light" in the name
-* Anything with "EXPERIMENTAL" in the name
-* Anything in the 3.0 line - rippled won't currently build with this version.
-
-Run the installer, and choose an appropriate location for your OpenSSL
-installation. In this guide we use `C:\lib\OpenSSL-Win64` as the destination
-location.
-
-You may be informed on running the installer that "Visual C++ 2008
-Redistributables" must first be installed first. If so, download it from the
-[same page](http://slproweb.com/products/Win32OpenSSL.html), again making sure
-to get the correct 32-/64-bit variant.
-
-* NOTE: Since rippled links statically to OpenSSL, it does not matter where the
-  OpenSSL .DLL files are placed, or what version they are. rippled does not use
-  or require any external .DLL files to run other than the standard operating
-  system ones.
-
-### Build Boost
-
-Boost 1.70 or later is required.
-
-[Download boost](http://www.boost.org/users/download/) and unpack it
-to `c:\lib`. As of this writing, the most recent version of boost is 1.80.0,
-which will unpack into a directory named `boost_1_80_0`. We recommended either
-renaming this directory to `boost`, or creating a junction link `mklink /J boost
-boost_1_80_0`, so that you can more easily switch between versions.
-
-Next, open **Developer Command Prompt** and type the following commands
-
-```powershell
-cd C:\lib\boost
-bootstrap
-```
-
-The rippled application is linked statically to the standard runtimes and
-external dependencies on Windows, to ensure that the behavior of the executable
-is not affected by changes in outside files. Therefore, it is necessary to build
-the required boost static libraries using this command:
-
-```powershell
-b2 -j<Num Parallel> --toolset=msvc-14.2 address-model=64 architecture=x86 link=static threading=multi runtime-link=shared,static stage
-```
-
-where you should replace `<Num Parallel>` with the number of parallel
-invocations to use build, e.g. `bjam -j8 ...` would use up to 8 concurrent build
-shell commands for the build.
-
-Building the boost libraries may take considerable time. When the build process
-is completed, take note of both the reported compiler include paths and linker
-library paths as they will be required later.
-
-### (Optional) Install CMake for Windows
-
-[CMake](http://cmake.org) is a cross platform build system generator. Visual
-Studio 2019 includes an integrated version of CMake that avoids having to
-manually run CMake, but it is undergoing continuous improvement. Users that
-prefer to use standard Visual Studio project and solution files need to install
-a dedicated version of CMake to generate them.  The latest version can be found
-at the [CMake download site](https://cmake.org/download/). It is recommended you
-select the install option to add CMake to your path.
-
-## Clone the rippled repository
-
-If you are familiar with cloning github repositories, just follow your normal
-process and clone `git@github.com:ripple/rippled.git`. Otherwise follow this
-section for instructions.
-
-1. If you don't have a github account, sign up for one at
-   [github.com](https://github.com/).
-2. Make sure you have Github ssh keys. For help see
-   [generating-ssh-keys](https://help.github.com/articles/generating-ssh-keys).
-
-Open the "Git Bash" shell that was installed with "Git for Windows" in the step
-above. Navigate to the directory where you want to clone rippled (git bash uses
-`/c` for windows's `C:` and forward slash where windows uses backslash, so
-`C:\Users\joe\projs` would be `/c/Users/joe/projs` in git bash). Now clone the
-repository and optionally switch to the *master* branch. Type the following at
-the bash prompt:
-
-```powershell
-git clone git@github.com:XRPLF/rippled.git
-cd rippled
-```
-If you receive an error about not having the "correct access rights" make sure
-you have Github ssh keys, as described above.
-
-For a stable release, choose the `master` branch or one of the tagged releases
-listed on [rippled's GitHub page](https://github.com/ripple/rippled/releases).
-
-```
-git checkout master
-```
-
-To test the latest release candidate, choose the `release` branch.
-
-```
-git checkout release
-```
-
-If you are doing development work and want the latest set of beta features,
-you can consider using the `develop` branch instead.
-
-```
-git checkout develop
-```
-
-# Build using Visual Studio integrated CMake
-
-In Visual Studio 2017, Microsoft added [integrated IDE support for
-cmake](https://blogs.msdn.microsoft.com/vcblog/2016/10/05/cmake-support-in-visual-studio/).
-To begin, simply:
-
-1. Launch Visual Studio and choose **File | Open | Folder**, navigating to the
-   cloned rippled folder.
-2. Right-click on `CMakeLists.txt` in the **Solution Explorer - Folder View** to
-   generate a `CMakeSettings.json` file. A sample settings file is provided
-   [here](/Builds/VisualStudio2019/CMakeSettings-example.json). Customize the
-   settings for `BOOST_ROOT`, `OPENSSL_ROOT` to match the install paths if they
-   differ from those in the file.
-4. Select either the `x64-Release` or `x64-Debug` configuration from the
-   **Project Settings** drop-down. This should invoke the built-in CMake project
-   generator. If not, you can right-click on the `CMakeLists.txt` file and
-   choose **Configure rippled**.
-5. Select the `rippled.exe`
-   option in the **Select Startup Item** drop-down. This will be the target
-   built when you press F7. Alternatively, you can choose a target to build from
-   the top-level **CMake | Build** menu. Note that at this time, there are other
-   targets listed that come from third party visual studio files embedded in the
-   rippled repo, e.g. `datagen.vcxproj`. Please ignore them.
-
-For details on configuring debugging sessions or further customization of CMake,
-please refer to the [CMake tools for VS
-documentation](https://docs.microsoft.com/en-us/cpp/ide/cmake-tools-for-visual-cpp).
-
-If using the provided `CMakeSettings.json` file, the executable will be in
-```
-.\build\x64-Release\Release\rippled.exe
-```
-or
-```
-.\build\x64-Debug\Debug\rippled.exe
-```
-These paths are relative to your cloned git repository.
-
-# Build using stand-alone CMake
-
-This requires having installed [CMake for
-Windows](README.md#optional-install-cmake-for-windows). We do not recommend
-mixing this method with the integrated CMake method for the same repository
-clone. Assuming you included the cmake executable folder in your path,
-execute the following commands within your `rippled` cloned repository:
-
-```
-mkdir build\cmake
-cd build\cmake
-cmake ..\.. -G"Visual Studio 16 2019" -Ax64 -DBOOST_ROOT="C:\lib\boost" -DOPENSSL_ROOT="C:\lib\OpenSSL-Win64" -DCMAKE_GENERATOR_TOOLSET=host=x64
-```
-Now launch Visual Studio 2019 and select **File | Open | Project/Solution**.
-Navigate to the `build\cmake` folder created above and select the `rippled.sln`
-file. You can then choose whether to build the `Debug` or `Release` solution
-configuration.
-
-The executable will be in
-```
-.\build\cmake\Release\rippled.exe
-```
-or
-```
-.\build\cmake\Debug\rippled.exe
-```
-These paths are relative to your cloned git repository.
-
-# Unity/No-Unity Builds
-
-The rippled build system defaults to using
-[unity source files](http://onqtam.com/programming/2018-07-07-unity-builds/)
-to improve build times. In some cases it might be desirable to disable the
-unity build and compile individual translation units. Here is how you can
-switch to a "no-unity" build configuration:
-
-## Visual Studio Integrated CMake
-
-Edit your `CmakeSettings.json` (described above) by adding `-Dunity=OFF`
-to the `cmakeCommandArgs` entry for each build configuration.
-
-## Standalone CMake Builds
-
-When running cmake to generate the Visual Studio project files, add
-`-Dunity=OFF` to the command line options passed to cmake.
-
-**Note:** you will need to re-run the cmake configuration step anytime you
-want to switch between unity/no-unity builds.
-
-# Unit Test (Recommended)
-
-`rippled` builds a set of unit tests into the server executable. To run these
-unit tests after building, pass the `--unittest` option to the compiled
-`rippled` executable. The executable will exit with summary info after running
-the unit tests.
-

Builds/containers/centos-builder/Dockerfile

@@ -1,26 +1,43 @@
-FROM rippleci/centos:7
+FROM centos:7
 ARG GIT_COMMIT=unknown
 ARG CI_USE=false
 
 LABEL git-commit=$GIT_COMMIT
 
 COPY centos-builder/centos_setup.sh /tmp/
+COPY shared/build_deps.sh /tmp/
 COPY shared/install_cmake.sh /tmp/
+COPY centos-builder/extras.sh /tmp/
+COPY shared/install_boost.sh /tmp/
 RUN chmod +x /tmp/centos_setup.sh && \
-    chmod +x /tmp/install_cmake.sh
+    chmod +x /tmp/build_deps.sh && \
+    chmod +x /tmp/install_boost.sh && \
+    chmod +x /tmp/install_cmake.sh && \
+    chmod +x /tmp/extras.sh
 RUN /tmp/centos_setup.sh
 
-RUN /tmp/install_cmake.sh 3.16.3 /opt/local/cmake-3.16
+RUN /tmp/install_cmake.sh 3.16.1 /opt/local/cmake-3.16
 RUN ln -s /opt/local/cmake-3.16 /opt/local/cmake
 ENV PATH="/opt/local/cmake/bin:$PATH"
-# TODO: Install latest CMake for testing
-RUN if [ "${CI_USE}" = true ] ; then /tmp/install_cmake.sh 3.16.3 /opt/local/cmake-3.16; fi
+# also install min supported cmake for testing
+RUN if [ "${CI_USE}" = true ] ; then /tmp/install_cmake.sh 3.9.0 /opt/local/cmake-3.9; fi
 
+RUN source scl_source enable devtoolset-7 python27 && \
+     /tmp/build_deps.sh
+ENV BOOST_ROOT="/opt/local/boost/_INSTALLED_"
+ENV PLANTUML_JAR="/opt/plantuml/plantuml.jar"
+ENV OPENSSL_ROOT="/opt/local/openssl"
+ENV GDB_ROOT="/opt/local/gdb"
+RUN source scl_source enable devtoolset-7 python27 && \
+     /tmp/extras.sh
+
+# prep files for package building
 RUN mkdir -m 777 -p /opt/rippled_bld/pkg
-
 WORKDIR /opt/rippled_bld/pkg
 RUN mkdir -m 777 ./rpmbuild
 RUN mkdir -m 777 ./rpmbuild/{BUILD,RPMS,SOURCES,SPECS,SRPMS}
 
 COPY packaging/rpm/build_rpm.sh ./
 CMD ./build_rpm.sh
+
+

Builds/containers/centos-builder/centos_setup.sh

@@ -7,16 +7,31 @@ yum -y upgrade
 yum -y update
 yum -y install epel-release centos-release-scl
 yum -y install \
-    wget curl time gcc-c++ yum-utils autoconf automake pkgconfig libtool \
+    wget curl time gcc-c++ time yum-utils autoconf automake pkgconfig libtool \
     libstdc++-static rpm-build gnupg which make cmake \
-    devtoolset-11 devtoolset-11-gdb devtoolset-11-binutils devtoolset-11-libstdc++-devel \
-    devtoolset-11-libasan-devel devtoolset-11-libtsan-devel devtoolset-11-libubsan-devel devtoolset-11-liblsan-devel \
+    devtoolset-7 devtoolset-7-gdb devtoolset-7-libasan-devel devtoolset-7-libtsan-devel devtoolset-7-libubsan-devel \
+    devtoolset-8 devtoolset-8-gdb devtoolset-8-binutils devtoolset-8-libstdc++-devel \
+    devtoolset-8-libasan-devel devtoolset-8-libtsan-devel devtoolset-8-libubsan-devel devtoolset-8-liblsan-devel \
     flex flex-devel bison bison-devel parallel \
     ncurses ncurses-devel ncurses-libs graphviz graphviz-devel \
     lzip p7zip bzip2 bzip2-devel lzma-sdk lzma-sdk-devel xz-devel \
     zlib zlib-devel zlib-static texinfo openssl openssl-static \
     jemalloc jemalloc-devel \
     libicu-devel htop \
-    rh-python38 \
+    python27-python rh-python35-python \
+    python-devel python27-python-devel rh-python35-python-devel \
+    python27 rh-python35 \
     ninja-build git svn \
-    swig perl-Digest-MD5
+    swig perl-Digest-MD5 python2-pip
+
+if [ "${CI_USE}" = true ] ; then
+    # TODO need permanent link
+    yum -y install ftp://ftp.pbone.net/mirror/archive.fedoraproject.org/fedora-secondary/updates/26/i386/Packages/p/python2-six-1.10.0-9.fc26.noarch.rpm
+
+    yum -y install \
+        llvm-toolset-7 llvm-toolset-7-runtime llvm-toolset-7-build llvm-toolset-7-clang \
+        llvm-toolset-7-clang-analyzer llvm-toolset-7-clang-devel llvm-toolset-7-clang-libs \
+        llvm-toolset-7-clang-tools-extra llvm-toolset-7-compiler-rt llvm-toolset-7-lldb \
+        llvm-toolset-7-lldb-devel llvm-toolset-7-python-lldb
+
+fi

Builds/containers/centos-builder/extras.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+set -ex
+
+if [ "${CI_USE}" = true ] ; then
+    cd /tmp
+    wget https://ftp.gnu.org/gnu/gdb/gdb-8.3.1.tar.xz
+    tar xf gdb-8.3.1.tar.xz
+    cd gdb-8.3
+    ./configure CFLAGS="-w -O2" CXXFLAGS="-std=gnu++11 -g -O2 -w" --prefix=/opt/local/gdb-8.3
+    make -j$(nproc)
+    make install
+    ln -s /opt/local/gdb-8.3 /opt/local/gdb
+    cd ..
+    rm -f gdb-8.3.tar.xz
+    rm -rf gdb-8.3
+
+    # clang from source
+    cd /tmp
+    git clone https://github.com/llvm/llvm-project.git
+    cd llvm-project
+    git checkout llvmorg-9.0.0
+    INSTALL=/opt/llvm-9/
+    mkdir mybuilddir && cd mybuilddir
+    # TODO figure out necessary options
+    cmake ../llvm -G Ninja \
+      -DCMAKE_BUILD_TYPE=Release \
+      -DLLVM_ENABLE_PROJECTS='clang;clang-tools-extra;libcxx;libcxxabi;lldb;compiler-rt;lld;polly' \
+      -DCMAKE_INSTALL_PREFIX=${INSTALL} \
+      -DLLVM_LIBDIR_SUFFIX=64
+    cmake --build . --parallel --target install
+    cd /tmp
+    rm -rf llvm-project
+fi

Builds/containers/gitlab-ci/build_package.sh

@@ -22,7 +22,6 @@ time cmake \
   -Dpackages_only=ON \
   -Dcontainer_label="${container_tag}" \
   -Dhave_package_container=ON \
-  -DCMAKE_VERBOSE_MAKEFILE=ON \
-  -Dunity=OFF \
+  -DCMAKE_VERBOSE_MAKEFILE=OFF \
   -G Ninja ../..
-time cmake --build . --target ${pkgtype} -- -v
+time cmake --build . --target ${pkgtype}

Builds/containers/gitlab-ci/docker_alpine_setup.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env sh
-set -e
+set -ex
 # used as a before/setup script for docker steps in gitlab-ci
 # expects to be run in standard alpine/dind image
 echo $(nproc)
@@ -13,3 +13,4 @@ apk add \
 pip3 install awscli
 # list curdir contents to build log:
 ls -la
+

Builds/containers/gitlab-ci/pkgbuild.yml

@@ -11,10 +11,10 @@ variables:
   # these containers are built manually using the rippled
   # cmake build (container targets) and tagged/pushed so they
   # can be used here
-  RPM_CONTAINER_TAG: "2023-02-13"
+  RPM_CONTAINER_TAG: "2020-02-10"
   RPM_CONTAINER_NAME: "rippled-rpm-builder"
   RPM_CONTAINER_FULLNAME: "${RPM_CONTAINER_NAME}:${RPM_CONTAINER_TAG}"
-  DPKG_CONTAINER_TAG: "2023-03-20"
+  DPKG_CONTAINER_TAG: "2020-02-10"
   DPKG_CONTAINER_NAME: "rippled-dpkg-builder"
   DPKG_CONTAINER_FULLNAME: "${DPKG_CONTAINER_NAME}:${DPKG_CONTAINER_TAG}"
   ARTIFACTORY_HOST: "artifactory.ops.ripple.com"
@@ -83,7 +83,6 @@ stages:
 #########################################################################
 
 rpm_build:
-  timeout: "1h 30m"
   stage: build_packages
   <<: *dind_param
   artifacts:
@@ -93,7 +92,6 @@ rpm_build:
     - . ./Builds/containers/gitlab-ci/build_package.sh rpm
 
 dpkg_build:
-  timeout: "1h 30m"
   stage: build_packages
   <<: *dind_param
   artifacts:
@@ -181,38 +179,47 @@ centos_7_smoketest:
   stage: smoketest
   dependencies:
     - rpm_build
+    - rpm_sign
   image:
     name: artifactory.ops.ripple.com/centos:7
   <<: *run_local_smoketest
 
+# TODO: Remove "allow_failure" when tests fixed
 rocky_8_smoketest:
   stage: smoketest
   dependencies:
     - rpm_build
+    - rpm_sign
   image:
-    name: artifactory.ops.ripple.com/rockylinux/rockylinux:8
+    name: rockylinux/rockylinux:8
   <<: *run_local_smoketest
+  allow_failure: true
 
-fedora_37_smoketest:
+fedora_34_smoketest:
   stage: smoketest
   dependencies:
     - rpm_build
+    - rpm_sign
   image:
-    name: artifactory.ops.ripple.com/fedora:37
+    name: artifactory.ops.ripple.com/fedora:34
   <<: *run_local_smoketest
+  allow_failure: true
 
-fedora_38_smoketest:
+fedora_35_smoketest:
   stage: smoketest
   dependencies:
     - rpm_build
+    - rpm_sign
   image:
-    name: artifactory.ops.ripple.com/fedora:38
+    name: artifactory.ops.ripple.com/fedora:35
   <<: *run_local_smoketest
+  allow_failure: true
 
 ubuntu_18_smoketest:
   stage: smoketest
   dependencies:
     - dpkg_build
+    - dpkg_sign
   image:
     name: artifactory.ops.ripple.com/ubuntu:18.04
   <<: *run_local_smoketest
@@ -221,22 +228,36 @@ ubuntu_20_smoketest:
   stage: smoketest
   dependencies:
     - dpkg_build
+    - dpkg_sign
   image:
     name: artifactory.ops.ripple.com/ubuntu:20.04
   <<: *run_local_smoketest
 
+# TODO: remove "allow_failure" when 22.04 released in 4/2022...
 ubuntu_22_smoketest:
   stage: smoketest
   dependencies:
     - dpkg_build
+    - dpkg_sign
   image:
     name: artifactory.ops.ripple.com/ubuntu:22.04
   <<: *run_local_smoketest
+  allow_failure: true
+
+debian_9_smoketest:
+  stage: smoketest
+  dependencies:
+    - dpkg_build
+    - dpkg_sign
+  image:
+    name: artifactory.ops.ripple.com/debian:9
+  <<: *run_local_smoketest
 
 debian_10_smoketest:
   stage: smoketest
   dependencies:
     - dpkg_build
+    - dpkg_sign
   image:
     name: artifactory.ops.ripple.com/debian:10
   <<: *run_local_smoketest
@@ -245,6 +266,7 @@ debian_11_smoketest:
   stage: smoketest
   dependencies:
     - dpkg_build
+    - dpkg_sign
   image:
     name: artifactory.ops.ripple.com/debian:11
   <<: *run_local_smoketest
@@ -345,39 +367,41 @@ centos_7_verify_repo_test:
   <<: *only_primary
   <<: *run_repo_smoketest
 
-
 rocky_8_verify_repo_test:
   stage: verify_from_test
   variables:
     RPM_REPO: "rippled-rpm-test-mirror"
   image:
-    name: artifactory.ops.ripple.com/rockylinux/rockylinux:8
+    name: rockylinux/rockylinux:8
   dependencies:
     - rpm_sign
   <<: *only_primary
   <<: *run_repo_smoketest
+  allow_failure: true
 
-fedora_37_verify_repo_test:
+fedora_34_verify_repo_test:
   stage: verify_from_test
   variables:
     RPM_REPO: "rippled-rpm-test-mirror"
   image:
-    name: artifactory.ops.ripple.com/fedora:37
+    name: artifactory.ops.ripple.com/fedora:34
   dependencies:
     - rpm_sign
   <<: *only_primary
   <<: *run_repo_smoketest
+  allow_failure: true
 
-fedora_38_verify_repo_test:
+fedora_35_verify_repo_test:
   stage: verify_from_test
   variables:
     RPM_REPO: "rippled-rpm-test-mirror"
   image:
-    name: artifactory.ops.ripple.com/fedora:38
+    name: artifactory.ops.ripple.com/fedora:35
   dependencies:
     - rpm_sign
   <<: *only_primary
   <<: *run_repo_smoketest
+  allow_failure: true
 
 ubuntu_18_verify_repo_test:
   stage: verify_from_test
@@ -403,6 +427,7 @@ ubuntu_20_verify_repo_test:
   <<: *only_primary
   <<: *run_repo_smoketest
 
+# TODO: remove "allow_failure" when 22.04 released in 4/2022...
 ubuntu_22_verify_repo_test:
   stage: verify_from_test
   variables:
@@ -414,6 +439,19 @@ ubuntu_22_verify_repo_test:
     - dpkg_sign
   <<: *only_primary
   <<: *run_repo_smoketest
+  allow_failure: true
+
+debian_9_verify_repo_test:
+  stage: verify_from_test
+  variables:
+    DISTRO: "stretch"
+    DEB_REPO: "rippled-deb-test-mirror"
+  image:
+    name: artifactory.ops.ripple.com/debian:9
+  dependencies:
+    - dpkg_sign
+  <<: *only_primary
+  <<: *run_repo_smoketest
 
 debian_10_verify_repo_test:
   stage: verify_from_test
@@ -504,38 +542,41 @@ centos_7_verify_repo_prod:
   <<: *only_primary
   <<: *run_repo_smoketest
 
-rocky_8_verify_repo_prod:
-  stage: verify_from_prod
+rocky_8_verify_repo_test:
+  stage: verify_from_test
   variables:
-    RPM_REPO: "rippled-rpm"
+    RPM_REPO: "rippled-rpm-test-mirror"
   image:
-    name: artifactory.ops.ripple.com/rockylinux/rockylinux:8
+    name: rockylinux/rockylinux:8
   dependencies:
     - rpm_sign
   <<: *only_primary
   <<: *run_repo_smoketest
+  allow_failure: true
 
-fedora_37_verify_repo_prod:
+fedora_34_verify_repo_prod:
   stage: verify_from_prod
   variables:
     RPM_REPO: "rippled-rpm"
   image:
-    name: artifactory.ops.ripple.com/fedora:37
+    name: artifactory.ops.ripple.com/fedora:34
   dependencies:
     - rpm_sign
   <<: *only_primary
   <<: *run_repo_smoketest
+  allow_failure: true
 
-fedora_38_verify_repo_prod:
+fedora_35_verify_repo_prod:
   stage: verify_from_prod
   variables:
     RPM_REPO: "rippled-rpm"
   image:
-    name: artifactory.ops.ripple.com/fedora:38
+    name: artifactory.ops.ripple.com/fedora:35
   dependencies:
     - rpm_sign
   <<: *only_primary
   <<: *run_repo_smoketest
+  allow_failure: true
 
 ubuntu_18_verify_repo_prod:
   stage: verify_from_prod
@@ -561,6 +602,7 @@ ubuntu_20_verify_repo_prod:
   <<: *only_primary
   <<: *run_repo_smoketest
 
+# TODO: remove "allow_failure" when 22.04 released in 4/2022...
 ubuntu_22_verify_repo_prod:
   stage: verify_from_prod
   variables:
@@ -572,6 +614,19 @@ ubuntu_22_verify_repo_prod:
     - dpkg_sign
   <<: *only_primary
   <<: *run_repo_smoketest
+  allow_failure: true
+
+debian_9_verify_repo_prod:
+  stage: verify_from_prod
+  variables:
+    DISTRO: "stretch"
+    DEB_REPO: "rippled-deb"
+  image:
+    name: artifactory.ops.ripple.com/debian:9
+  dependencies:
+    - dpkg_sign
+  <<: *only_primary
+  <<: *run_repo_smoketest
 
 debian_10_verify_repo_prod:
   stage: verify_from_prod
@@ -638,9 +693,11 @@ build_centos_container:
   <<: *dind_param
   script:
     - . ./Builds/containers/gitlab-ci/build_container.sh rpm
+  allow_failure: true
 
 build_ubuntu_container:
   stage: build_containers
   <<: *dind_param
   script:
     - . ./Builds/containers/gitlab-ci/build_container.sh dpkg
+  allow_failure: true

Builds/containers/gitlab-ci/push_to_artifactory.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env sh
-set -e
+set -ex
 action=$1
 filter=$2
 
@@ -14,17 +14,16 @@ cd $TOPDIR
 cd build/dpkg/packages
 CURLARGS="-sk -X${action} -urippled:${ARTIFACTORY_DEPLOY_KEY_RIPPLED}"
 RIPPLED_PKG=$(ls rippled_*.deb)
-RIPPLED_REPORTING_PKG=$(ls rippled-reporting_*.deb)
-RIPPLED_DBG_PKG=$(ls rippled-dbgsym_*.*deb)
-RIPPLED_REPORTING_DBG_PKG=$(ls rippled-reporting-dbgsym_*.*deb)
+RIPPLED_DEV_PKG=$(ls rippled-dev_*.deb)
+RIPPLED_DBG_PKG=$(ls rippled-dbgsym_*.deb)
 # TODO - where to upload src tgz?
 RIPPLED_SRC=$(ls rippled_*.orig.tar.gz)
 DEB_MATRIX=";deb.component=${COMPONENT};deb.architecture=amd64"
-for dist in buster bullseye bionic focal jammy; do
+for dist in stretch buster bullseye bionic focal jammy; do
     DEB_MATRIX="${DEB_MATRIX};deb.distribution=${dist}"
 done
 echo "{ \"debs\": {" > "${TOPDIR}/files.info"
-for deb in ${RIPPLED_PKG} ${RIPPLED_DBG_PKG} ${RIPPLED_REPORTING_PKG} ${RIPPLED_REPORTING_DBG_PKG}; do
+for deb in ${RIPPLED_PKG} ${RIPPLED_DEV_PKG} ${RIPPLED_DBG_PKG} ; do
     # first item doesn't get a comma separator
     if [ $deb != $RIPPLED_PKG ] ; then
         echo "," >> "${TOPDIR}/files.info"
@@ -49,11 +48,10 @@ cd build/rpm/packages
 RIPPLED_PKG=$(ls rippled-[0-9]*.x86_64.rpm)
 RIPPLED_DEV_PKG=$(ls rippled-devel*.rpm)
 RIPPLED_DBG_PKG=$(ls rippled-debuginfo*.rpm)
-RIPPLED_REPORTING_PKG=$(ls rippled-reporting*.rpm)
 # TODO - where to upload src rpm ?
 RIPPLED_SRC=$(ls rippled-[0-9]*.src.rpm)
 echo "\"rpms\": {" >> "${TOPDIR}/files.info"
-for rpm in ${RIPPLED_PKG} ${RIPPLED_DEV_PKG} ${RIPPLED_DBG_PKG} ${RIPPLED_REPORTING_PKG}; do
+for rpm in ${RIPPLED_PKG} ${RIPPLED_DEV_PKG} ${RIPPLED_DBG_PKG} ; do
     # first item doesn't get a comma separator
     if [ $rpm != $RIPPLED_PKG ] ; then
         echo "," >> "${TOPDIR}/files.info"

Builds/containers/gitlab-ci/smoketest.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env sh
-set -e
+set -ex
 install_from=$1
 use_private=${2:-0} # this option not currently needed by any CI scripts,
                     # reserved for possible future use
@@ -61,11 +61,7 @@ if [ "${pkgtype}" = "dpkg" ] ; then
 else
     yum -y update
     if [ "${install_from}" = "repo" ] ; then
-        pkgs=("yum-utils coreutils util-linux")
-        if [ "$ID" = "rocky" ]; then
-            pkgs="${pkgs[@]/coreutils}"
-        fi
-        yum install -y $pkgs
+        yum -y install yum-utils coreutils util-linux
         REPOFILE="/etc/yum.repos.d/artifactory.repo"
         echo "[Artifactory]" > ${REPOFILE}
         echo "name=Artifactory" >> ${REPOFILE}
@@ -81,10 +77,8 @@ else
     elif [ "${install_from}" = "local" ] ; then
         # cached pkg install
         pkgs=("yum-utils openssl-static zlib-static")
-        if [[ "$ID" =~ rocky|fedora ]]; then
-            if [[ "$ID" =~ "rocky" ]]; then
-                sed -i 's/enabled=0/enabled=1/g' /etc/yum.repos.d/Rocky-PowerTools.repo
-            fi
+        if [ "$ID" = "rocky" ]; then
+            sed -i 's/enabled=0/enabled=1/g' /etc/yum.repos.d/Rocky-PowerTools.repo
             pkgs="${pkgs[@]/openssl-static}"
         fi
         yum install -y $pkgs

Builds/containers/gitlab-ci/tag_docker_image.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env sh
-set -e
+set -ex
 docker login -u rippled \
     -p ${ARTIFACTORY_DEPLOY_KEY_RIPPLED} "${ARTIFACTORY_HUB}"
 # this gives us rippled_version :
@@ -19,3 +19,4 @@ for label in ${rippled_version} latest ; do
     docker push \
         "${ARTIFACTORY_HUB}/${DPKG_CONTAINER_NAME}:${label}_${CI_COMMIT_REF_SLUG}"
 done
+

Builds/containers/packaging/dpkg/build_dpkg.sh

@@ -4,7 +4,7 @@ set -ex
 # make sure pkg source files are up to date with repo
 cd /opt/rippled_bld/pkg
 cp -fpru rippled/Builds/containers/packaging/dpkg/debian/. debian/
-cp -fpu rippled/Builds/containers/shared/rippled*.service debian/
+cp -fpu rippled/Builds/containers/shared/rippled.service debian/
 cp -fpu rippled/Builds/containers/shared/update_sources.sh .
 source update_sources.sh
 
@@ -15,7 +15,7 @@ RIPPLED_DPKG_VERSION=$(echo "${RIPPLED_VERSION}" | sed 's!-!~!g')
 # TODO - decide how to handle the trailing/release
 # version here (hardcoded to 1). Does it ever need to change?
 RIPPLED_DPKG_FULL_VERSION="${RIPPLED_DPKG_VERSION}-1"
-git config --global --add safe.directory /opt/rippled_bld/pkg/rippled
+
 cd /opt/rippled_bld/pkg/rippled
 if [[ -n $(git status --porcelain) ]]; then
     git status
@@ -45,19 +45,21 @@ CHANGELOG
 # PATH must be preserved for our more modern cmake in /opt/local
 # TODO : consider allowing lintian to run in future ?
 export DH_BUILD_DDEBS=1
+export CC=gcc-8
+export CXX=g++-8
 debuild --no-lintian --preserve-envvar PATH --preserve-env -us -uc
 rc=$?; if [[ $rc != 0 ]]; then
     error "error building dpkg"
 fi
 cd ..
+ls -latr
 
 # copy artifacts
-cp rippled-reporting_${RIPPLED_DPKG_FULL_VERSION}_amd64.deb ${PKG_OUTDIR}
+cp rippled-dev_${RIPPLED_DPKG_FULL_VERSION}_amd64.deb ${PKG_OUTDIR}
 cp rippled_${RIPPLED_DPKG_FULL_VERSION}_amd64.deb ${PKG_OUTDIR}
 cp rippled_${RIPPLED_DPKG_FULL_VERSION}.dsc ${PKG_OUTDIR}
 # dbgsym suffix is ddeb under newer debuild, but just deb under earlier
 cp rippled-dbgsym_${RIPPLED_DPKG_FULL_VERSION}_amd64.* ${PKG_OUTDIR}
-cp rippled-reporting-dbgsym_${RIPPLED_DPKG_FULL_VERSION}_amd64.* ${PKG_OUTDIR}
 cp rippled_${RIPPLED_DPKG_FULL_VERSION}_amd64.changes ${PKG_OUTDIR}
 cp rippled_${RIPPLED_DPKG_FULL_VERSION}_amd64.build ${PKG_OUTDIR}
 cp rippled_${RIPPLED_DPKG_VERSION}.orig.tar.gz ${PKG_OUTDIR}
@@ -79,17 +81,15 @@ DEB_SHA256=$(cat shasums | \
     grep "rippled_${RIPPLED_DPKG_VERSION}-1_amd64.deb" | cut -d " " -f 1)
 DBG_SHA256=$(cat shasums | \
     grep "rippled-dbgsym_${RIPPLED_DPKG_VERSION}-1_amd64.*" | cut -d " " -f 1)
-REPORTING_DBG_SHA256=$(cat shasums | \
-    grep "rippled-reporting-dbgsym_${RIPPLED_DPKG_VERSION}-1_amd64.*" | cut -d " " -f 1)
-REPORTING_SHA256=$(cat shasums | \
-    grep "rippled-reporting_${RIPPLED_DPKG_VERSION}-1_amd64.deb" | cut -d " " -f 1)
+DEV_SHA256=$(cat shasums | \
+    grep "rippled-dev_${RIPPLED_DPKG_VERSION}-1_amd64.deb" | cut -d " " -f 1)
 SRC_SHA256=$(cat shasums | \
     grep "rippled_${RIPPLED_DPKG_VERSION}.orig.tar.gz" | cut -d " " -f 1)
 echo "deb_sha256=${DEB_SHA256}" >> ${PKG_OUTDIR}/build_vars
 echo "dbg_sha256=${DBG_SHA256}" >> ${PKG_OUTDIR}/build_vars
-echo "reporting_sha256=${REPORTING_SHA256}" >> ${PKG_OUTDIR}/build_vars
-echo "reporting_dbg_sha256=${REPORTING_DBG_SHA256}" >> ${PKG_OUTDIR}/build_vars
+echo "dev_sha256=${DEV_SHA256}" >> ${PKG_OUTDIR}/build_vars
 echo "src_sha256=${SRC_SHA256}" >> ${PKG_OUTDIR}/build_vars
 echo "rippled_version=${RIPPLED_VERSION}" >> ${PKG_OUTDIR}/build_vars
 echo "dpkg_version=${RIPPLED_DPKG_VERSION}" >> ${PKG_OUTDIR}/build_vars
 echo "dpkg_full_version=${RIPPLED_DPKG_FULL_VERSION}" >> ${PKG_OUTDIR}/build_vars
+

Builds/containers/packaging/dpkg/debian/compat

@@ -1 +1 @@
-10
+9

Builds/containers/packaging/dpkg/debian/conffiles

@@ -1,2 +1,3 @@
 /opt/ripple/etc/rippled.cfg
 /opt/ripple/etc/validators.txt
+/etc/logrotate.d/rippled

Builds/containers/packaging/dpkg/debian/control

@@ -12,8 +12,10 @@ Multi-Arch: foreign
 Depends: ${misc:Depends}, ${shlibs:Depends}
 Description: rippled daemon
 
-Package: rippled-reporting
+Package: rippled-dev
+Section: devel
+Recommends: rippled (= ${binary:Version})
 Architecture: any
-Multi-Arch: foreign
-Depends: ${misc:Depends}, ${shlibs:Depends}
-Description: rippled reporting daemon
+Multi-Arch: same
+Depends: ${misc:Depends}, ${shlibs:Depends}, libprotobuf-dev, libprotoc-dev, protobuf-compiler
+Description: development files for applications using xrpl core library (serialize + sign)

Builds/containers/packaging/dpkg/debian/rippled-reporting.dirs

@@ -1,3 +0,0 @@
-/var/log/rippled-reporting/
-/var/lib/rippled-reporting/
-/etc/systemd/system/rippled-reporting.service.d/

Builds/containers/packaging/dpkg/debian/rippled-reporting.install

@@ -1,8 +0,0 @@
-bld/rippled-reporting/rippled-reporting opt/rippled-reporting/bin
-cfg/rippled-reporting.cfg opt/rippled-reporting/etc
-debian/tmp/opt/rippled-reporting/etc/validators.txt opt/rippled-reporting/etc
-
-opt/rippled-reporting/bin/update-rippled-reporting.sh
-opt/rippled-reporting/bin/getRippledReportingInfo
-opt/rippled-reporting/etc/update-rippled-reporting-cron
-etc/logrotate.d/rippled-reporting

Builds/containers/packaging/dpkg/debian/rippled-reporting.links

@@ -1,3 +0,0 @@
-opt/rippled-reporting/etc/rippled-reporting.cfg etc/opt/rippled-reporting/rippled-reporting.cfg
-opt/rippled-reporting/etc/validators.txt etc/opt/rippled-reporting/validators.txt
-opt/rippled-reporting/bin/rippled-reporting usr/local/bin/rippled-reporting

Builds/containers/packaging/dpkg/debian/rippled-reporting.postinst

@@ -1,33 +0,0 @@
-#!/bin/sh
-set -e
-
-USER_NAME=rippled-reporting
-GROUP_NAME=rippled-reporting
-case "$1" in
-    configure)
-        id -u $USER_NAME >/dev/null 2>&1 || \
-        adduser --system --quiet \
-            --home /nonexistent --no-create-home \
-            --disabled-password \
-            --group "$GROUP_NAME"
-        chown -R $USER_NAME:$GROUP_NAME /var/log/rippled-reporting/
-        chown -R $USER_NAME:$GROUP_NAME /var/lib/rippled-reporting/
-        chmod 755 /var/log/rippled-reporting/
-        chmod 755 /var/lib/rippled-reporting/
-        chown -R $USER_NAME:$GROUP_NAME /opt/rippled-reporting
-
-    ;;
-
-    abort-upgrade|abort-remove|abort-deconfigure)
-    ;;
-
-    *)
-        echo "postinst called with unknown argument \`$1'" >&2
-        exit 1
-    ;;
-esac
-
-
-#DEBHELPER#
-
-exit 0

Builds/containers/packaging/dpkg/debian/rules

@@ -17,64 +17,27 @@ override_dh_systemd_start:
 override_dh_auto_configure:
 	env
 	rm -rf bld
-
-	conan export external/snappy snappy/1.1.9@
-
-	conan install . \
-		--install-folder bld/rippled  \
-		--build missing \
-		--build boost \
-		--build sqlite3 \
-		--settings build_type=Release
-
-	cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \
-		-G Ninja \
-		-DCMAKE_BUILD_TYPE=Release \
+	mkdir -p bld
+	cd bld && \
+	cmake .. -G Ninja \
 		-DCMAKE_INSTALL_PREFIX=/opt/ripple \
-		-Dstatic=ON \
-		-Dunity=OFF \
-		-DCMAKE_VERBOSE_MAKEFILE=ON \
-		-Dvalidator_keys=ON \
-		-B bld/rippled
-
-	conan install . \
-		--install-folder bld/rippled-reporting \
-		--build missing \
-		--build boost \
-		--build sqlite3 \
-		--build libuv \
-		--settings build_type=Release \
-		--options reporting=True
-
-	cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \
-		-G Ninja \
 		-DCMAKE_BUILD_TYPE=Release \
-		-DCMAKE_INSTALL_PREFIX=/opt/rippled-reporting \
 		-Dstatic=ON \
 		-Dunity=OFF \
-		-DCMAKE_VERBOSE_MAKEFILE=ON \
-		-Dreporting=ON \
-		-B bld/rippled-reporting
+		-Dvalidator_keys=ON \
+		-DCMAKE_VERBOSE_MAKEFILE=OFF
 
 override_dh_auto_build:
-	cmake --build bld/rippled --target rippled --target validator-keys -j${nproc}
-
-	cmake --build bld/rippled-reporting --target rippled -j${nproc}
+	cd bld && \
+	cmake --build . --target rippled --target validator-keys --parallel
 
 override_dh_auto_install:
-	cmake --install bld/rippled --prefix debian/tmp/opt/ripple
-	install -D bld/rippled/validator-keys/validator-keys debian/tmp/opt/ripple/bin/validator-keys
+	cd bld && DESTDIR=../debian/tmp cmake --build . --target install
+	install -D bld/validator-keys/validator-keys debian/tmp/opt/ripple/bin/validator-keys
 	install -D Builds/containers/shared/update-rippled.sh debian/tmp/opt/ripple/bin/update-rippled.sh
 	install -D bin/getRippledInfo debian/tmp/opt/ripple/bin/getRippledInfo
 	install -D Builds/containers/shared/update-rippled-cron debian/tmp/opt/ripple/etc/update-rippled-cron
 	install -D Builds/containers/shared/rippled-logrotate debian/tmp/etc/logrotate.d/rippled
 	rm -rf debian/tmp/opt/ripple/lib64/cmake/date
-
-	mkdir -p debian/tmp/opt/rippled-reporting/etc
-	mkdir -p debian/tmp/opt/rippled-reporting/bin
-	cp cfg/validators-example.txt debian/tmp/opt/rippled-reporting/etc/validators.txt
-
-	sed -E 's/rippled?/rippled-reporting/g' Builds/containers/shared/update-rippled.sh > debian/tmp/opt/rippled-reporting/bin/update-rippled-reporting.sh
-	sed -E 's/rippled?/rippled-reporting/g' bin/getRippledInfo > debian/tmp/opt/rippled-reporting/bin/getRippledReportingInfo
-	sed -E 's/rippled?/rippled-reporting/g' Builds/containers/shared/update-rippled-cron > debian/tmp/opt/rippled-reporting/etc/update-rippled-reporting-cron
-	sed -E 's/rippled?/rippled-reporting/g' Builds/containers/shared/rippled-logrotate > debian/tmp/etc/logrotate.d/rippled-reporting
+	rm -rf bld
+	rm -rf bld_vl

Builds/containers/packaging/rpm/50-rippled-reporting.preset

@@ -1 +0,0 @@
-enable rippled-reporting.service

Builds/containers/packaging/rpm/build_rpm.sh

@@ -29,20 +29,17 @@ if [[ $RPM_PATCH ]]; then
 fi
 
 cd /opt/rippled_bld/pkg/rippled
-
 if [[ -n $(git status --porcelain) ]]; then
-   git status
-   error "Unstaged changes in this repo - please commit first"
+    git status
+    error "Unstaged changes in this repo - please commit first"
 fi
-
 git archive --format tar.gz --prefix rippled/ -o ../rpmbuild/SOURCES/rippled.tar.gz HEAD
-
+# TODO include validator-keys sources
 cd ..
 
-source /opt/rh/devtoolset-11/enable
+source /opt/rh/devtoolset-8/enable
 
 rpmbuild --define "_topdir ${PWD}/rpmbuild" -ba rippled.spec
-
 rc=$?; if [[ $rc != 0 ]]; then
     error "error building rpm"
 fi
@@ -57,22 +54,18 @@ cp ./rpmbuild/SRPMS/* ${PKG_OUTDIR}
 RPM_MD5SUM=$(rpm -q --queryformat '%{SIGMD5}\n' -p ./rpmbuild/RPMS/x86_64/rippled-[0-9]*.rpm 2>/dev/null)
 DBG_MD5SUM=$(rpm -q --queryformat '%{SIGMD5}\n' -p ./rpmbuild/RPMS/x86_64/rippled-debuginfo*.rpm 2>/dev/null)
 DEV_MD5SUM=$(rpm -q --queryformat '%{SIGMD5}\n' -p ./rpmbuild/RPMS/x86_64/rippled-devel*.rpm 2>/dev/null)
-REP_MD5SUM=$(rpm -q --queryformat '%{SIGMD5}\n' -p ./rpmbuild/RPMS/x86_64/rippled-reporting*.rpm 2>/dev/null)
 SRC_MD5SUM=$(rpm -q --queryformat '%{SIGMD5}\n' -p ./rpmbuild/SRPMS/*.rpm 2>/dev/null)
 
 RPM_SHA256="$(sha256sum ./rpmbuild/RPMS/x86_64/rippled-[0-9]*.rpm | awk '{ print $1}')"
 DBG_SHA256="$(sha256sum ./rpmbuild/RPMS/x86_64/rippled-debuginfo*.rpm | awk '{ print $1}')"
-REP_SHA256="$(sha256sum ./rpmbuild/RPMS/x86_64/rippled-reporting*.rpm | awk '{ print $1}')"
 DEV_SHA256="$(sha256sum ./rpmbuild/RPMS/x86_64/rippled-devel*.rpm | awk '{ print $1}')"
 SRC_SHA256="$(sha256sum ./rpmbuild/SRPMS/*.rpm | awk '{ print $1}')"
 
 echo "rpm_md5sum=$RPM_MD5SUM" >  ${PKG_OUTDIR}/build_vars
-echo "rep_md5sum=$REP_MD5SUM" >> ${PKG_OUTDIR}/build_vars
 echo "dbg_md5sum=$DBG_MD5SUM" >> ${PKG_OUTDIR}/build_vars
 echo "dev_md5sum=$DEV_MD5SUM" >> ${PKG_OUTDIR}/build_vars
 echo "src_md5sum=$SRC_MD5SUM" >> ${PKG_OUTDIR}/build_vars
 echo "rpm_sha256=$RPM_SHA256" >> ${PKG_OUTDIR}/build_vars
-echo "rep_sha256=$REP_SHA256" >> ${PKG_OUTDIR}/build_vars
 echo "dbg_sha256=$DBG_SHA256" >> ${PKG_OUTDIR}/build_vars
 echo "dev_sha256=$DEV_SHA256" >> ${PKG_OUTDIR}/build_vars
 echo "src_sha256=$SRC_SHA256" >> ${PKG_OUTDIR}/build_vars
@@ -80,3 +73,4 @@ echo "rippled_version=$RIPPLED_VERSION" >> ${PKG_OUTDIR}/build_vars
 echo "rpm_version=$RIPPLED_RPM_VERSION" >> ${PKG_OUTDIR}/build_vars
 echo "rpm_file_name=$tar_file" >> ${PKG_OUTDIR}/build_vars
 echo "rpm_version_release=$RPM_VERSION_RELEASE" >> ${PKG_OUTDIR}/build_vars
+

Builds/containers/packaging/rpm/rippled.spec

@@ -2,7 +2,6 @@
 %define rpm_release %(echo $RPM_RELEASE)
 %define rpm_patch %(echo $RPM_PATCH)
 %define _prefix /opt/ripple
-
 Name:           rippled
 # Dashes in Version extensions must be converted to underscores
 Version:        %{rippled_version}
@@ -26,94 +25,29 @@ Requires: zlib-static
 %description devel
 core library for development of standalone applications that sign transactions.
 
-%package reporting
-Summary: Reporting Server for rippled
-
-%description reporting
-History server for XRP Ledger
-
 %prep
 %setup -c -n rippled
 
 %build
-rm -rf ~/.conan/profiles/default
-
-cp /opt/libcstd/libstdc++.so.6.0.22 /usr/lib64
-cp /opt/libcstd/libstdc++.so.6.0.22 /lib64
-ln -sf /usr/lib64/libstdc++.so.6.0.22 /usr/lib64/libstdc++.so.6
-ln -sf /lib64/libstdc++.so.6.0.22 /usr/lib64/libstdc++.so.6
-
-source /opt/rh/rh-python38/enable
-pip install "conan<2"
-conan profile new default --detect
-conan profile update settings.compiler.libcxx=libstdc++11 default
-conan profile update settings.compiler.cppstd=20 default
-
 cd rippled
-
-mkdir -p bld.rippled
-conan export external/snappy snappy/1.1.9@
-
-pushd bld.rippled
-conan install .. \
-     --settings build_type=Release \
-     --output-folder . \
-     --build missing
-
-cmake -G Ninja \
-     -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \
-     -DCMAKE_INSTALL_PREFIX=%{_prefix} \
-     -DCMAKE_BUILD_TYPE=Release \
-     -Dunity=OFF \
-     -Dstatic=ON \
-     -Dvalidator_keys=ON \
-     -DCMAKE_VERBOSE_MAKEFILE=ON \
-     ..
-
-cmake --build . --parallel $(nproc) --target rippled --target validator-keys
-popd
-
-mkdir -p bld.rippled-reporting
-pushd bld.rippled-reporting
-
-conan install .. \
-     --settings build_type=Release \
-     --output-folder . \
-     --build missing \
-     --settings compiler.cppstd=17 \
-     --options reporting=True
-
-cmake -G Ninja \
-     -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \
-     -DCMAKE_INSTALL_PREFIX=%{_prefix} \
-     -DCMAKE_BUILD_TYPE=Release \
-     -Dunity=OFF \
-     -Dstatic=ON \
-     -Dvalidator_keys=ON \
-     -Dreporting=ON \
-     -DCMAKE_VERBOSE_MAKEFILE=ON \
-     ..
-
-cmake --build . --parallel $(nproc) --target rippled
+mkdir -p bld.release
+cd bld.release
+cmake .. -G Ninja -DCMAKE_INSTALL_PREFIX=%{_prefix} -DCMAKE_BUILD_TYPE=Release -Dstatic=true -Dunity=OFF -DCMAKE_VERBOSE_MAKEFILE=OFF -Dvalidator_keys=ON
+cmake --build . --parallel --target rippled --target validator-keys
 
 %pre
 test -e /etc/pki/tls || { mkdir -p /etc/pki; ln -s /usr/lib/ssl /etc/pki/tls; }
 
 %install
 rm -rf $RPM_BUILD_ROOT
-DESTDIR=$RPM_BUILD_ROOT cmake --build rippled/bld.rippled --target install #-- -v
-mkdir -p $RPM_BUILD_ROOT
-rm -rf ${RPM_BUILD_ROOT}/%{_prefix}/lib64/
+DESTDIR=$RPM_BUILD_ROOT cmake --build rippled/bld.release --target install
+rm -rf ${RPM_BUILD_ROOT}/%{_prefix}/lib64/cmake/date
 install -d ${RPM_BUILD_ROOT}/etc/opt/ripple
 install -d ${RPM_BUILD_ROOT}/usr/local/bin
-
-install -D ./rippled/cfg/rippled-example.cfg ${RPM_BUILD_ROOT}/%{_prefix}/etc/rippled.cfg
-install -D ./rippled/cfg/validators-example.txt ${RPM_BUILD_ROOT}/%{_prefix}/etc/validators.txt
-
-ln -sf %{_prefix}/etc/rippled.cfg ${RPM_BUILD_ROOT}/etc/opt/ripple/rippled.cfg
-ln -sf %{_prefix}/etc/validators.txt ${RPM_BUILD_ROOT}/etc/opt/ripple/validators.txt
-ln -sf %{_prefix}/bin/rippled ${RPM_BUILD_ROOT}/usr/local/bin/rippled
-install -D rippled/bld.rippled/validator-keys/validator-keys ${RPM_BUILD_ROOT}%{_bindir}/validator-keys
+ln -s %{_prefix}/etc/rippled.cfg ${RPM_BUILD_ROOT}/etc/opt/ripple/rippled.cfg
+ln -s %{_prefix}/etc/validators.txt ${RPM_BUILD_ROOT}/etc/opt/ripple/validators.txt
+ln -s %{_prefix}/bin/rippled ${RPM_BUILD_ROOT}/usr/local/bin/rippled
+install -D rippled/bld.release/validator-keys/validator-keys ${RPM_BUILD_ROOT}%{_bindir}/validator-keys
 install -D ./rippled/Builds/containers/shared/rippled.service ${RPM_BUILD_ROOT}/usr/lib/systemd/system/rippled.service
 install -D ./rippled/Builds/containers/packaging/rpm/50-rippled.preset ${RPM_BUILD_ROOT}/usr/lib/systemd/system-preset/50-rippled.preset
 install -D ./rippled/Builds/containers/shared/update-rippled.sh ${RPM_BUILD_ROOT}%{_bindir}/update-rippled.sh
@@ -123,27 +57,7 @@ install -D ./rippled/Builds/containers/shared/rippled-logrotate ${RPM_BUILD_ROOT
 install -d $RPM_BUILD_ROOT/var/log/rippled
 install -d $RPM_BUILD_ROOT/var/lib/rippled
 
-# reporting mode
-%define _prefix /opt/rippled-reporting
-mkdir -p ${RPM_BUILD_ROOT}/etc/opt/rippled-reporting/
-install -D rippled/bld.rippled-reporting/rippled-reporting ${RPM_BUILD_ROOT}%{_bindir}/rippled-reporting
-install -D ./rippled/cfg/rippled-reporting.cfg ${RPM_BUILD_ROOT}%{_prefix}/etc/rippled-reporting.cfg
-install -D ./rippled/cfg/validators-example.txt ${RPM_BUILD_ROOT}%{_prefix}/etc/validators.txt
-install -D ./rippled/Builds/containers/packaging/rpm/50-rippled-reporting.preset ${RPM_BUILD_ROOT}/usr/lib/systemd/system-preset/50-rippled-reporting.preset
-ln -s %{_prefix}/bin/rippled-reporting ${RPM_BUILD_ROOT}/usr/local/bin/rippled-reporting
-ln -s %{_prefix}/etc/rippled-reporting.cfg ${RPM_BUILD_ROOT}/etc/opt/rippled-reporting/rippled-reporting.cfg
-ln -s %{_prefix}/etc/validators.txt ${RPM_BUILD_ROOT}/etc/opt/rippled-reporting/validators.txt
-install -d $RPM_BUILD_ROOT/var/log/rippled-reporting
-install -d $RPM_BUILD_ROOT/var/lib/rippled-reporting
-install -D ./rippled/Builds/containers/shared/rippled-reporting.service ${RPM_BUILD_ROOT}/usr/lib/systemd/system/rippled-reporting.service
-sed -E 's/rippled?/rippled-reporting/g' ./rippled/Builds/containers/shared/update-rippled.sh > ${RPM_BUILD_ROOT}%{_bindir}/update-rippled-reporting.sh
-sed -E 's/rippled?/rippled-reporting/g' ./rippled/bin/getRippledInfo > ${RPM_BUILD_ROOT}%{_bindir}/getRippledReportingInfo
-sed -E 's/rippled?/rippled-reporting/g' ./rippled/Builds/containers/shared/update-rippled-cron > ${RPM_BUILD_ROOT}%{_prefix}/etc/update-rippled-reporting-cron
-sed -E 's/rippled?/rippled-reporting/g' ./rippled/Builds/containers/shared/rippled-logrotate > ${RPM_BUILD_ROOT}/etc/logrotate.d/rippled-reporting
-
-
 %post
-%define _prefix /opt/ripple
 USER_NAME=rippled
 GROUP_NAME=rippled
 
@@ -161,25 +75,7 @@ chmod 644 %{_prefix}/etc/update-rippled-cron
 chmod 644 /etc/logrotate.d/rippled
 chown -R root:$GROUP_NAME %{_prefix}/etc/update-rippled-cron
 
-%post reporting
-%define _prefix /opt/rippled-reporting
-USER_NAME=rippled-reporting
-GROUP_NAME=rippled-reporting
-
-getent passwd $USER_NAME &>/dev/null || useradd -r $USER_NAME
-getent group $GROUP_NAME &>/dev/null || groupadd $GROUP_NAME
-
-chown -R $USER_NAME:$GROUP_NAME /var/log/rippled-reporting/
-chown -R $USER_NAME:$GROUP_NAME /var/lib/rippled-reporting/
-chown -R $USER_NAME:$GROUP_NAME %{_prefix}/
-
-chmod 755 /var/log/rippled-reporting/
-chmod 755 /var/lib/rippled-reporting/
-chmod -x /usr/lib/systemd/system/rippled-reporting.service
-
-
 %files
-%define _prefix /opt/ripple
 %doc rippled/README.md rippled/LICENSE.md
 %{_bindir}/rippled
 /usr/local/bin/rippled
@@ -194,7 +90,6 @@ chmod -x /usr/lib/systemd/system/rippled-reporting.service
 %config(noreplace) /etc/logrotate.d/rippled
 %config(noreplace) /usr/lib/systemd/system/rippled.service
 %config(noreplace) /usr/lib/systemd/system-preset/50-rippled.preset
-
 %dir /var/log/rippled/
 %dir /var/lib/rippled/
 
@@ -203,25 +98,6 @@ chmod -x /usr/lib/systemd/system/rippled-reporting.service
 %{_prefix}/lib/*.a
 %{_prefix}/lib/cmake/ripple
 
-%files reporting
-%define _prefix /opt/rippled-reporting
-%doc rippled/README.md rippled/LICENSE.md
-
-%{_bindir}/rippled-reporting
-/usr/local/bin/rippled-reporting
-%config(noreplace) /etc/opt/rippled-reporting/rippled-reporting.cfg
-%config(noreplace) %{_prefix}/etc/rippled-reporting.cfg
-%config(noreplace) %{_prefix}/etc/validators.txt
-%config(noreplace) /etc/opt/rippled-reporting/validators.txt
-%config(noreplace) /usr/lib/systemd/system/rippled-reporting.service
-%config(noreplace) /usr/lib/systemd/system-preset/50-rippled-reporting.preset
-%dir /var/log/rippled-reporting/
-%dir /var/lib/rippled-reporting/
-%{_bindir}/update-rippled-reporting.sh
-%{_bindir}/getRippledReportingInfo
-%{_prefix}/etc/update-rippled-reporting-cron
-%config(noreplace) /etc/logrotate.d/rippled-reporting
-
 %changelog
 * Wed Aug 28 2019 Mike Ellery <mellery451@gmail.com>
 - Switch to subproject build for validator-keys

Builds/containers/shared/build_deps.sh

@@ -0,0 +1,147 @@
+#!/usr/bin/env bash
+set -ex
+
+function build_boost()
+{
+    local boost_ver=$1
+    local do_link=$2
+    local boost_path=$(echo "${boost_ver}" | sed -e 's!\.!_!g')
+    mkdir -p /opt/local
+    cd /opt/local
+    BOOST_ROOT=/opt/local/boost_${boost_path}
+    BOOST_URL="https://boostorg.jfrog.io/artifactory/main/release/${boost_ver}/source/boost_${boost_path}.tar.gz"
+    BOOST_BUILD_ALL=true
+    . /tmp/install_boost.sh
+    if [ "$do_link" = true ] ; then
+        ln -s ./boost_${boost_path} boost
+    fi
+}
+
+build_boost "1.70.0" true
+
+# installed in opt, so won't be used
+# unless specified by OPENSSL_ROOT_DIR
+cd /tmp
+OPENSSL_VER=1.1.1d
+wget https://www.openssl.org/source/openssl-${OPENSSL_VER}.tar.gz
+tar xf openssl-${OPENSSL_VER}.tar.gz
+cd openssl-${OPENSSL_VER}
+# NOTE: add -g to the end of the following line if we want debug symbols for openssl
+SSLDIR=$(openssl version -d | cut -d: -f2 | tr -d [:space:]\")
+./config -fPIC --prefix=/opt/local/openssl --openssldir=${SSLDIR} zlib shared
+make -j$(nproc) >> make_output.txt 2>&1
+make install
+cd ..
+rm -f openssl-${OPENSSL_VER}.tar.gz
+rm -rf openssl-${OPENSSL_VER}
+LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:/opt/local/openssl/lib /opt/local/openssl/bin/openssl version -a
+
+cd /tmp
+wget https://libarchive.org/downloads/libarchive-3.4.1.tar.gz
+tar xzf libarchive-3.4.1.tar.gz
+cd libarchive-3.4.1
+mkdir _bld && cd _bld
+cmake -DCMAKE_BUILD_TYPE=Release ..
+make -j$(nproc) >> make_output.txt 2>&1
+make install
+cd ../..
+rm -f libarchive-3.4.1.tar.gz
+rm -rf libarchive-3.4.1
+
+cd /tmp
+wget https://github.com/protocolbuffers/protobuf/releases/download/v3.10.1/protobuf-all-3.10.1.tar.gz
+tar xf protobuf-all-3.10.1.tar.gz
+cd protobuf-3.10.1
+./autogen.sh
+./configure
+make -j$(nproc) >> make_output.txt 2>&1
+make install
+ldconfig
+cd ..
+rm -f protobuf-all-3.10.1.tar.gz
+rm -rf protobuf-3.10.1
+
+cd /tmp
+wget https://c-ares.haxx.se/download/c-ares-1.15.0.tar.gz
+tar xf c-ares-1.15.0.tar.gz
+cd c-ares-1.15.0
+mkdir _bld && cd _bld
+cmake \
+  -DHAVE_LIBNSL=OFF \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DCARES_STATIC=ON \
+  -DCARES_SHARED=OFF \
+  -DCARES_INSTALL=ON \
+  -DCARES_STATIC_PIC=ON \
+  -DCARES_BUILD_TOOLS=OFF \
+  -DCARES_BUILD_TESTS=OFF \
+  -DCARES_BUILD_CONTAINER_TESTS=OFF \
+  ..
+make -j$(nproc) >> make_output.txt 2>&1
+make install
+cd ../..
+rm -f c-ares-1.15.0.tar.gz
+rm -rf c-ares-1.15.0
+
+cd /tmp
+wget https://github.com/grpc/grpc/archive/v1.25.0.tar.gz
+tar xf v1.25.0.tar.gz
+cd grpc-1.25.0
+mkdir _bld && cd _bld
+cmake \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DBUILD_SHARED_LIBS=OFF \
+  -DgRPC_ZLIB_PROVIDER=package \
+  -DgRPC_CARES_PROVIDER=package \
+  -DgRPC_SSL_PROVIDER=package \
+  -DgRPC_PROTOBUF_PROVIDER=package \
+  -DProtobuf_USE_STATIC_LIBS=ON \
+  ..
+make -j$(nproc) >> make_output.txt 2>&1
+make install
+cd ../..
+rm -f xf v1.25.0.tar.gz
+rm -rf grpc-1.25.0
+
+if [ "${CI_USE}" = true ] ; then
+
+    build_boost "1.71.0" false
+
+    cd /tmp
+    wget https://github.com/doxygen/doxygen/archive/Release_1_8_16.tar.gz
+    tar xf Release_1_8_16.tar.gz
+    cd doxygen-Release_1_8_16
+    mkdir build
+    cd build
+    cmake -G "Unix Makefiles" ..
+    make -j$(nproc) >> make_output.txt 2>&1
+    make install
+    cd ../..
+    rm -f Release_1_8_16.tar.gz
+    rm -rf doxygen-Release_1_8_16
+
+    mkdir -p /opt/plantuml
+    wget -O /opt/plantuml/plantuml.jar https://downloads.sourceforge.net/project/plantuml/plantuml.jar
+
+    cd /tmp
+    wget https://github.com/linux-test-project/lcov/releases/download/v1.14/lcov-1.14.tar.gz
+    tar xfz lcov-1.14.tar.gz
+    cd lcov-1.14
+    make install PREFIX=/usr/local
+    cd ..
+    rm -r lcov-1.14 lcov-1.14.tar.gz
+
+    cd /tmp
+    wget https://github.com/ccache/ccache/releases/download/v3.7.6/ccache-3.7.6.tar.gz
+    tar xf ccache-3.7.6.tar.gz
+    cd ccache-3.7.6
+    ./configure --prefix=/usr/local
+    make
+    make install
+    cd ..
+    rm -f ccache-3.7.6.tar.gz
+    rm -rf ccache-3.7.6
+
+    pip install requests
+    pip install https://github.com/codecov/codecov-python/archive/master.zip
+fi

Builds/containers/shared/install_boost.sh

@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+# Assumptions:
+# 1) BOOST_ROOT and BOOST_URL are already defined,
+# and contain valid values. BOOST_URL2 may be defined
+# as a fallback. BOOST_WGET_OPTIONS may be defined with
+# retry options if the download(s) fail on the first try.
+# 2) The last namepart of BOOST_ROOT matches the
+# folder name internal to boost's .tar.gz
+# When testing you can force a boost build by clearing travis caches:
+# https://travis-ci.org/ripple/rippled/caches
+set -exu
+
+odir=$(pwd)
+: ${BOOST_TOOLSET:=msvc-14.1}
+
+if [[ -d "$BOOST_ROOT/lib" || -d "${BOOST_ROOT}/stage/lib" ]] ; then
+    echo "Using cached boost at $BOOST_ROOT"
+    exit
+fi
+
+#fetch/unpack:
+fn=$(basename -- "$BOOST_URL")
+ext="${fn##*.}"
+wopt="--quiet"
+wget ${wopt} $BOOST_URL -O /tmp/boost.tar.${ext} || \
+  ( [ -n "${BOOST_URL2}" ] && \
+    wget ${wopt} $BOOST_URL2 -O /tmp/boost.tar.${ext} ) || \
+  ( [ -n "${BOOST_WGET_OPTIONS}" ] &&
+    ( wget ${wopt} ${BOOST_WGET_OPTIONS} $BOOST_URL -O /tmp/boost.tar.${ext} || \
+      ( [ -n "${BOOST_URL2}" ] && \
+        wget ${wopt} ${BOOST_WGET_OPTIONS} $BOOST_URL2 -O /tmp/boost.tar.${ext} )
+    )
+  )
+cd $(dirname $BOOST_ROOT)
+rm -fr ${BOOST_ROOT}
+mkdir ${BOOST_ROOT}
+tar xf /tmp/boost.tar.${ext} -C ${BOOST_ROOT} --strip-components 1
+cd $BOOST_ROOT
+
+BLDARGS=()
+if [[ ${BOOST_BUILD_ALL:-false} == "true" ]]; then
+    # we never need boost-python...so even for ALL
+    # option we can skip it
+    BLDARGS+=(--without-python)
+else
+    BLDARGS+=(--with-chrono)
+    BLDARGS+=(--with-container)
+    BLDARGS+=(--with-context)
+    BLDARGS+=(--with-coroutine)
+    BLDARGS+=(--with-date_time)
+    BLDARGS+=(--with-filesystem)
+    BLDARGS+=(--with-program_options)
+    BLDARGS+=(--with-regex)
+    BLDARGS+=(--with-system)
+    BLDARGS+=(--with-atomic)
+    BLDARGS+=(--with-thread)
+fi
+BLDARGS+=(-j$((2*${NUM_PROCESSORS:-2})))
+BLDARGS+=(--prefix=${BOOST_ROOT}/_INSTALLED_)
+BLDARGS+=(-d0) # suppress messages/output
+
+if [[ -z ${COMSPEC:-} ]]; then
+    if [[ "$(uname)" == "Darwin" ]] ; then
+        BLDARGS+=(cxxflags="-std=c++14 -fvisibility=default")
+    else
+        BLDARGS+=(cxxflags="-std=c++14")
+        BLDARGS+=(runtime-link="static,shared")
+    fi
+    BLDARGS+=(--layout=tagged)
+    ./bootstrap.sh
+    ./b2 "${BLDARGS[@]}" stage
+    ./b2 "${BLDARGS[@]}" install
+else
+    BLDARGS+=(runtime-link="static,shared")
+    BLDARGS+=(--layout=versioned)
+    BLDARGS+=(--toolset="${BOOST_TOOLSET}")
+    BLDARGS+=(address-model=64)
+    BLDARGS+=(architecture=x86)
+    BLDARGS+=(link=static)
+    BLDARGS+=(threading=multi)
+    cmd /E:ON /D /S /C"bootstrap.bat"
+    ./b2.exe "${BLDARGS[@]}" stage
+    ./b2.exe "${BLDARGS[@]}" install
+fi
+
+if [[ ${CI:-false} == "true" ]]; then
+    # save some disk space...these are mostly
+    # obj files and don't need to be kept in CI contexts
+    rm -rf bin.v2
+fi
+
+cd $odir
+

Builds/containers/shared/install_cmake.sh

@@ -21,13 +21,8 @@ if [[ "${installed}" != "" && ${installed} =~ ${cm_maj}.${cm_min}.${cm_rel} ]] ;
     echo "cmake already installed: ${installed}"
     exit
 fi
-# From CMake 20+ "Linux" is lowercase so using `uname` won't create be the correct path
-if [ ${cm_min} -gt 19 ]; then
-    linux="linux"
-else
-    linux=$(uname)
-fi
-pkgname="cmake-${cm_maj}.${cm_min}.${cm_rel}-${linux}-x86_64.tar.gz"
+
+pkgname="cmake-${cm_maj}.${cm_min}.${cm_rel}-$(uname)-x86_64.tar.gz"
 tmppkg="/tmp/cmake.tar.gz"
 wget --quiet https://cmake.org/files/v${cm_maj}.${cm_min}/${pkgname} -O ${tmppkg}
 mkdir -p ${CMAKE_ROOT}
@@ -35,3 +30,5 @@ cd ${CMAKE_ROOT}
 tar --strip-components 1 -xf ${tmppkg}
 rm -f ${tmppkg}
 echo "installed: $(cmake_version)"
+
+

Builds/containers/shared/rippled-reporting.service

@@ -1,15 +0,0 @@
-[Unit]
-Description=Ripple Daemon
-After=network-online.target
-Wants=network-online.target
-
-[Service]
-Type=simple
-ExecStart=/opt/rippled-reporting/bin/rippled-reporting --silent --conf /etc/opt/rippled-reporting/rippled-reporting.cfg
-Restart=on-failure
-User=rippled-reporting
-Group=rippled-reporting
-LimitNOFILE=65536
-
-[Install]
-WantedBy=multi-user.target

Builds/containers/ubuntu-builder/Dockerfile

@@ -4,12 +4,33 @@ ARG GIT_COMMIT=unknown
 ARG CI_USE=false
 LABEL git-commit=$GIT_COMMIT
 
-WORKDIR /root
-COPY ubuntu-builder/ubuntu_setup.sh .
-RUN ./ubuntu_setup.sh && rm ubuntu_setup.sh
+# install/setup prerequisites:
+COPY ubuntu-builder/ubuntu_setup.sh /tmp/
+COPY shared/build_deps.sh /tmp/
+COPY shared/install_cmake.sh /tmp/
+COPY shared/install_boost.sh /tmp/
+RUN chmod +x /tmp/ubuntu_setup.sh && \
+    chmod +x /tmp/build_deps.sh && \
+    chmod +x /tmp/install_boost.sh && \
+    chmod +x /tmp/install_cmake.sh
+RUN /tmp/ubuntu_setup.sh
 
-RUN mkdir -m 777 -p /opt/rippled_bld/pkg/
+RUN /tmp/install_cmake.sh 3.16.1 /opt/local/cmake-3.16
+RUN ln -s /opt/local/cmake-3.16 /opt/local/cmake
+ENV PATH="/opt/local/cmake/bin:$PATH"
+# also install min supported cmake for testing
+RUN if [ "${CI_USE}" = true ] ; then /tmp/install_cmake.sh 3.9.0 /opt/local/cmake-3.9; fi
+
+RUN /tmp/build_deps.sh
+ENV PLANTUML_JAR="/opt/plantuml/plantuml.jar"
+ENV BOOST_ROOT="/opt/local/boost/_INSTALLED_"
+ENV OPENSSL_ROOT="/opt/local/openssl"
+
+# prep files for package building
+RUN mkdir -m 777 -p /opt/rippled_bld/pkg/debian
+RUN update-alternatives --set gcc /usr/bin/gcc-8
 WORKDIR /opt/rippled_bld/pkg
 
 COPY packaging/dpkg/build_dpkg.sh ./
 CMD ./build_dpkg.sh
+

Builds/containers/ubuntu-builder/ubuntu_setup.sh

@@ -1,76 +1,189 @@
 #!/usr/bin/env bash
+set -ex
 
-set -o errexit
-set -o nounset
-set -o xtrace
+source /etc/os-release
 
-# Parameters
+if [[ ${VERSION_ID} =~ ^18\. || ${VERSION_ID} =~ ^16\. ]] ; then
+    echo "setup for ${PRETTY_NAME}"
+else
+    echo "${VERSION} not supported"
+    exit 1
+fi
 
-gcc_version=${GCC_VERSION:-10}
-cmake_version=${CMAKE_VERSION:-3.25.1}
-conan_version=${CONAN_VERSION:-1.59}
+export DEBIAN_FRONTEND="noninteractive"
+echo "Acquire::Retries 3;" > /etc/apt/apt.conf.d/80-retries
+echo "Acquire::http::Pipeline-Depth 0;" >> /etc/apt/apt.conf.d/80-retries
+echo "Acquire::http::No-Cache true;" >> /etc/apt/apt.conf.d/80-retries
+echo "Acquire::BrokenProxy    true;" >> /etc/apt/apt.conf.d/80-retries
+apt-get update -o Acquire::CompressionTypes::Order::=gz
 
-apt update
-# Iteratively build the list of packages to install so that we can interleave
-# the lines with comments explaining their inclusion.
-dependencies=''
-# - to identify the Ubuntu version
-dependencies+=' lsb-release'
-# - for add-apt-repository
-dependencies+=' software-properties-common'
-# - to download CMake
-dependencies+=' curl'
-# - to build CMake
-dependencies+=' libssl-dev'
-# - Python headers for Boost.Python
-dependencies+=' python3-dev'
-# - to install Conan
-dependencies+=' python3-pip'
-# - to download rippled
-dependencies+=' git'
-# - CMake generators (but not CMake itself)
-dependencies+=' make ninja-build'
-apt install --yes ${dependencies}
+apt-get -y update
+apt-get -y install apt-utils
+apt-get -y install software-properties-common wget
+apt-get -y upgrade
+if [[ ${VERSION_ID} =~ ^18\. ]] ; then
+    apt-add-repository -y multiverse
+    apt-add-repository -y universe
+fi
+add-apt-repository -y ppa:ubuntu-toolchain-r/test
+apt-get -y clean
+apt-get -y update
 
-add-apt-repository --yes ppa:ubuntu-toolchain-r/test
-apt install --yes gcc-${gcc_version} g++-${gcc_version} \
-  debhelper debmake debsums gnupg dh-buildinfo dh-make dh-systemd cmake \
-  ninja-build zlib1g-dev make cmake ninja-build autoconf automake \
-  pkg-config apt-transport-https
+apt-get -y --fix-missing install \
+    make cmake ninja-build autoconf automake libtool pkg-config libtool \
+    openssl libssl-dev \
+    liblzma-dev libbz2-dev zlib1g-dev \
+    libjemalloc-dev \
+    python-pip \
+    gdb gdbserver \
+    libstdc++6 \
+    flex bison parallel \
+    libicu-dev texinfo \
+    java-common javacc \
+    dpkg-dev debhelper devscripts fakeroot \
+    debmake git-buildpackage dh-make gitpkg debsums gnupg \
+    dh-buildinfo dh-make dh-systemd \
+    apt-transport-https
 
-# Give us nice unversioned aliases for gcc and company.
+apt-get -y install gcc-7 g++-7
 update-alternatives --install \
-  /usr/bin/gcc gcc /usr/bin/gcc-${gcc_version} 100 \
-  --slave /usr/bin/g++ g++ /usr/bin/g++-${gcc_version} \
-  --slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-${gcc_version} \
-  --slave /usr/bin/gcc-nm gcc-nm /usr/bin/gcc-nm-${gcc_version} \
-  --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-${gcc_version} \
-  --slave /usr/bin/gcov gcov /usr/bin/gcov-${gcc_version} \
-  --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-dump-${gcc_version} \
-  --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-tool-${gcc_version}
+    /usr/bin/gcc gcc /usr/bin/gcc-7 40 \
+    --slave /usr/bin/g++ g++ /usr/bin/g++-7 \
+    --slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-7 \
+    --slave /usr/bin/gcc-nm gcc-nm /usr/bin/gcc-nm-7 \
+    --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-7 \
+    --slave /usr/bin/gcov gcov /usr/bin/gcov-7 \
+    --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-dump-7 \
+    --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-tool-7
+
+apt-get -y install gcc-8 g++-8
+update-alternatives --install \
+    /usr/bin/gcc gcc /usr/bin/gcc-8 20 \
+    --slave /usr/bin/g++ g++ /usr/bin/g++-8 \
+    --slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-8 \
+    --slave /usr/bin/gcc-nm gcc-nm /usr/bin/gcc-nm-8 \
+    --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-8 \
+    --slave /usr/bin/gcov gcov /usr/bin/gcov-8 \
+    --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-dump-8 \
+    --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-tool-8
 update-alternatives --auto gcc
 
-# Download and unpack CMake.
-cmake_slug="cmake-${cmake_version}"
-curl --location --remote-name \
-  "https://github.com/Kitware/CMake/releases/download/v${cmake_version}/${cmake_slug}.tar.gz"
-tar xzf ${cmake_slug}.tar.gz
-rm ${cmake_slug}.tar.gz
+update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-7 40
+update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-8 20
+update-alternatives --auto cpp
 
-# Build and install CMake.
-cd ${cmake_slug}
-./bootstrap --parallel=$(nproc)
-make --jobs $(nproc)
-make install
-cd ..
-rm --recursive --force ${cmake_slug}
+if [ "${CI_USE}" = true ] ; then
+    apt-get -y install gcc-6 g++-6
+    update-alternatives --install \
+        /usr/bin/gcc gcc /usr/bin/gcc-6 10 \
+        --slave /usr/bin/g++ g++ /usr/bin/g++-6 \
+        --slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-6 \
+        --slave /usr/bin/gcc-nm gcc-nm /usr/bin/gcc-nm-6 \
+        --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-6 \
+        --slave /usr/bin/gcov gcov /usr/bin/gcov-6 \
+        --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-dump-6 \
+        --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-tool-6
 
-# Install Conan.
-pip3 install conan==${conan_version}
+    apt-get -y install gcc-9 g++-9
+    update-alternatives --install \
+        /usr/bin/gcc gcc /usr/bin/gcc-9 15 \
+        --slave /usr/bin/g++ g++ /usr/bin/g++-9 \
+        --slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-9 \
+        --slave /usr/bin/gcc-nm gcc-nm /usr/bin/gcc-nm-9 \
+        --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-9 \
+        --slave /usr/bin/gcov gcov /usr/bin/gcov-9 \
+        --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-dump-9 \
+        --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-tool-9
+fi
+
+if [[ ${VERSION_ID} =~ ^18\. ]] ; then
+    apt-get -y install binutils
+elif [[ ${VERSION_ID} =~ ^16\. ]] ; then
+    apt-get -y install python-software-properties  binutils-gold
+fi
+
+wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
+if [[ ${VERSION_ID} =~ ^18\. ]] ; then
+    cat << EOF > /etc/apt/sources.list.d/llvm.list
+deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic main
+deb-src http://apt.llvm.org/bionic/ llvm-toolchain-bionic main
+deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-7 main
+deb-src http://apt.llvm.org/bionic/ llvm-toolchain-bionic-7 main
+deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-8 main
+deb-src http://apt.llvm.org/bionic/ llvm-toolchain-bionic-8 main
+deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-9 main
+deb-src http://apt.llvm.org/bionic/ llvm-toolchain-bionic-9 main
+EOF
+elif [[ ${VERSION_ID} =~ ^16\. ]] ; then
+    cat << EOF > /etc/apt/sources.list.d/llvm.list
+deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial main
+deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial main
+deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-7 main
+deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-7 main
+deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main
+deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main
+deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-9 main
+deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-9 main
+EOF
+fi
+apt-get -y update
+
+apt-get -y install \
+    clang-7 libclang-common-7-dev libclang-7-dev libllvm7 llvm-7 \
+    llvm-7-dev llvm-7-runtime clang-format-7 python-clang-7 \
+    lld-7 libfuzzer-7-dev libc++-7-dev
+update-alternatives --install \
+    /usr/bin/clang clang /usr/bin/clang-7 40 \
+    --slave /usr/bin/clang++ clang++ /usr/bin/clang++-7 \
+    --slave /usr/bin/llvm-profdata llvm-profdata /usr/bin/llvm-profdata-7 \
+    --slave /usr/bin/asan-symbolize asan-symbolize /usr/bin/asan_symbolize-7 \
+    --slave /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-7 \
+    --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-7 \
+    --slave /usr/bin/llvm-ar llvm-ar /usr/bin/llvm-ar-7 \
+    --slave /usr/bin/llvm-cov llvm-cov /usr/bin/llvm-cov-7 \
+    --slave /usr/bin/llvm-nm llvm-nm /usr/bin/llvm-nm-7
+apt-get -y install \
+    clang-8 libclang-common-8-dev libclang-8-dev libllvm8 llvm-8 \
+    llvm-8-dev llvm-8-runtime clang-format-8 python-clang-8 \
+    lld-8 libfuzzer-8-dev libc++-8-dev
+update-alternatives --install \
+    /usr/bin/clang clang /usr/bin/clang-8 20 \
+    --slave /usr/bin/clang++ clang++ /usr/bin/clang++-8 \
+    --slave /usr/bin/llvm-profdata llvm-profdata /usr/bin/llvm-profdata-8 \
+    --slave /usr/bin/asan-symbolize asan-symbolize /usr/bin/asan_symbolize-8 \
+    --slave /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-8 \
+    --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-8 \
+    --slave /usr/bin/llvm-ar llvm-ar /usr/bin/llvm-ar-8 \
+    --slave /usr/bin/llvm-cov llvm-cov /usr/bin/llvm-cov-8 \
+    --slave /usr/bin/llvm-nm llvm-nm /usr/bin/llvm-nm-8
+update-alternatives --auto clang
+
+if [ "${CI_USE}" = true ] ; then
+    apt-get -y install \
+        clang-9 libclang-common-9-dev libclang-9-dev libllvm9 llvm-9 \
+        llvm-9-dev llvm-9-runtime clang-format-9 python-clang-9 \
+        lld-9 libfuzzer-9-dev libc++-9-dev
+    update-alternatives --install \
+        /usr/bin/clang clang /usr/bin/clang-9 20 \
+        --slave /usr/bin/clang++ clang++ /usr/bin/clang++-9 \
+        --slave /usr/bin/llvm-profdata llvm-profdata /usr/bin/llvm-profdata-9 \
+        --slave /usr/bin/asan-symbolize asan-symbolize /usr/bin/asan_symbolize-9 \
+        --slave /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-9 \
+        --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-9 \
+        --slave /usr/bin/llvm-ar llvm-ar /usr/bin/llvm-ar-9 \
+        --slave /usr/bin/llvm-cov llvm-cov /usr/bin/llvm-cov-9 \
+        --slave /usr/bin/llvm-nm llvm-nm /usr/bin/llvm-nm-9
+
+    # only install latest lldb
+    apt-get -y install lldb-9 python-lldb-9 liblldb-9-dev
+    update-alternatives --install \
+        /usr/bin/lldb lldb /usr/bin/lldb-9 50 \
+        --slave /usr/bin/lldb-server lldb-server /usr/bin/lldb-server-9 \
+        --slave /usr/bin/lldb-argdumper lldb-argdumper /usr/bin/lldb-argdumper-9 \
+        --slave /usr/bin/lldb-instr lldb-instr /usr/bin/lldb-instr-9 \
+        --slave /usr/bin/lldb-mi lldb-mi /usr/bin/lldb-mi-9
+    update-alternatives --auto clang
+fi
+
+apt-get -y autoremove
 
-conan profile new --detect gcc
-conan profile update settings.compiler=gcc gcc
-conan profile update settings.compiler.version=${gcc_version} gcc
-conan profile update settings.compiler.libcxx=libstdc++11 gcc
-conan profile update env.CC=/usr/bin/gcc gcc
-conan profile update env.CXX=/usr/bin/g++ gcc

Builds/levelization/results/loops.txt

@@ -11,10 +11,10 @@ Loop: ripple.app ripple.nodestore
   ripple.app > ripple.nodestore
 
 Loop: ripple.app ripple.overlay
-  ripple.overlay ~= ripple.app
+  ripple.overlay == ripple.app
 
 Loop: ripple.app ripple.peerfinder
-  ripple.app > ripple.peerfinder
+  ripple.peerfinder ~= ripple.app
 
 Loop: ripple.app ripple.rpc
   ripple.rpc > ripple.app

Builds/levelization/results/ordering.txt

@@ -6,6 +6,7 @@ ripple.app > ripple.crypto
 ripple.app > ripple.json
 ripple.app > ripple.protocol
 ripple.app > ripple.resource
+ripple.app > ripple.server
 ripple.app > test.unit_test
 ripple.basics > ripple.beast
 ripple.conditions > ripple.basics
@@ -113,6 +114,7 @@ test.consensus > ripple.basics
 test.consensus > ripple.beast
 test.consensus > ripple.consensus
 test.consensus > ripple.ledger
+test.consensus > ripple.rpc
 test.consensus > test.csf
 test.consensus > test.toplevel
 test.consensus > test.unit_test

Builds/linux/README.md

@@ -1 +1,241 @@
-[Build instructions are currently located in `BUILD.md`](../../BUILD.md)
+# Linux Build Instructions
+
+This document focuses on building rippled for development purposes under recent
+Ubuntu linux distributions. To build rippled for Redhat, Fedora or Centos
+builds, including docker based builds for those distributions, please consult
+the [rippled-package-builder](https://github.com/ripple/rippled-package-builder)
+repository. 
+
+Note: Ubuntu 16.04 users may need to update their compiler (see the dependencies
+section). For non Ubuntu distributions, the steps below should work be
+installing the appropriate dependencies using that distribution's package
+management tools.
+
+## Dependencies
+
+gcc-8 or later is required.
+
+Use `apt-get` to install the dependencies provided by the distribution
+
+```
+$ apt-get update
+$ apt-get install -y gcc g++ wget git cmake pkg-config libprotoc-dev protobuf-compiler libprotobuf-dev libssl-dev
+```
+
+To build the software in reporting mode, install these additional dependencies:
+```
+$ apt-get install -y autoconf flex bison
+```
+
+Advanced users can choose to install newer versions of gcc, or the clang compiler.
+At this time, rippled only supports protobuf version 2. Using version 3 of 
+protobuf will give errors.
+
+### Build Boost
+
+Boost 1.70 or later is required. We recommend downloading and compiling boost
+with the following process: After changing to the directory where
+you wish to download and compile boost, run
+``` 
+$ wget https://boostorg.jfrog.io/artifactory/main/release/1.70.0/source/boost_1_70_0.tar.gz
+$ tar -xzf boost_1_70_0.tar.gz
+$ cd boost_1_70_0
+$ ./bootstrap.sh
+$ ./b2 headers
+$ ./b2 -j<Num Parallel>
+```
+
+### (Optional) Dependencies for Building Source Documentation
+
+Source code documentation is not required for running/debugging rippled. That
+said, the documentation contains some helpful information about specific
+components of the application. For more information on how to install and run
+the necessary components, see [this document](../../docs/README.md)
+
+## Build
+
+### Clone the rippled repository
+
+From a shell:
+
+```
+git clone git@github.com:ripple/rippled.git
+cd rippled
+```
+
+For a stable release, choose the `master` branch or one of the tagged releases
+listed on [GitHub](https://github.com/ripple/rippled/releases). 
+
+```
+git checkout master
+```
+
+or to test the latest release candidate, choose the `release` branch.
+
+```
+git checkout release
+```
+
+If you are doing development work and want the latest set of untested
+features, you can consider using the `develop` branch instead.
+
+```
+git checkout develop
+```
+
+### Configure Library Paths
+
+If you didn't persistently set the `BOOST_ROOT` environment variable to the
+directory in which you compiled boost, then you should set it temporarily.
+
+For example, you built Boost in your home directory `~/boost_1_70_0`, you
+would do for any shell in which you want to build:
+
+```
+export BOOST_ROOT=~/boost_1_70_0
+```
+
+Alternatively, you can add `DBOOST_ROOT=~/boost_1_70_0` to the command line when
+invoking `cmake`.
+
+### Generate Configuration
+
+All builds should be done in a separate directory from the source tree root 
+(a subdirectory is fine). For example, from the root of the ripple source tree:
+
+```
+mkdir my_build
+cd my_build
+```
+
+followed by:
+
+```
+cmake -DCMAKE_BUILD_TYPE=Debug ..
+```
+
+If your operating system does not provide static libraries (Arch Linux, and 
+Manjaro Linux, for example), you must configure a non-static build by adding
+`-Dstatic=OFF` to the above cmake line.
+
+`CMAKE_BUILD_TYPE` can be changed as desired for `Debug` vs.
+`Release` builds (all four standard cmake build types are supported).
+
+To select a different compiler (most likely gcc will be found by default), pass 
+`-DCMAKE_C_COMPILER=<path/to/c-compiler>` and
+`-DCMAKE_CXX_COMPILER=</path/to/cxx-compiler>` when configuring. If you prefer, 
+you can instead set `CC` and `CXX` environment variables which cmake will honor.
+
+#### Options During Configuration:
+
+The CMake file defines a number of configure-time options which can be
+examined by running `cmake-gui` or `ccmake` to generated the build. In
+particular, the `unity` option allows you to select between the unity and
+non-unity builds. `unity` builds are faster to compile since they combine
+multiple sources into a single compiliation unit - this is the default if you
+don't specify. `nounity` builds can be helpful for detecting include omissions
+or for finding other build-related issues, but aren't generally needed for
+testing and running.
+
+* `-Dunity=ON` to enable/disable unity builds (defaults to ON)  
+* `-Dassert=ON` to enable asserts
+* `-Djemalloc=ON` to enable jemalloc support for heap checking
+* `-Dsan=thread` to enable the thread sanitizer with clang
+* `-Dsan=address` to enable the address sanitizer with clang
+* `-Dstatic=ON` to enable static linking library dependencies
+* `-Dreporting=ON` to build code necessary for reporting mode (defaults to OFF)
+
+Several other infrequently used options are available - run `ccmake` or
+`cmake-gui` for a list of all options.
+
+### Build
+
+Once you have generated the build system, you can run the build via cmake:
+
+```
+cmake --build . -- -j <parallel jobs>
+```
+
+the `-j` parameter in this example tells the build tool to compile several
+files in parallel. This value should be chosen roughly based on the number of
+cores you have available and/or want to use for building.
+
+When the build completes successfully, you will have a `rippled` executable in
+the current directory, which can be used to connect to the network (when
+properly configured) or to run unit tests.
+
+
+#### Optional Installation
+
+The rippled cmake build supports an installation target that will install
+rippled as well as a support library that can be used to sign transactions. In
+order to build and install the files, specify the `install` target when
+building, e.g.:
+
+```
+cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=/opt/local ..
+cmake --build . --target install -- -j <parallel jobs>
+```
+
+We recommend specifying `CMAKE_INSTALL_PREFIX` when configuring in order to
+explicitly control the install location for your files. Without this setting,
+cmake will typically install in `/usr/local`. It is also possible to "rehome"
+the installation by specifying the `DESTDIR` env variable during the install phase,
+e.g.:
+
+```
+DESTDIR=~/mylibs cmake --build . --target install -- -j <parallel jobs>
+```
+
+in which case, the files would be installed in the `CMAKE_INSTALL_PREFIX` within
+the specified `DESTDIR` path.
+
+#### Signing Library
+
+If you want to use the signing support library to create an application, there
+are two simple mechanisms with cmake + git that facilitate this.
+
+With either option below, you will have access to a library from the
+rippled project that you can link to in your own project's CMakeLists.txt, e.g.:
+
+```
+target_link_libraries (my-signing-app Ripple::xrpl_core)
+```
+
+##### Option 1: git submodules + add_subdirectory
+
+First, add the rippled repo as a submodule to your project repo:
+
+```
+git submodule add -b master https://github.com/ripple/rippled.git vendor/rippled
+```
+
+change the `vendor/rippled` path as desired for your repo layout. Furthermore,
+change the branch name if you want to track a different rippled branch, such
+as `develop`.
+   
+Second, to bring this submodule into your project, just add the rippled subdirectory:
+
+```
+add_subdirectory (vendor/rippled)
+```
+    
+##### Option 2: installed rippled + find_package
+
+First, follow the "Optional Installation" instructions above to
+build and install the desired version of rippled.
+
+To make use of the installed files, add the following to your CMakeLists.txt file:
+
+```
+set (CMAKE_MODULE_PATH /opt/local/lib/cmake/ripple ${CMAKE_MODULE_PATH})
+find_package(Ripple REQUIRED)
+```
+
+change the `/opt/local` module path above to match your chosen installation prefix.
+
+## Unit Tests (Recommended)
+
+`rippled` builds a set of unit tests into the server executable. To run these unit
+tests after building, pass the `--unittest` option to the compiled `rippled`
+executable. The executable will exit with summary info after running the unit tests.

Builds/macos/README.md

@@ -1 +1,3 @@
-[Build instructions are currently located in `BUILD.md`](../../BUILD.md)
+# macOS Build Instructions
+
+[Build and Run rippled on macOS](https://xrpl.org/build-run-rippled-macos.html)

CMakeLists.txt

@@ -1,19 +1,12 @@
-cmake_minimum_required(VERSION 3.16)
+cmake_minimum_required (VERSION 3.16)
 
-if(POLICY CMP0074)
+if (POLICY CMP0074)
   cmake_policy(SET CMP0074 NEW)
-endif()
-if(POLICY CMP0077)
-  cmake_policy(SET CMP0077 NEW)
-endif()
+endif ()
 
-# Fix "unrecognized escape" issues when passing CMAKE_MODULE_PATH on Windows.
-file(TO_CMAKE_PATH "${CMAKE_MODULE_PATH}" CMAKE_MODULE_PATH)
-list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/Builds/CMake")
-
-project(rippled)
+project (rippled)
 set(CMAKE_CXX_EXTENSIONS OFF)
-set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
 # make GIT_COMMIT_HASH define available to all sources
@@ -28,14 +21,22 @@ if(Git_FOUND)
     endif()
 endif() #git
 
-if(thread_safety_analysis)
+if (thread_safety_analysis)
   add_compile_options(-Wthread-safety -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS -DRIPPLE_ENABLE_THREAD_SAFETY_ANNOTATIONS)
   add_compile_options("-stdlib=libc++")
   add_link_options("-stdlib=libc++")
 endif()
 
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/Builds/CMake")
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/Builds/CMake/deps")
+
 include (CheckCXXCompilerFlag)
-include (FetchContent)
+if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.11)
+  include (FetchContent)
+endif ()
+if (MSVC AND CMAKE_VERSION VERSION_LESS 3.12)
+  message (FATAL_ERROR "MSVC requires cmake 3.12 or greater for proper boost support")
+endif ()
 include (ExternalProject)
 include (CMakeFuncs) # must come *after* ExternalProject b/c it overrides one function in EP
 include (ProcessorCount)
@@ -46,6 +47,7 @@ endif ()
 include(RippledSanity)
 include(RippledVersion)
 include(RippledSettings)
+include(RippledNIH)
 include(RippledRelease)
 # this check has to remain in the top-level cmake
 # because of the early return statement
@@ -58,66 +60,24 @@ endif ()
 include(RippledCompiler)
 include(RippledInterface)
 
-option(only_docs "Include only the docs target?" FALSE)
-include(RippledDocs)
-if(only_docs)
-  return()
-endif()
-
 ###
 
 include(deps/Boost)
-find_package(OpenSSL 1.1.1 REQUIRED)
-set_target_properties(OpenSSL::SSL PROPERTIES
-  INTERFACE_COMPILE_DEFINITIONS OPENSSL_NO_SSL2
-)
-add_subdirectory(src/secp256k1)
-add_subdirectory(src/ed25519-donna)
-find_package(lz4 REQUIRED)
-# Target names with :: are not allowed in a generator expression.
-# We need to pull the include directories and imported location properties
-# from separate targets.
-find_package(LibArchive REQUIRED)
-find_package(SOCI REQUIRED)
-find_package(SQLite3 REQUIRED)
-find_package(Snappy REQUIRED)
-
-option(rocksdb "Enable RocksDB" ON)
-if(rocksdb)
-  find_package(RocksDB REQUIRED)
-  set_target_properties(RocksDB::rocksdb PROPERTIES
-    INTERFACE_COMPILE_DEFINITIONS RIPPLE_ROCKSDB_AVAILABLE=1
-  )
-  target_link_libraries(ripple_libs INTERFACE RocksDB::rocksdb)
-endif()
-
-find_package(nudb REQUIRED)
-find_package(date REQUIRED)
+include(deps/OpenSSL)
+include(deps/Secp256k1)
+include(deps/Ed25519-donna)
+include(deps/Lz4)
+include(deps/Libarchive)
+include(deps/Sqlite)
+include(deps/Soci)
+include(deps/Snappy)
+include(deps/Rocksdb)
+include(deps/Nudb)
+include(deps/date)
 include(deps/Protobuf)
 include(deps/gRPC)
-
-target_link_libraries(ripple_libs INTERFACE
-  ed25519::ed25519
-  LibArchive::LibArchive
-  lz4::lz4
-  nudb::core
-  OpenSSL::Crypto
-  OpenSSL::SSL
-  Ripple::grpc_pbufs
-  Ripple::pbufs
-  secp256k1::secp256k1
-  soci::soci
-  SQLite::SQLite3
-)
-
-if(reporting)
-  find_package(cassandra-cpp-driver REQUIRED)
-  find_package(PostgreSQL REQUIRED)
-  target_link_libraries(ripple_libs INTERFACE
-    cassandra-cpp-driver::cassandra-cpp-driver
-    PostgreSQL::PostgreSQL
-  )
-endif()
+include(deps/cassandra)
+include(deps/Postgres)
 
 ###
 
@@ -125,4 +85,5 @@ include(RippledCore)
 include(RippledInstall)
 include(RippledCov)
 include(RippledMultiConfig)
+include(RippledDocs)
 include(RippledValidatorKeys)

CONTRIBUTING.md

@@ -1,184 +0,0 @@
-The XRP Ledger has many and diverse stakeholders, and everyone deserves
-a chance to contribute meaningful changes to the code that runs the XRPL.
-
-# Contributing
-
-We assume you are familiar with the general practice of [making contributions
-on GitHub][1].
-This file includes only special instructions specific to this project.
-
-
-## Before you start
-
-All of your contributions must be developed in your personal
-[fork](https://github.com/XRPLF/rippled/fork).
-No personal branches may ever be pushed to the [main project][rippled].
-These are the only branches that may ever exist in the main project:
-
-- `develop`: The latest set of unreleased features, and the most common
-    starting point for contributions.
-- `release`: The latest release candidate.
-- `master`: The latest stable release.
-- `gh-pages`: The documentation for this project, built by Doxygen.
-
-The tip of each branch must be signed.
-In order for GitHub to sign a squashed commit that it builds from your pull
-request,
-all of your commits must be signed,
-and GitHub must know your verifying key.
-Please walk through the excellent documentation from GitHub to set
-up [signature verification][signing].
-
-[rippled]: https://github.com/XRPLF/rippled
-[signing]: https://docs.github.com/en/authentication/managing-commit-signature-verification/about-commit-signature-verification
-
-
-## Major contributions
-
-If your contribution is a major feature or breaking change,
-then you must first write an XRP Ledger Standard (XLS) describing it.
-Go to [XRPL-Standards](https://github.com/XRPLF/XRPL-Standards/discussions),
-choose the next available standard number, and
-open a discussion with an appropriate title to propose your draft standard.
-
-When you submit a pull request, please link the corresponding XLS in the
-description.
-An XLS still in draft status is considered a work-in-progress and open for
-discussion.
-Please do not submit a pull request before allowing due time for questions,
-suggestions, and changes to the XLS draft.
-It is the responsibility of the XLS author to update the draft to match the
-final implementation when its corresponding pull request is merged.
-
-
-## Before making a pull request
-
-Changes that alter transaction processing must be guarded by an
-[Amendment](https://xrpl.org/amendments.html).
-All other changes that maintain the existing behavior do not need an
-Amendment.
-
-Ensure that your code compiles according to the build instructions in
-[`BUILD.md`](./BUILD.md).
-If you create new source files, they must go under `src/ripple`.
-You will need to add them to one of the
-[source lists](./Builds/CMake/RippledCore.cmake) in CMake.
-
-Please write tests for your code.
-If you create new test source files, they must go under `src/test`.
-You will need to add them to one of the
-[source lists](./Builds/CMake/RippledCore.cmake) in CMake.
-If your test can be run offline, in under 60 seconds, then it can be an
-automatic test run by `rippled --unittest`.
-Otherwise, it must be a manual test.
-
-The source must be formatted according to the style guide below.
-
-Header includes must be [levelized](./Builds/levelization).
-
-
-## Pull requests
-
-Pull requests must target the `develop` branch.[^1]
-
-[^1]: There are exceptions to this policy for hotfixes, but no one consulting
-  this document will be in that situation.
-
-Changes to pull requests must be added as new commits.
-You may **never force push a branch in a pull request** (e.g. after a rebase).
-This preserves the ability for reviewers to filter changes since their last
-review.
-
-A pull request must obtain **approvals from at least two reviewers** before it
-can be considered for merge by a Maintainer.
-Maintainers retain discretion to require more approvals if they feel the
-credibility of the existing approvals is insufficient.
-
-Pull requests must be merged by [squash-and-merge][2]
-to preserve a linear history for the `develop` branch.
-
-
-# Style guide
-
-This is a non-exhaustive list of recommended style guidelines.
-These are not always strictly enforced and serve as a way to keep the codebase coherent rather than a set of _thou shalt not_ commandments.
-
-
-## Formatting
-
-All code must conform to `clang-format` version 10,
-according to the settings in [`.clang-format`](./.clang-format),
-unless the result would be unreasonably difficult to read or maintain.
-To demarcate lines that should be left as-is, surround them with comments like
-this:
-
-```
-// clang-format off
-...
-// clang-format on
-```
-
-You can format individual files in place by running `clang-format -i <file>...`
-from any directory within this project.
-
-
-## Avoid
-
-1. Proliferation of nearly identical code.
-2. Proliferation of new files and classes.
-3. Complex inheritance and complex OOP patterns.
-4. Unmanaged memory allocation and raw pointers.
-5. Macros and non-trivial templates (unless they add significant value.)
-6. Lambda patterns (unless these add significant value.)
-7. CPU or architecture-specific code unless there is a good reason to include it, and where it is used guard it with macros and provide explanatory comments.
-8. Importing new libraries unless there is a very good reason to do so.
-
-
-## Seek to
-
-9. Extend functionality of existing code rather than creating new code.
-10. Prefer readability over terseness where important logic is concerned.
-11. Inline functions that are not used or are not likely to be used elsewhere in the codebase.
-12. Use clear and self-explanatory names for functions, variables, structs and classes.
-13. Use TitleCase for classes, structs and filenames, camelCase for function and variable names, lower case for namespaces and folders.
-14. Provide as many comments as you feel that a competent programmer would need to understand what your code does.
-
-
-# Maintainers
-
-Maintainers are ecosystem participants with elevated access to the repository.
-They are able to push new code, make decisions on when a release should be
-made, etc.
-
-
-## Adding and removing
-
-New maintainers can be proposed by two existing maintainers, subject to a vote
-by a quorum of the existing maintainers.
-A minimum of 50% support and a 50% participation is required.
-In the event of a tie vote, the addition of the new maintainer will be
-rejected.
-
-Existing maintainers can resign, or be subject to a vote for removal at the
-behest of two existing maintainers.
-A minimum of 60% agreement and 50% participation are required.
-The XRP Ledger Foundation will have the ability, for cause, to remove an
-existing maintainer without a vote.
-
-
-## Current Maintainers
-
-* [JoelKatz](https://github.com/JoelKatz) (Ripple)
-* [manojsdoshi](https://github.com/manojsdoshi) (Ripple)
-* [n3tc4t](https://github.com/n3tc4t) (XRPL Labs)
-* [Nik Bougalis](https://github.com/nbougalis)
-* [nixer89](https://github.com/nixer89) (XRP Ledger Foundation)
-* [RichardAH](https://github.com/RichardAH) (XRPL Labs + XRP Ledger Foundation)
-* [seelabs](https://github.com/seelabs) (Ripple)
-* [Silkjaer](https://github.com/Silkjaer) (XRP Ledger Foundation)
-* [WietseWind](https://github.com/WietseWind) (XRPL Labs + XRP Ledger Foundation)
-* [Ed Hennis](https://github.com/ximinez) (Ripple)
-
-
-[1]: https://docs.github.com/en/get-started/quickstart/contributing-to-projects
-[2]: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/incorporating-changes-from-a-pull-request/about-pull-request-merges#squash-and-merge-your-commits

README.md

@@ -1,3 +1,97 @@
+# XRP Ledger Side chain Branch
+
+## Warning
+
+This is not the main branch of the XRP Ledger. This branch supports side chains
+on the XRP ledger and integrates an implementation of side chain federators.
+This is a developers prelease and it should not be used for production or to
+transfer real value. Consider this "alpha" quality software. There will be bugs.
+See "Status" for a fuller description.
+
+The latest production code for the XRP ledger is on the "master" branch.
+
+Until this branch is merged with the mainline branch, it will periodically be
+rebased on that branch and force pushed to github.
+
+## What are side chains?
+
+Side chains are independent ledgers. They can have their own transaction types,
+their own UNL list (or even their own consensus algorithm), and their own set of
+other unique features (like a smart contracts layer). What's special about them
+is there is a way to transfer assets from the XRP ledger to the side chain, and
+a way to return those assets back from the side chain to the XRP ledger. Both
+XRP and issued assets may be exchanged.
+
+The motivation for creating a side chain is to implement an idea that may not be
+a great fit for the main XRP ledger, or may take a long time before such a
+feature is adopted by the main XRP ledger. The advantage of a side chain over a
+brand new ledger is it allows the side chain to immediate use tokens with real
+monetary value.
+
+This implementation is meant to support side chains that are similar to the XRP
+ledger and use the XRP ledger as the main chain. The idea is to develop a new
+side chain, first this code will be forked and the new features specific to the
+new chain will be implemented.
+
+## Status
+
+All the functionality needed to build side chains should be complete. However,
+it has not been well tested or polished.
+
+In particular, all of the following are built:
+
+* Cross chain transactions for both XRP and Issued Assets
+* Refunds if transactions fail
+* Allowing federators to rejoin a network
+* Detecting and handling when federators fall too far behind in processing
+  transactions
+* A python library to easily create configuration files for testing side chains
+  and spin up side chains on a local machine
+* Python scripts to test side chains
+* An interactive shell to explore side chain functionality
+
+The biggest missing pieces are:
+
+* Testing: While the functionality is there, it has just begun to be tested.
+  There will be bugs. Even horrible and embarrassing bugs. Of course, this will
+  improve as testing progresses.
+
+* Tooling: There is a python library and an interactive shell that was built to
+  help development. However, these tools are geared to run a test network on a
+  local machine. They are not geared to new users or to production systems.
+  Better tooling is coming.
+
+* Documentation: There is documentation that describes the technical details of
+  how side chains works, how to run the python scripts to set up side chains on
+  the local machine, and the changes to the configuration files. However, like
+  the tooling, this is not geared to new users or production systems. Better
+  documentation is coming. In particular, good documentation for how to set up a
+  production side chain - or even a test net that doesn't run on a local
+  machine - needs to be written.
+
+## Getting Started
+
+See the instructions [here](docs/sidechain/GettingStarted.md) for how to
+run an interactive shell that will spin up a set of federators on your local
+machine and allow you to transfer assets between the main chain and a side
+chain.
+
+After setting things up and completing a cross chain transaction with the
+"getting started" script above, it may to useful to browse some other
+documentation:
+
+* [This](bin/sidechain/python/README.md) document describes the scripts and
+  python modules used to test and explore side chains on your local machine.
+  
+* [This](docs/sidechain/configFile.md) document describes the new stanzas in the
+  config file needed for side chains.
+  
+* [This](docs/sidechain/federatorAccountSetup.md) document describes how to set
+  up the federator accounts if not using the python scripts.
+
+* [This](docs/sidechain/design.md) document describes the low-level details for
+  how side chains work.
+
 # The XRP Ledger
 
 The [XRP Ledger](https://xrpl.org/) is a decentralized cryptographic ledger powered by a network of peer-to-peer nodes. The XRP Ledger uses a novel Byzantine Fault Tolerant consensus algorithm to settle and record transactions in a secure distributed database without a central operator.
@@ -10,7 +104,9 @@ The server software that powers the XRP Ledger is called `rippled` and is availa
 
 ### Build from Source
 
-* [Read the build instructions in `BUILD.md`](BUILD.md)
+* [Linux](Builds/linux/README.md)
+* [Mac](Builds/macos/README.md) (Not recommended for production)
+* [Windows](Builds/VisualStudio2017/README.md) (Not recommended for production)
 
 ## Key Features of the XRP Ledger
 
@@ -32,12 +128,8 @@ The server software that powers the XRP Ledger is called `rippled` and is availa
 
 
 ## Source Code
-
-Here are some good places to start learning the source code:
-
-- Read the markdown files in the source tree: `src/ripple/**/*.md`.
-- Read [the levelization document](./Builds/levelization) to get an idea of the internal dependency graph.
-- In the big picture, the `main` function constructs an `ApplicationImp` object, which implements the `Application` virtual interface. Almost every component in the application takes an `Application&` parameter in its constructor, typically named `app` and stored as a member variable `app_`. This allows most components to depend on any other component.
+[![travis-ci.com: Build Status](https://travis-ci.com/ripple/rippled.svg?branch=develop)](https://travis-ci.com/ripple/rippled)
+[![codecov.io: Code Coverage](https://codecov.io/gh/ripple/rippled/branch/develop/graph/badge.svg)](https://codecov.io/gh/ripple/rippled)
 
 ### Repository Contents
 
@@ -57,6 +149,5 @@ git-subtree. See those directories' README files for more details.
 
 * [XRP Ledger Dev Portal](https://xrpl.org/)
 * [Setup and Installation](https://xrpl.org/install-rippled.html)
-* [Source Documentation (Doxygen)](https://xrplf.github.io/rippled/)
-* [Mailing List for Release Announcements](https://groups.google.com/g/ripple-server)
+* [Source Documentation (Doxygen)](https://ripple.github.io/rippled)
 * [Learn more about the XRP Ledger (YouTube)](https://www.youtube.com/playlist?list=PLJQ55Tj1hIVZtJ_JdTvSum2qMTsedWkNi)

RELEASENOTES.md

@@ -5,428 +5,7 @@
 This document contains the release notes for `rippled`, the reference server implementation of the XRP Ledger protocol. To learn more about how to build, run or update a `rippled` server, visit https://xrpl.org/install-rippled.html
 
  
-Have new ideas? Need help with setting up your node? [Please open an issue here](https://github.com/xrplf/rippled/issues/new/choose).
-
-
-# Introducing XRP Ledger version 1.10.1
-
-Version 1.10.1 of `rippled`, the reference server implementation of the XRP Ledger protocol, is now available. This release restores packages for Ubuntu 18.04.
-
-Compared to version 1.10.0, the only C++ code change fixes an edge case in Reporting Mode.
-
-If you are already running version 1.10.0, then upgrading to version 1.10.1 is generally not required.
-
-[Sign Up for Future Release Announcements](https://groups.google.com/g/ripple-server)
-
-<!-- BREAK -->
-
-## Install / Upgrade
-
-On supported platforms, see the [instructions on installing or updating `rippled`](https://xrpl.org/install-rippled.html).
-
-## Changelog
-
-- [`da18c86cbf`](https://github.com/ripple/rippled/commit/da18c86cbfea1d8fe6940035f9103e15890d47ce) Build packages with Ubuntu 18.04
-- [`f7b3ddd87b`](https://github.com/ripple/rippled/commit/f7b3ddd87b8ef093a06ab1420bea57ed1e77643a) Reporting Mode: Do not attempt to acquire missing data from peer network (#4458)
-
-### GitHub
-
-The public source code repository for `rippled` is hosted on GitHub at <https://github.com/XRPLF/rippled>.
-
-We welcome all contributions and invite everyone to join the community of XRP Ledger developers to help build the Internet of Value.
-
-### Credits
-
-The following people contributed directly to this release:
-
-- John Freeman <jfreeman08@gmail.com>
-- Mark Travis <mtrippled@users.noreply.github.com>
-- Michael Legleux <mlegleux@ripple.com>
-
-Bug Bounties and Responsible Disclosures:
-We welcome reviews of the rippled code and urge researchers to
-responsibly disclose any issues they may find.
-
-To report a bug, please send a detailed report to:
-
-    bugs@xrpl.org
-
-
-# Introducing XRP Ledger version 1.10.0
-
-Version 1.10.0 of `rippled`, the reference server implementation of the XRP Ledger protocol, is now available. This release introduces six new amendments, detailed below, and cleans up code to improve performance.
-
-[Sign Up for Future Release Announcements](https://groups.google.com/g/ripple-server)
-
-<!-- BREAK -->
-
-## Action Required
-
-Six new amendments are now open for voting according to the XRP Ledger's [amendment process](https://xrpl.org/amendments.html), which enables protocol changes following two weeks of >80% support from trusted validators.
-
-If you operate an XRP Ledger server, upgrade to version 1.10.0 by March 21 to ensure service continuity. The exact time that protocol changes take effect depends on the voting decisions of the decentralized network.
-
-
-## Install / Upgrade
-
-On supported platforms, see the [instructions on installing or updating `rippled`](https://xrpl.org/install-rippled.html).
-
-
-## New Amendments
-
-- **`featureImmediateOfferKilled`**: Changes the response code of an `OfferCreate` transaction with the `tfImmediateOrCancel` flag to return `tecKILLED` when no funds are moved. The previous return code of `tecSUCCESS` was unintuitive. [#4157](https://github.com/XRPLF/rippled/pull/4157)
-
-- **`featureDisallowIncoming`**: Enables an account to block incoming checks, payment channels, NFToken offers, and trust lines. [#4336](https://github.com/XRPLF/rippled/pull/4336)
-
-- **`featureXRPFees`**: Simplifies transaction cost calculations to use XRP directly, rather than calculating indirectly in "fee units" and translating the results to XRP. Updates all instances of "fee units" in the protocol and ledger data to be drops of XRP instead. [#4247](https://github.com/XRPLF/rippled/pull/4247) 
-
-- **`fixUniversalNumber`**: Simplifies and unifies the code for decimal floating point math. In some cases, this provides slightly better accuracy than the previous code, resulting in calculations whose least significant digits are different than when calculated with the previous code. The different results may cause other edge case differences where precise calculations are used, such as ranking of offers or processing of payments that use several different paths. [#4192](https://github.com/XRPLF/rippled/pull/4192)
-
-- **`fixNonFungibleTokensV1_2`**: This amendment is a combination of NFToken fixes. [#4417](https://github.com/XRPLF/rippled/pull/4417)
-  - Fixes unburnable NFTokens when it has over 500 offers. [#4346](https://github.com/XRPLF/rippled/pull/4346)
-  - Fixes 3 NFToken offer acceptance issues. [#4380](https://github.com/XRPLF/rippled/pull/4380)
-  - Prevents brokered sales of NFTokens to owners. [#4403](https://github.com/XRPLF/rippled/pull/4403)
-  - Only allows the destination to settle NFToken offers through brokerage. [#4399](https://github.com/XRPLF/rippled/pull/4399)
-
-- **`fixTrustLinesToSelf`**: Trust lines must be between two different accounts, but two exceptions exist because of a bug that briefly existed. This amendment removes those trust lines. [69bb2be](https://github.com/XRPLF/rippled/pull/4270/commits/69bb2be446e3cc24c694c0835b48bd2ecd3d119e)
-
-
-## Changelog
-
-
-### New Features and Improvements
-
-- **Improve Handshake in the peer protocol**: Switched to using a cryptographically secure PRNG for the Instance Cookie. `rippled` now uses hex encoding for the `Closed-Ledger` and `Previous-Ledger` fields in the Handshake. Also added `--newnodeid` and `--nodeid` command line options. [5a15229](https://github.com/XRPLF/rippled/pull/4270/commits/5a15229eeb13b69c8adf1f653b88a8f8b9480546)
-
-- **RPC tooBusy response now has 503 HTTP status code**: Added ripplerpc 3.0, enabling RPC tooBusy responses to return relevant HTTP status codes. This is a non-breaking change that only applies to JSON-RPC when you include `"ripplerpc": "3.0"` in the request. [#4143](https://github.com/XRPLF/rippled/pull/4143)
-
-- **Use the Conan package manager**: Added a `conanfile.py` and Conan recipe for Snappy. Removed the RocksDB recipe from the repo; you can now get it from Conan Center. [#4367](https://github.com/XRPLF/rippled/pull/4367), [c2b03fe](https://github.com/XRPLF/rippled/commit/c2b03fecca19a304b37467b01fa78593d3dce3fb)
-
-- **Update Build Instructions**: Updated the build instructions to build with the Conan package manager and restructured info for easier comprehension. [#4376](https://github.com/XRPLF/rippled/pull/4376), [#4383](https://github.com/XRPLF/rippled/pull/4383)
-
-- **Revise CONTRIBUTING**: Updated code contribution guidelines. `rippled` is an open source project and contributions are very welcome. [#4382](https://github.com/XRPLF/rippled/pull/4382)
-
-- **Update documented pathfinding configuration defaults**: `417cfc2` changed the default Path Finding configuration values, but missed updating the values documented in rippled-example.cfg. Updated those defaults and added recommended values for nodes that want to support advanced pathfinding. [#4409](https://github.com/XRPLF/rippled/pull/4409)
-
-- **Remove gRPC code previously used for the Xpring SDK**: Removed gRPC code used for the Xpring SDK. The gRPC API is also enabled locally by default in `rippled-example.cfg`. This API is used for [Reporting Mode](https://xrpl.org/build-run-rippled-in-reporting-mode.html) and [Clio](https://github.com/XRPLF/clio). [28f4cc7](https://github.com/XRPLF/rippled/pull/4321/commits/28f4cc7817c2e477f0d7e9ade8f07a45ff2b81f1)
-
-- **Switch from C++17 to C++20**: Updated `rippled` to use C++20. [92d35e5](https://github.com/XRPLF/rippled/pull/4270/commits/92d35e54c7de6bbe44ff6c7c52cc0765b3f78258)
-
-- **Support for Boost 1.80.0:**: [04ef885](https://github.com/XRPLF/rippled/pull/4321/commits/04ef8851081f6ee9176783ad3725960b8a931ebb)
-
-- **Reduce default reserves to 10/2**: Updated the hard-coded default reserves to match the current settings on Mainnet. [#4329](https://github.com/XRPLF/rippled/pull/4329)
-
-- **Improve self-signed certificate generation**: Improved speed and security of TLS certificate generation on fresh startup. [0ecfc7c](https://github.com/XRPLF/rippled/pull/4270/commits/0ecfc7cb1a958b731e5f184876ea89ae2d4214ee)
-
-
-### Bug Fixes
-
-- **Update command-line usage help message**: Added `manifest` and `validator_info` to the `rippled` CLI usage statement. [b88ed5a](https://github.com/XRPLF/rippled/pull/4270/commits/b88ed5a8ec2a0735031ca23dc6569d54787dc2f2)
-
-- **Work around gdb bug by changing a template parameter**: Added a workaround for a bug in gdb, where unsigned template parameters caused issues with RTTI. [#4332](https://github.com/XRPLF/rippled/pull/4332)
-
-- **Fix clang 15 warnings**: [#4325](https://github.com/XRPLF/rippled/pull/4325)
-
-- **Catch transaction deserialization error in doLedgerGrpc**: Fixed an issue in the gRPC API, so `Clio` can extract ledger headers and state objects from specific transactions that can't be deserialized by `rippled` code. [#4323](https://github.com/XRPLF/rippled/pull/4323)
-
-- **Update dependency: gRPC**: New Conan recipes broke the old version of gRPC, so the dependency was updated. [#4407](https://github.com/XRPLF/rippled/pull/4407)
-
-- **Fix Doxygen workflow**: Added options to build documentation that don't depend on the library dependencies of `rippled`. [#4372](https://github.com/XRPLF/rippled/pull/4372)
-
-- **Don't try to read SLE with key 0 from the ledger**: Fixed the `preclaim` function to check for 0 in `NFTokenSellOffer` and `NFTokenBuyOffer` before calling `Ledger::read`. This issue only affected debug builds. [#4351](https://github.com/XRPLF/rippled/pull/4351)
-
-- **Update broken link to hosted Doxygen content**: [5e1cb09](https://github.com/XRPLF/rippled/pull/4270/commits/5e1cb09b8892e650f6c34a66521b6b1673bd6b65)
-
-
-### Code Cleanup
-
-- **Prevent unnecessary `shared_ptr` copies by accepting a value in `SHAMapInnerNode::setChild`**: [#4266](https://github.com/XRPLF/rippled/pull/4266)
-
-- **Release TaggedCache object memory outside the lock**: [3726f8b](https://github.com/XRPLF/rippled/pull/4321/commits/3726f8bf31b3eab8bab39dce139656fd705ae9a0)
-
-- **Rename SHAMapStoreImp::stopping() to healthWait()**: [7e9e910](https://github.com/XRPLF/rippled/pull/4321/commits/7e9e9104eabbf0391a0837de5630af17a788e233)
-
-- **Improve wrapper around OpenSSL RAND**: [7b3507b](https://github.com/XRPLF/rippled/pull/4270/commits/7b3507bb873495a974db33c57a888221ddabcacc)
-
-- **Improve AccountID string conversion caching**: Improved memory cache usage. [e2eed96](https://github.com/XRPLF/rippled/pull/4270/commits/e2eed966b0ecb6445027e6a023b48d702c5f4832)
-
-- **Build the command map at compile time**: [9aaa0df](https://github.com/XRPLF/rippled/pull/4270/commits/9aaa0dff5fd422e5f6880df8e20a1fd5ad3b4424)
-
-- **Avoid unnecessary copying and dynamic memory allocations**: [d318ab6](https://github.com/XRPLF/rippled/pull/4270/commits/d318ab612adc86f1fd8527a50af232f377ca89ef)
-
-- **Use constexpr to check memo validity**: [e67f905](https://github.com/XRPLF/rippled/pull/4270/commits/e67f90588a9050162881389d7e7d1d0fb31066b0)
-
-- **Remove charUnHex**: [83ac141](https://github.com/XRPLF/rippled/pull/4270/commits/83ac141f656b1a95b5661853951ebd95b3ffba99)
-
-- **Remove deprecated AccountTxOld.cpp**: [ce64f7a](https://github.com/XRPLF/rippled/pull/4270/commits/ce64f7a90f99c6b5e68d3c3d913443023de061a6)
-
-- **Remove const_cast usage**: [23ce431](https://github.com/XRPLF/rippled/pull/4321/commits/23ce4318768b718c82e01004d23f1abc9a9549ff)
-
-- **Remove inaccessible code paths and outdated data format wchar_t**: [95fabd5](https://github.com/XRPLF/rippled/pull/4321/commits/95fabd5762a4917753c06268192e4d4e4baef8e4)
-
-- **Improve move semantics in Expected**: [#4326](https://github.com/XRPLF/rippled/pull/4326)
-
-
-### GitHub
-
-The public source code repository for `rippled` is hosted on GitHub at <https://github.com/XRPLF/rippled>.
-
-We welcome all contributions and invite everyone to join the community of XRP Ledger developers to help build the Internet of Value.
-
-### Credits
-
-The following people contributed directly to this release:
-
-- Alexander Kremer <akremer@ripple.com>
-- Alloy Networks <45832257+alloynetworks@users.noreply.github.com>
-- CJ Cobb <46455409+cjcobb23@users.noreply.github.com>
-- Chenna Keshava B S <ckbs.keshava56@gmail.com>
-- Crypto Brad Garlinghouse <cryptobradgarlinghouse@protonmail.com>
-- Denis Angell <dangell@transia.co>
-- Ed Hennis <ed@ripple.com>
-- Elliot Lee <github.public@intelliot.com>
-- Gregory Popovitch <greg7mdp@gmail.com>
-- Howard Hinnant <howard.hinnant@gmail.com>
-- J. Scott Branson <18340247+crypticrabbit@users.noreply.github.com>
-- John Freeman <jfreeman08@gmail.com>
-- ledhed2222 <ledhed2222@users.noreply.github.com>
-- Levin Winter <33220502+levinwinter@users.noreply.github.com>
-- manojsdoshi <mdoshi@ripple.com>
-- Nik Bougalis <nikb@bougalis.net>
-- RichardAH <richard.holland@starstone.co.nz>
-- Scott Determan <scott.determan@yahoo.com>
-- Scott Schurr <scott@ripple.com>
-- Shawn Xie <35279399+shawnxie999@users.noreply.github.com>
-
-Security Bug Bounty Acknowledgements:
-- Aaron Hook
-- Levin Winter
-
-Bug Bounties and Responsible Disclosures:
-We welcome reviews of the rippled code and urge researchers to
-responsibly disclose any issues they may find.
-
-To report a bug, please send a detailed report to:
-
-    bugs@xrpl.org
-
-
-# Introducing XRP Ledger version 1.9.4
-
-Version 1.9.4 of `rippled`, the reference implementation of the XRP Ledger protocol is now available. This release introduces an amendment that removes the ability for an NFT issuer to indicate that trust lines should be automatically created for royalty payments from secondary sales of NFTs, in response to a bug report that indicated how this functionality could be abused to mount a denial of service attack against the issuer.
-
-## Action Required
-
-This release introduces a new amendment to the XRP Ledger protocol, **`fixRemoveNFTokenAutoTrustLine`** to mitigate a potential denial-of-service attack against NFT issuers that minted NFTs and allowed secondary trading of those NFTs to create trust lines for any asset.
-
-This amendment is open for voting according to the XRP Ledger's [amendment process](https://xrpl.org/amendments.html), which enables protocol changes following two weeks of >80% support from trusted validators.
-
-If you operate an XRP Ledger server, then you should upgrade to version 1.9.4 within two weeks, to ensure service continuity. The exact time that protocol changes take effect depends on the voting decisions of the decentralized network.
-
-For more information about NFTs on the XRP Ledger, see [NFT Conceptual Overview](https://xrpl.org/nft-conceptual-overview.html).
-
-
-## Install / Upgrade
-
-On supported platforms, see the [instructions on installing or updating `rippled`](https://xrpl.org/install-rippled.html).
-
-## Changelog
-
-## Contributions
-
-The primary change in this release is the following bug fix:
-
-- **Introduce fixRemoveNFTokenAutoTrustLine amendment**: Introduces the `fixRemoveNFTokenAutoTrustLine` amendment, which disables the `tfTrustLine` flag, which a malicious attacker could exploit to mount denial-of-service attacks against NFT issuers that specified the flag on their NFTs. ([#4301](https://github.com/XRPLF/rippled/4301))
-
-
-### GitHub
-
-The public source code repository for `rippled` is hosted on GitHub at <https://github.com/XRPLF/rippled>.
-
-We welcome all contributions and invite everyone to join the community of XRP Ledger developers and help us build the Internet of Value.
-
-### Credits
-
-The following people contributed directly to this release:
-
-- Scott Schurr <scott@ripple.com>
-- Howard Hinnant <howard@ripple.com>
-- Scott Determan <scott.determan@yahoo.com>
-- Ikko Ashimine <eltociear@gmail.com>
-
-
-# Introducing XRP Ledger version 1.9.3
-
-Version 1.9.3 of `rippled`, the reference server implementation of the XRP Ledger protocol is now available. This release corrects minor technical flaws with the code that loads configured amendment votes after a startup and the copy constructor of `PublicKey`.
-
-## Install / Upgrade
-
-On supported platforms, see the [instructions on installing or updating `rippled`](https://xrpl.org/install-rippled.html).
-
-## Changelog
-
-## Contributions
-
-This release contains the following bug fixes:
-
-- **Change by-value to by-reference to persist vote**: A minor technical flaw, caused by use of a copy instead of a reference, resulted in operator-configured "yes" votes to not be properly loaded after a restart. ([#4256](https://github.com/XRPLF/rippled/pull/4256))
-- **Properly handle self-assignment of PublicKey**: The `PublicKey` copy assignment operator mishandled the case where a `PublicKey` would be assigned to itself, and could result in undefined behavior. 
-
-### GitHub
-
-The public source code repository for `rippled` is hosted on GitHub at <https://github.com/XRPLF/rippled>.
-
-We welcome contributions, big and small, and invite everyone to join the community of XRP Ledger developers and help us build the Internet of Value.
-
-### Credits
-
-The following people contributed directly to this release:
-
-- Howard Hinnant <howard@ripple.com>
-- Crypto Brad Garlinghouse <cryptobradgarlinghouse@protonmail.com>
-- Wo Jake <87929946+wojake@users.noreply.github.com>
-
-
-# Introducing XRP Ledger version 1.9.2
-
-Version 1.9.2 of `rippled`, the reference server implementation of the XRP Ledger protocol, is now available. This release includes several fixes and improvements, including a second new fix amendment to correct a bug in Non-Fungible Tokens (NFTs) code, a new API method for order book changes, less noisy logging, and other small fixes.
-
-<!-- BREAK -->
-
-
-## Action Required
-
-This release introduces a two new amendments to the XRP Ledger protocol. The first, **fixNFTokenNegOffer**, fixes a bug in code associated with the **NonFungibleTokensV1** amendment, originally introduced in [version 1.9.0](https://xrpl.org/blog/2022/rippled-1.9.0.html). The second, **NonFungibleTokensV1_1**, is a "roll-up" amendment that enables the **NonFungibleTokensV1** feature plus the two fix amendments associated with it, **fixNFTokenDirV1** and **fixNFTokenNegOffer**.
-
-If you want to enable NFT code on the XRP Ledger Mainnet, you can vote in favor of only the **NonFungibleTokensV1_1** amendment to support enabling the feature and fixes together, without risk that the unfixed NFT code may become enabled first.
-
-These amendments are now open for voting according to the XRP Ledger's [amendment process](https://xrpl.org/amendments.html), which enables protocol changes following two weeks of >80% support from trusted validators.
-
-If you operate an XRP Ledger server, then you should upgrade to version 1.9.2 within two weeks, to ensure service continuity. The exact time that protocol changes take effect depends on the voting decisions of the decentralized network.
-
-For more information about NFTs on the XRP Ledger, see [NFT Conceptual Overview](https://xrpl.org/nft-conceptual-overview.html).
-
-## Install / Upgrade
-
-On supported platforms, see the [instructions on installing or updating `rippled`](https://xrpl.org/install-rippled.html).
-
-## Changelog
-
-This release contains the following features and improvements.
-
-- **Introduce fixNFTokenNegOffer amendment.** This amendment fixes a bug in the Non-Fungible Tokens (NFTs) functionality provided by the NonFungibleTokensV1 amendment (not currently enabled on Mainnet). The bug allowed users to place offers to buy tokens for negative amounts of money when using Brokered Mode. Anyone who accepted such an offer would transfer the token _and_ pay money. This amendment explicitly disallows offers to buy or sell NFTs for negative amounts of money, and returns an appropriate error code. This also corrects the error code returned when placing offers to buy or sell NFTs for negative amounts in Direct Mode. ([8266d9d](https://github.com/XRPLF/rippled/commit/8266d9d598d19f05e1155956b30ca443c27e119e))
-- **Introduce `NonFungibleTokensV1_1` amendment.** This amendment encompasses three NFT-related amendments: the original NonFungibleTokensV1 amendment (from version 1.9.0), the fixNFTokenDirV1 amendment (from version 1.9.1), and the new fixNFTokenNegOffer amendment from this release. This amendment contains no changes other than enabling those three amendments together; this allows validators to vote in favor of _only_ enabling the feature and fixes at the same time. ([59326bb](https://github.com/XRPLF/rippled/commit/59326bbbc552287e44b3a0d7b8afbb1ddddb3e3b))
-- **Handle invalid port numbers.** If the user specifies a URL with an invalid port number, the server would silently attempt to use port 0 instead. Now it raises an error instead. This affects admin API methods and config file parameters for downloading history shards and specifying validator list sites. ([#4213](https://github.com/XRPLF/rippled/pull/4213))
-- **Reduce log noisiness.** Decreased the severity of benign log messages in several places: "addPathsForType" messages during regular operation, expected errors during unit tests, and missing optional documentation components when compiling from source. ([#4178](https://github.com/XRPLF/rippled/pull/4178), [#4166](https://github.com/XRPLF/rippled/pull/4166), [#4180](https://github.com/XRPLF/rippled/pull/4180))
-- **Fix race condition in history shard implementation and support clang's ThreadSafetyAnalysis tool.**  Added build settings so that developers can use this feature of the clang compiler to analyze the code for correctness, and fix an error found by this tool, which was the source of rare crashes in unit tests. ([#4188](https://github.com/XRPLF/rippled/pull/4188))
-- **Prevent crash when rotating a database with missing data.** When rotating databases, a missing entry could cause the server to crash. While there should never be a missing database entry, this change keeps the server running by aborting database rotation. ([#4182](https://github.com/XRPLF/rippled/pull/4182))
-- **Fix bitwise comparison in OfferCreate.** Fixed an expression that incorrectly used a bitwise comparison for two boolean values rather than a true boolean comparison. The outcome of the two comparisons is equivalent, so this is not a transaction processing change, but the bitwise comparison relied on compilers to implicitly fix the expression. ([#4183](https://github.com/XRPLF/rippled/pull/4183))
-- **Disable cluster timer when not in a cluster.** Disabled a timer that was unused on servers not running in clustered mode. The functionality of clustered servers is unchanged. ([#4173](https://github.com/XRPLF/rippled/pull/4173))
-- **Limit how often to process peer discovery messages.** In the peer-to-peer network, servers periodically share IP addresses of their peers with each other to facilitate peer discovery. It is not necessary to process these types of messages too often; previously, the code tracked whether it needed to process new messages of this type but always processed them anyway. With this change, the server no longer processes peer discovery messages if it has done so recently. ([#4202](https://github.com/XRPLF/rippled/pull/4202))
-- **Improve STVector256 deserialization.** Optimized the processing of this data type in protocol messages. This data type is used in several types of ledger entry that are important for bookkeeping, including directory pages that track other ledger types, amendments tracking, and the ledger hashes history. ([#4204](https://github.com/XRPLF/rippled/pull/4204))
-- **Fix and refactor spinlock code.** The spinlock code, which protects the `SHAMapInnerNode` child lists, had a mistake that allowed the same child to be repeatedly locked under some circumstances. Fixed this bug and improved the spinlock code to make it easier to use correctly and easier to verify that the code works correctly. ([#4201](https://github.com/XRPLF/rippled/pull/4201))
-- **Improve comments and contributor documentation.** Various minor documentation changes including some to reflect the fact that the source code repository is now owned by the XRP Ledger Foundation. ([#4214](https://github.com/XRPLF/rippled/pull/4214), [#4179](https://github.com/XRPLF/rippled/pull/4179), [#4222](https://github.com/XRPLF/rippled/pull/4222))
-- **Introduces a new API book_changes to provide information in a format that is useful for building charts that highlight DEX activity at a per-ledger level.**  ([#4212](https://github.com/XRPLF/rippled/pull/4212))
-
-## Contributions
-
-### GitHub
-
-The public source code repository for `rippled` is hosted on GitHub at <https://github.com/XRPLF/rippled>.
-
-We welcome contributions, big and small, and invite everyone to join the community of XRP Ledger developers and help us build the Internet of Value.
-
-### Credits
-
-The following people contributed directly to this release:
-
-- Chenna Keshava B S <ckbs.keshava56@gmail.com>
-- Ed Hennis <ed@ripple.com>
-- Ikko Ashimine <eltociear@gmail.com>
-- Nik Bougalis <nikb@bougalis.net>
-- Richard Holland <richard.holland@starstone.co.nz>
-- Scott Schurr <scott@ripple.com>
-- Scott Determan <scott.determan@yahoo.com>
-
-For a real-time view of all lifetime contributors, including links to the commits made by each, please visit the "Contributors" section of the GitHub repository: <https://github.com/XRPLF/rippled/graphs/contributors>.
-
-# Introducing XRP Ledger version 1.9.1
-
-Version 1.9.1 of `rippled`, the reference server implementation of the XRP Ledger protocol, is now available. This release includes several important fixes, including a fix for a syncing issue from 1.9.0, a new fix amendment to correct a bug in the new Non-Fungible Tokens (NFTs) code, and a new amendment to allow multi-signing by up to 32 signers.
-
-<!-- BREAK -->
-
-
-## Action Required
-
-This release introduces two new amendments to the XRP Ledger protocol. These amendments are now open for voting according to the XRP Ledger's [amendment process](https://xrpl.org/amendments.html), which enables protocol changes following two weeks of >80% support from trusted validators.
-
-If you operate an XRP Ledger server, then you should upgrade to version 1.9.1 within two weeks, to ensure service continuity. The exact time that protocol changes take effect depends on the voting decisions of the decentralized network.
-
-The **fixNFTokenDirV1** amendment fixes a bug in code associated with the **NonFungibleTokensV1** amendment, so the fixNFTokenDirV1 amendment should be enabled first. All validator operators are encouraged to [configure amendment voting](https://xrpl.org/configure-amendment-voting.html) to oppose the NonFungibleTokensV1 amendment until _after_ the fixNFTokenDirV1 amendment has become enabled. For more information about NFTs on the XRP Ledger, see [NFT Conceptual Overview](https://xrpl.org/nft-conceptual-overview.html).
-
-The **ExpandedSignerList** amendment extends the ledger's built-in multi-signing functionality so that each list can contain up to 32 entries instead of the current limit of 8. Additionally, this amendment allows each signer to have an arbitrary 256-bit data field associated with it. This data can be used to identify the signer or provide other metadata that is useful for organizations, smart contracts, or other purposes.
-
-## Install / Upgrade
-
-On supported platforms, see the [instructions on installing or updating `rippled`](https://xrpl.org/install-rippled.html).
-
-## Changelog
-
-This release contains the following features and improvements.
-
-## New Features and Amendments
-
-- **Introduce fixNFTokenDirV1 Amendment** - This amendment fixes an off-by-one error that occurred in some corner cases when determining which `NFTokenPage` an `NFToken` object belongs on. It also adjusts the constraints of `NFTokenPage` invariant checks, so that certain error cases fail with a suitable error code such as `tecNO_SUITABLE_TOKEN_PAGE` instead of failing with a `tecINVARIANT_FAILED` error code. ([#4155](https://github.com/ripple/rippled/pull/4155))
-
-- **Introduce ExpandedSignerList Amendment** - This amendment expands the maximum signer list size to 32 entries and allows each signer to have an optional 256-bit `WalletLocator` field containing arbitrary data. ([#4097](https://github.com/ripple/rippled/pull/4097))
-
-- **Pause online deletion rather than canceling it if the server fails health check** - The server stops performing online deletion of old ledger history if the server fails its internal health check during this time. Online deletion can now resume after the server recovers, rather than having to start over. ([#4139](https://github.com/ripple/rippled/pull/4139))
-
-
-## Bug Fixes and Performance Improvements
-
-- **Fix performance issues introduced in 1.9.0** - Readjusts some parameters of the ledger acquisition engine to revert some changes introduced in 1.9.0 that had adverse effects on some systems, including causing some systems to fail to sync to the network. ([#4152](https://github.com/ripple/rippled/pull/4152))
-
-- **Improve Memory Efficiency of Path Finding** - Finding paths for cross-currency payments is a resource-intensive operation. While that remains true, this fix improves memory usage of pathfinding by discarding trust line results that cannot be used before those results are fully loaded or cached. ([#4111](https://github.com/ripple/rippled/pull/4111))
-
-- **Fix incorrect CMake behavior on Windows when platform is unspecified or x64** - Fixes handling of platform selection when using the cmake-gui tool to build on Windows. The generator expects `Win64` but the GUI only provides `x64` as an option, which raises an error. This fix only raises an error if the platform is `Win32` instead, allowing the generation of solution files to succeed. ([#4150](https://github.com/ripple/rippled/pull/4150))
-
-- **Fix test failures with newer MSVC compilers on Windows** - Fixes some cases where the API handler code used string pointer comparisons, which may not work correctly with some versions of the MSVC compiler. ([#4149](https://github.com/ripple/rippled/pull/4149))
-
-- **Update minimum Boost version to 1.71.0** - This release is compatible with Boost library versions 1.71.0 through 1.77.0. The build configuration and documentation have been updated to reflect this. ([#4134](https://github.com/ripple/rippled/pull/4134))
-
-- **Fix unit test failures for DatabaseDownloader** - Increases a timeout in the `DatabaseDownloader` code and adjusts unit tests so that the code does not return spurious failures, and more data is logged if it does fail. ([#4021](https://github.com/ripple/rippled/pull/4021))
-
-- **Refactor relational database interface** - Improves code comments, naming, and organization of the module that interfaces with relational databases (such as the SQLite database used for tracking transaction history). ([#3965](https://github.com/ripple/rippled/pull/3965))
-
-
-## Contributions
-
-### GitHub
-
-The public source code repository for `rippled` is hosted on GitHub at <https://github.com/ripple/rippled>.
-
-We welcome contributions, big and small, and invite everyone to join the community of XRP Ledger developers and help us build the Internet of Value.
-
-
-### Credits
-
-The following people contributed directly to this release:
-
-- Devon White <dwhite@ripple.com>
-- Ed Hennis <ed@ripple.com>
-- Gregory Popovitch <greg7mdp@gmail.com>
-- Mark Travis <mtravis@ripple.com>
-- Manoj Doshi <mdoshi@ripple.com>
-- Nik Bougalis <nikb@bougalis.net>
-- Richard Holland <richard.holland@starstone.co.nz>
-- Scott Schurr <scott@ripple.com>
-
-For a real-time view of all lifetime contributors, including links to the commits made by each, please visit the "Contributors" section of the GitHub repository: <https://github.com/ripple/rippled/graphs/contributors>.
-
-We welcome external contributions and are excited to see the broader XRP Ledger community continue to grow and thrive.
-
+Have new ideas? Need help with setting up your node? Come visit us [here](https://github.com/ripple/rippled/issues/new/choose)
 
 # Change log
 
@@ -434,68 +13,6 @@ We welcome external contributions and are excited to see the broader XRP Ledger
 
 # Releases
 
-## Version 1.9.0
-This is the 1.9.0 release of `rippled`, the reference implementation of the XRP Ledger protocol. This release brings several features and improvements.
-
-### New and Improved Features
-- **Introduce NFT support (XLS020):** This release introduces support for non-fungible tokens, currently available to the developer community for broader review and testing.  Developers can create applications that allow users to mint, transfer, and ultimately burn (if desired) NFTs on the XRP Ledger. You can try out the new NFT transactions using the [nft-devnet](https://xrpl.org/xrp-testnet-faucet.html). Note that some fields and error codes from earlier releases of the supporting code have been refactored for this release, shown in the Code Refactoring section, below. [70779f](https://github.com/ripple/rippled/commit/70779f6850b5f33cdbb9cf4129bc1c259af0013e)
-
-- **Simplify the Job Queue:** This is a refactor aimed at cleaning up and simplifying the existing job queue. Currently, all jobs are canceled at the same time and in the same way, so this commit removes the unnecessary per-job cancellation token. [#3656](https://github.com/ripple/rippled/pull/3656)
-
-- **Optimize trust line caching:** The existing trust line caching code was suboptimal in that it stored redundant information, pinned SLEs into memory, and required multiple memory allocations per cached object. This commit eliminates redundant data, reduces the size of cached objects and unpinning SLEs from memory, and uses value types to avoid the need for `std::shared_ptr`. As a result of these changes, the effective size of a cached object includes the overhead of the memory allocator, and the `std::shared_ptr` should be reduced by at least 64 bytes. This is significant, as there can easily be tens of millions of these objects. [4d5459](https://github.com/ripple/rippled/commit/4d5459d041da8f5a349c5f458d664e5865e1f1b5)
-
-- **Incremental improvements to pathfinding memory usage:** This commit aborts background pathfinding when closed or disconnected, exits the pathfinding job thread if there are no requests left, does not create the path find a job if there are no requests, and refactors to remove the circular dependency between InfoSub and PathRequest. [#4111](https://github.com/ripple/rippled/pull/4111)
-
-- **Improve deterministic transaction sorting in TxQ:** This commit ensures that transactions with the same fee level are sorted by TxID XORed with the parent ledger hash, the TxQ is re-sorted after every ledger, and attempts to future-proof the TxQ tie-breaking test. [#4077](https://github.com/ripple/rippled/pull/4077)
-
-- **Improve stop signaling for Application:** [34ca45](https://github.com/ripple/rippled/commit/34ca45713244d0defc39549dd43821784b2a5c1d)
-
-- **Eliminate SHAMapInnerNode lock contention:** The `SHAMapInnerNode` class had a global mutex to protect the array of node children. Profiling suggested that around 4% of all attempts to lock the global would block. This commit removes that global mutex, and replaces it with a new per-node 16-way spinlock (implemented so as not to affect the size of an inner node object), effectively eliminating the lock contention. [1b9387](https://github.com/ripple/rippled/commit/1b9387eddc1f52165d3243d2ace9be0c62495eea)
-
-- **Improve ledger-fetching logic:** When fetching ledgers, the existing code would isolate the peer that sent the most useful responses, and issue follow-up queries only to that peer. This commit increases the query aggressiveness, and changes the mechanism used to select which peers to issue follow-up queries to so as to more evenly spread the load among those peers that provided useful responses. [48803a](https://github.com/ripple/rippled/commit/48803a48afc3bede55d71618c2ee38fd9dbfd3b0)
-
-- **Simplify and improve order book tracking:** The order book tracking code would use `std::shared_ptr` to track the lifetime of objects. This commit changes the logic to eliminate the overhead of `std::shared_ptr` by using value types, resulting in significant memory savings. [b9903b](https://github.com/ripple/rippled/commit/b9903bbcc483a384decf8d2665f559d123baaba2)
-
-- **Negative cache support for node store:** This commit allows the cache to service requests for nodes that were previously looked up but not found, reducing the need to perform I/O in several common scenarios. [3eb8aa](https://github.com/ripple/rippled/commit/3eb8aa8b80bd818f04c99cee2cfc243192709667)
-
-- **Improve asynchronous database handlers:** This commit optimizes the way asynchronous node store operations are processed, both by reducing the number of times locks are held and by minimizing the number of memory allocations and data copying. [6faaa9](https://github.com/ripple/rippled/commit/6faaa91850d6b2eb9fbf16c1256bf7ef11ac4646)
-
-- **Cleanup AcceptedLedger and AcceptedLedgerTx:** This commit modernizes the `AcceptedLedger` and `AcceptedLedgerTx` classes, reduces their memory footprint, and reduces unnecessary dynamic memory allocations. [8f5868](https://github.com/ripple/rippled/commit/8f586870917818133924bf2e11acab5321c2b588)
-
-### Code Refactoring
-
-This release includes name changes in the NFToken API for SFields, RPC return labels, and error codes for clarity and consistency. To refactor your code, migrate the names of these items to the new names as listed below.
- 
-#### `SField` name changes:
-* `TokenTaxon -> NFTokenTaxon`
-* `MintedTokens -> MintedNFTokens`
-* `BurnedTokens -> BurnedNFTokens`
-* `TokenID -> NFTokenID`
-* `TokenOffers -> NFTokenOffers`
-* `BrokerFee -> NFTokenBrokerFee`
-* `Minter -> NFTokenMinter`
-* `NonFungibleToken -> NFToken`
-* `NonFungibleTokens -> NFTokens`
-* `BuyOffer -> NFTokenBuyOffer`
-* `SellOffer -> NFTokenSellOffer`
-* `OfferNode -> NFTokenOfferNode`
- 
-#### RPC return labels
-* `tokenid -> nft_id`
-* `index -> nft_offer_index`
- 
-#### Error codes
-* `temBAD_TRANSFER_FEE -> temBAD_NFTOKEN_TRANSFER_FEE`
-* `tefTOKEN_IS_NOT_TRANSFERABLE -> tefNFTOKEN_IS_NOT_TRANSFERABLE`
-* `tecNO_SUITABLE_PAGE -> tecNO_SUITABLE_NFTOKEN_PAGE`
-* `tecBUY_SELL_MISMATCH -> tecNFTOKEN_BUY_SELL_MISMATCH`
-* `tecOFFER_TYPE_MISMATCH -> tecNFTOKEN_OFFER_TYPE_MISMATCH`
-* `tecCANT_ACCEPT_OWN_OFFER -> tecCANT_ACCEPT_OWN_NFTOKEN_OFFER`
-
-
-### Bug Fixes
-- **Fix deletion of orphan node store directories:** Orphaned node store directories should only be deleted if the proper node store directories are confirmed to exist. [06e87e](https://github.com/ripple/rippled/commit/06e87e0f6add5b880d647e14ab3d950decfcf416)
-
 ## Version 1.8.5
 This is the 1.8.5 release of `rippled`, the reference implementation of the XRP Ledger protocol. This release includes fixes and updates for stability and security, and improvements to build scripts. There are no user-facing API or protocol changes in this release.
 

bin/sidechain/python/README.md

@@ -0,0 +1,183 @@
+## Introduction
+
+This directory contains python scripts to tests and explore side chains. 
+
+See the instructions [here](docs/sidechain/GettingStarted.md) for how to install
+the necessary dependencies and run an interactive shell that will spin up a set
+of federators on your local machine and allow you to transfer assets between the
+main chain and a side chain.
+
+For all these scripts, make sure the `RIPPLED_MAINCHAIN_EXE`,
+`RIPPLED_SIDECHAIN_EXE`, and `RIPPLED_SIDECHAIN_CFG_DIR` environment variables
+are correctly set, and the side chain configuration files exist. Also make sure the python
+dependencies are installed and the virtual environment is activated.
+
+Note: the unit tests do not use the configuration files, so the `RIPPLED_SIDECHAIN_CFG_DIR` is
+not needed for that script.
+
+## Unit tests
+
+The "tests" directory contains a simple unit test. It take several minutes to
+run, and will create the necessary configuration files, start a test main chain
+in standalone mode, and a test side chain with 5 federators, and do some simple
+cross chain transactions. Side chains do not yet have extensive tests. Testing
+is being actively worked on.
+
+To run the tests, change directories to the `bin/sidechain/python/tests` directory and type:
+```
+pytest
+```
+
+To capture logging information and to set the log level (to help with debugging), type this instead:
+```
+pytest --log-file=log.txt --log-file-level=info
+```
+
+The response should be something like the following:
+```
+============================= test session starts ==============================
+platform linux -- Python 3.8.5, pytest-6.2.5, py-1.10.0, pluggy-1.0.0
+rootdir: /home/swd/projs/ripple/mine/bin/sidechain/python/tests
+collected 1 item
+
+simple_xchain_transfer_test.py .                                         [100%]
+
+======================== 1 passed in 215.20s (0:03:35) =========================
+
+```
+
+## Scripts
+### riplrepl.py
+
+This is an interactive shell for experimenting with side chains. It will spin up
+a test main chain running in standalone mode, and a test side chain with five
+federators - all running on the local machine. There are commands to make
+payments within a chain, make cross chain payments, check balances, check server
+info, and check federator info. There is a simple "help" system, but more
+documentation is needed for this tool (or more likely we need to replace this
+with some web front end).
+
+Note: a "repl" is another name for an interactive shell. It stands for
+"read-eval-print-loop". It is pronounced "rep-ul".
+
+### create_config_file.py
+
+This is a script used to create the config files needed to run a test side chain
+on your local machine. To run this, make sure the rippled is built,
+`RIPPLED_MAINCHAIN_EXE`, `RIPPLED_SIDECHAIN_EXE`, and
+`RIPPLED_SIDECHAIN_CFG_DIR` environment variables are correctly set, and the
+side chain configuration files exist. Also make sure the python dependencies are
+installed and the virtual environment is activated. Running this will create
+config files in the directory specified by the `RIPPLED_SIDECHAIN_CFG_DIR`
+environment variable.
+
+### log_analyzer.py
+
+This is a script used to take structured log files and convert them to json for easier debugging.
+
+## Python modules
+
+### sidechain.py
+
+A python module that can be used to write python scripts to interact with
+side chains. This is used to write unit tests and the interactive shell. To write
+a standalone script, look at how the tests are written in
+`test/simple_xchain_transfer_test.py`. The idea is to call
+`sidechain._multinode_with_callback`, which sets up the two chains, and place
+your code in the callback. For example:
+
+```
+def multinode_test(params: Params):
+    def callback(mc_app: App, sc_app: App):
+        my_function(mc_app, sc_app, params)
+
+    sidechain._multinode_with_callback(params,
+                                       callback,
+                                       setup_user_accounts=False)
+```
+
+The functions `sidechain.main_to_side_transfer` and
+`sidechain.side_to_main_transfer` can be used as convenience functions to initiate
+cross chain transfers. Of course, these transactions can also be initiated with
+a payment to the door account with the memo data set to the destination account
+on the destination chain (which is what those convenience functions do under the
+hood).
+
+Transactions execute asynchonously. Use the function
+`test_utils.wait_for_balance_change` to ensure a transaction has completed.
+
+### transaction.py
+
+A python module for transactions. Currently there are transactions for:
+
+* Payment
+* Trust (trust set)
+* SetRegularKey
+* SignerLisetSet
+* AccountSet
+* Offer
+* Ticket
+* Hook (experimental - useful paying with the hook amendment from XRPL Labs).
+
+Typically, a transaction is submitted through the call operator on an `App` object. For example, to make a payment from the account `alice` to the account `bob` for 500 XRP:
+```
+    mc_app(Payment(account=alice, dst=bob, amt=XRP(500)))
+```
+(where mc_app is an App object representing the main chain).
+
+### command.py
+
+A python module for RPC commands. Currently there are commands for:
+* PathFind
+* Sign
+* LedgerAccept (for standalone mode)
+* Stop
+* LogLevel
+* WalletPropose
+* ValidationCreate
+* AccountInfo
+* AccountLines
+* AccountTx
+* BookOffers
+* BookSubscription
+* ServerInfo
+* FederatorInfo
+* Subscribe
+
+### common.py
+
+Python module for common ledger objects, including:
+* Account
+* Asset
+* Path
+* Pathlist
+
+### app.py
+
+Python module for an application. An application is responsible for local
+network (or single server) and an address book that maps aliases to accounts.
+
+### config_file.py
+
+Python module representing a config file that is read from disk.
+
+### interactive.py
+
+Python module with the implementation of the RiplRepl interactive shell.
+
+### ripple_client.py
+
+A python module representing a rippled server.
+
+### testnet.py
+
+A python module representing a rippled testnet running on the local machine.
+
+## Other
+### requirements.txt
+
+These are the python dependencies needed by the scripts. Use `pip3 install -r
+requirements.txt` to install them. A python virtual environment is recommended.
+See the instructions [here](docs/sidechain/GettingStarted.md) for how to install
+the dependencies.
+

bin/sidechain/python/app.py

@@ -0,0 +1,624 @@
+from contextlib import contextmanager
+import json
+import logging
+import os
+import pandas as pd
+from pathlib import Path
+import subprocess
+import time
+from typing import Callable, Dict, List, Optional, Set, Union
+
+from ripple_client import RippleClient
+from common import Account, Asset, XRP
+from command import AccountInfo, AccountLines, BookOffers, Command, FederatorInfo, LedgerAccept, Sign, Submit, SubscriptionCommand, WalletPropose
+from config_file import ConfigFile
+import testnet
+from transaction import Payment, Transaction
+
+
+class KeyManager:
+    def __init__(self):
+        self._aliases = {}  # alias -> account
+        self._accounts = {}  # account id -> account
+
+    def add(self, account: Account) -> bool:
+        if account.nickname:
+            self._aliases[account.nickname] = account
+        self._accounts[account.account_id] = account
+
+    def is_alias(self, name: str):
+        return name in self._aliases
+
+    def account_from_alias(self, name: str) -> Account:
+        assert name in self._aliases
+        return self._aliases[name]
+
+    def known_accounts(self) -> List[Account]:
+        return list(self._accounts.values())
+
+    def account_id_dict(self) -> Dict[str, Account]:
+        return self._accounts
+
+    def alias_or_account_id(self, id: Union[Account, str]) -> str:
+        '''
+        return the alias if it exists, otherwise return the id
+        '''
+        if isinstance(id, Account):
+            return id.alias_or_account_id()
+
+        if id in self._accounts:
+            return self._accounts[id].nickname
+        return id
+
+    def alias_to_account_id(self, alias: str) -> Optional[str]:
+        if id in self._aliases:
+            return self._aliases[id].account_id
+        return None
+
+    def to_string(self, nickname: Optional[str] = None):
+        names = []
+        account_ids = []
+        if nickname:
+            names = [nickname]
+            if nickname in self._aliases:
+                account_ids = [self._aliases[nickname].account_id]
+            else:
+                account_id = ['NA']
+        else:
+            for (k, v) in self._aliases.items():
+                names.append(k)
+                account_ids.append(v.account_id)
+        # use a dataframe to get a nice table output
+        df = pd.DataFrame(data={'name': names, 'id': account_ids})
+        return f'{df.to_string(index=False)}'
+
+
+class AssetAliases:
+    def __init__(self):
+        self._aliases = {}  # alias -> asset
+
+    def add(self, asset: Asset, name: str):
+        self._aliases[name] = asset
+
+    def is_alias(self, name: str):
+        return name in self._aliases
+
+    def asset_from_alias(self, name: str) -> Asset:
+        assert name in self._aliases
+        return self._aliases[name]
+
+    def known_aliases(self) -> List[str]:
+        return list(self._aliases.keys())
+
+    def known_assets(self) -> List[Asset]:
+        return list(self._aliases.values())
+
+    def to_string(self, nickname: Optional[str] = None):
+        names = []
+        currencies = []
+        issuers = []
+        if nickname:
+            names = [nickname]
+            if nickname in self._aliases:
+                v = self._aliases[nickname]
+                currencies = [v.currency]
+                iss = v.issuer if v.issuer else ''
+                issuers = [v.issuer if v.issuer else '']
+            else:
+                currencies = ['NA']
+                issuers = ['NA']
+        else:
+            for (k, v) in self._aliases.items():
+                names.append(k)
+                currencies.append(v.currency)
+                issuers.append(v.issuer if v.issuer else '')
+        # use a dataframe to get a nice table output
+        df = pd.DataFrame(data={
+            'name': names,
+            'currency': currencies,
+            'issuer': issuers
+        })
+        return f'{df.to_string(index=False)}'
+
+
+class App:
+    '''App to to interact with rippled servers'''
+    def __init__(self,
+                 *,
+                 standalone: bool,
+                 network: Optional[testnet.Network] = None,
+                 client: Optional[RippleClient] = None):
+        if network and client:
+            raise ValueError('Cannot specify both a testnet and client in App')
+        if not network and not client:
+            raise ValueError('Must specify a testnet or a client in App')
+
+        self.standalone = standalone
+        self.network = network
+
+        if client:
+            self.client = client
+        else:
+            self.client = self.network.get_client(0)
+
+        self.key_manager = KeyManager()
+        self.asset_aliases = AssetAliases()
+        root_account = Account(nickname='root',
+                               account_id='rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh',
+                               secret_key='masterpassphrase')
+        self.key_manager.add(root_account)
+
+    def shutdown(self):
+        if self.network:
+            self.network.shutdown()
+        else:
+            self.client.shutdown()
+
+    def send_signed(self, txn: Transaction) -> dict:
+        '''Sign then send the given transaction'''
+        if not txn.account.secret_key:
+            raise ValueError('Cannot sign transaction without secret key')
+        r = self(Sign(txn.account.secret_key, txn.to_cmd_obj()))
+        raw_signed = r['tx_blob']
+        r = self(Submit(raw_signed))
+        logging.info(f'App.send_signed: {json.dumps(r, indent=1)}')
+        return r
+
+    def send_command(self, cmd: Command) -> dict:
+        '''Send the command to the rippled server'''
+        r = self.client.send_command(cmd)
+        logging.info(
+            f'App.send_command : {cmd.cmd_name()} : {json.dumps(r, indent=1)}')
+        return r
+
+    # Need async version to close ledgers from async functions
+    async def async_send_command(self, cmd: Command) -> dict:
+        '''Send the command to the rippled server'''
+        return await self.client.async_send_command(cmd)
+
+    def send_subscribe_command(
+            self,
+            cmd: SubscriptionCommand,
+            callback: Optional[Callable[[dict], None]] = None) -> dict:
+        '''Send the subscription command to the rippled server. If already subscribed, it will unsubscribe'''
+        return self.client.send_subscribe_command(cmd, callback)
+
+    def get_pids(self) -> List[int]:
+        if self.network:
+            return self.network.get_pids()
+        if pid := self.client.get_pid():
+            return [pid]
+
+    def get_running_status(self) -> List[bool]:
+        if self.network:
+            return self.network.get_running_status()
+        if self.client.get_pid():
+            return [True]
+        else:
+            return [False]
+
+    # Get a dict of the server_state, validated_ledger_seq, and complete_ledgers
+    def get_brief_server_info(self) -> dict:
+        if self.network:
+            return self.network.get_brief_server_info()
+        else:
+            ret = {}
+            for (k, v) in self.client.get_brief_server_info().items():
+                ret[k] = [v]
+            return ret
+
+    def servers_start(self,
+                      server_indexes: Optional[Union[Set[int],
+                                                     List[int]]] = None,
+                      *,
+                      extra_args: Optional[List[List[str]]] = None):
+        if self.network:
+            return self.network.servers_start(server_indexes,
+                                              extra_args=extra_args)
+        else:
+            raise ValueError('Cannot start stand alone server')
+
+    def servers_stop(self,
+                     server_indexes: Optional[Union[Set[int],
+                                                    List[int]]] = None):
+        if self.network:
+            return self.network.servers_stop(server_indexes)
+        else:
+            raise ValueError('Cannot stop stand alone server')
+
+    def federator_info(self,
+                       server_indexes: Optional[Union[Set[int],
+                                                      List[int]]] = None):
+        # key is server index. value is federator_info result
+        result_dict = {}
+        if self.network:
+            if not server_indexes:
+                server_indexes = [
+                    i for i in range(self.network.num_clients())
+                    if self.network.is_running(i)
+                ]
+            for i in server_indexes:
+                if self.network.is_running(i):
+                    result_dict[i] = self.network.get_client(i).send_command(
+                        FederatorInfo())
+        else:
+            if 0 in server_indexes:
+                result_dict[0] = self.client.send_command(FederatorInfo())
+        return result_dict
+
+    def __call__(self,
+                 to_send: Union[Transaction, Command, SubscriptionCommand],
+                 callback: Optional[Callable[[dict], None]] = None,
+                 *,
+                 insert_seq_and_fee=False) -> dict:
+        '''Call `send_signed` for transactions or `send_command` for commands'''
+        if isinstance(to_send, SubscriptionCommand):
+            return self.send_subscribe_command(to_send, callback)
+        assert callback is None
+        if isinstance(to_send, Transaction):
+            if insert_seq_and_fee:
+                self.insert_seq_and_fee(to_send)
+            return self.send_signed(to_send)
+        if isinstance(to_send, Command):
+            return self.send_command(to_send)
+        raise ValueError(
+            'Expected `to_send` to be either a Transaction, Command, or SubscriptionCommand'
+        )
+
+    def get_configs(self) -> List[str]:
+        if self.network:
+            return self.network.get_configs()
+        return [self.client.config]
+
+    def create_account(self, name: str) -> Account:
+        ''' Create an account. Use the name as the alias. '''
+        if name == 'root':
+            return
+        assert not self.key_manager.is_alias(name)
+
+        account = Account(nickname=name, result_dict=self(WalletPropose()))
+        self.key_manager.add(account)
+        return account
+
+    def create_accounts(self,
+                        names: List[str],
+                        funding_account: Union[Account, str] = 'root',
+                        amt: Union[int, Asset] = 1000000000) -> List[Account]:
+        '''Fund the accounts with nicknames 'names' by using the funding account and amt'''
+        accounts = [self.create_account(n) for n in names]
+        if not isinstance(funding_account, Account):
+            org_funding_account = funding_account
+            funding_account = self.key_manager.account_from_alias(
+                funding_account)
+        if not isinstance(funding_account, Account):
+            raise ValueError(
+                f'Could not find funding account {org_funding_account}')
+        if not isinstance(amt, Asset):
+            assert isinstance(amt, int)
+            amt = Asset(value=amt)
+        for a in accounts:
+            p = Payment(account=funding_account, dst=a, amt=amt)
+            self.send_signed(p)
+        return accounts
+
+    def maybe_ledger_accept(self):
+        if not self.standalone:
+            return
+        self(LedgerAccept())
+
+    # Need async version to close ledgers from async functions
+    async def async_maybe_ledger_accept(self):
+        if not self.standalone:
+            return
+        await self.async_send_command(LedgerAccept())
+
+    def get_balances(
+        self,
+        account: Union[Account, List[Account], None] = None,
+        asset: Union[Asset, List[Asset]] = Asset()
+    ) -> pd.DataFrame:
+        '''Return a pandas dataframe of account balances. If account is None, treat as a wildcard (use address book)'''
+        if account is None:
+            account = self.key_manager.known_accounts()
+        if isinstance(account, list):
+            result = [self.get_balances(acc, asset) for acc in account]
+            return pd.concat(result, ignore_index=True)
+        if isinstance(asset, list):
+            result = [self.get_balances(account, ass) for ass in asset]
+            return pd.concat(result, ignore_index=True)
+        if asset.is_xrp():
+            try:
+                df = self.get_account_info(account)
+            except:
+                # Most likely the account does not exist on the ledger. Give a balance of zero.
+                df = pd.DataFrame({
+                    'account': [account],
+                    'balance': [0],
+                    'flags': [0],
+                    'owner_count': [0],
+                    'previous_txn_id': ['NA'],
+                    'previous_txn_lgr_seq': [-1],
+                    'sequence': [-1]
+                })
+            df = df.assign(currency='XRP', peer='', limit='')
+            return df.loc[:,
+                          ['account', 'balance', 'currency', 'peer', 'limit']]
+        else:
+            try:
+                df = self.get_trust_lines(account)
+                if df.empty: return df
+                df = df[(df['peer'] == asset.issuer.account_id)
+                        & (df['currency'] == asset.currency)]
+            except:
+                # Most likely the account does not exist on the ledger. Return an empty data frame
+                df = pd.DataFrame({
+                    'account': [],
+                    'balance': [],
+                    'currency': [],
+                    'peer': [],
+                    'limit': [],
+                })
+            return df.loc[:,
+                          ['account', 'balance', 'currency', 'peer', 'limit']]
+
+    def get_balance(self, account: Account, asset: Asset) -> Asset:
+        '''Get a balance from a single account in a single asset'''
+        try:
+            df = self.get_balances(account, asset)
+            return asset(df.iloc[0]['balance'])
+        except:
+            return asset(0)
+
+    def get_account_info(self,
+                         account: Optional[Account] = None) -> pd.DataFrame:
+        '''Return a pandas dataframe of account info. If account is None, treat as a wildcard (use address book)'''
+        if account is None:
+            known_accounts = self.key_manager.known_accounts()
+            result = [self.get_account_info(acc) for acc in known_accounts]
+            return pd.concat(result, ignore_index=True)
+        try:
+            result = self.client.send_command(AccountInfo(account))
+        except:
+            # Most likely the account does not exist on the ledger. Give a balance of zero.
+            return pd.DataFrame({
+                'account': [account],
+                'balance': [0],
+                'flags': [0],
+                'owner_count': [0],
+                'previous_txn_id': ['NA'],
+                'previous_txn_lgr_seq': [-1],
+                'sequence': [-1]
+            })
+        if 'account_data' not in result:
+            raise ValueError('Bad result from account_info command')
+        info = result['account_data']
+        for dk in ['LedgerEntryType', 'index']:
+            del info[dk]
+        df = pd.DataFrame([info])
+        df.rename(columns={
+            'Account': 'account',
+            'Balance': 'balance',
+            'Flags': 'flags',
+            'OwnerCount': 'owner_count',
+            'PreviousTxnID': 'previous_txn_id',
+            'PreviousTxnLgrSeq': 'previous_txn_lgr_seq',
+            'Sequence': 'sequence'
+        },
+                  inplace=True)
+        df['balance'] = df['balance'].astype(int)
+        return df
+
+    def get_trust_lines(self,
+                        account: Account,
+                        peer: Optional[Account] = None) -> pd.DataFrame:
+        '''Return a pandas dataframe account trust lines. If peer account is None, treat as a wildcard'''
+        result = self.send_command(AccountLines(account, peer=peer))
+        if 'lines' not in result or 'account' not in result:
+            raise ValueError('Bad result from account_lines command')
+        account = result['account']
+        lines = result['lines']
+        for d in lines:
+            d['peer'] = d['account']
+            d['account'] = account
+        return pd.DataFrame(lines)
+
+    def get_offers(self, taker_pays: Asset, taker_gets: Asset) -> pd.DataFrame:
+        '''Return a pandas dataframe of offers'''
+        result = self.send_command(BookOffers(taker_pays, taker_gets))
+        if 'offers' not in result:
+            raise ValueError('Bad result from book_offers command')
+
+        offers = result['offers']
+        delete_keys = [
+            'BookDirectory', 'BookNode', 'LedgerEntryType', 'OwnerNode',
+            'PreviousTxnID', 'PreviousTxnLgrSeq', 'Sequence', 'index'
+        ]
+        for d in offers:
+            for dk in delete_keys:
+                del d[dk]
+            for t in ['TakerPays', 'TakerGets', 'owner_funds']:
+                if 'value' in d[t]:
+                    d[t] = d[t]['value']
+        df = pd.DataFrame(offers)
+        df.rename(columns={
+            'Account': 'account',
+            'Flags': 'flags',
+            'TakerGets': 'taker_gets',
+            'TakerPays': 'taker_pays'
+        },
+                  inplace=True)
+        return df
+
+    def account_balance(self, account: Account, asset: Asset) -> Asset:
+        '''get the account's balance of the asset'''
+        pass
+
+    def substitute_nicknames(
+            self,
+            df: pd.DataFrame,
+            cols: List[str] = ['account', 'peer']) -> pd.DataFrame:
+        result = df.copy(deep=True)
+        for c in cols:
+            if c not in result:
+                continue
+            result[c] = result[c].map(
+                lambda x: self.key_manager.alias_or_account_id(x))
+        return result
+
+    def add_to_keymanager(self, account: Account):
+        self.key_manager.add(account)
+
+    def is_alias(self, name: str) -> bool:
+        return self.key_manager.is_alias(name)
+
+    def account_from_alias(self, name: str) -> Account:
+        return self.key_manager.account_from_alias(name)
+
+    def known_accounts(self) -> List[Account]:
+        return self.key_manager.known_accounts()
+
+    def known_asset_aliases(self) -> List[str]:
+        return self.asset_aliases.known_aliases()
+
+    def known_iou_assets(self) -> List[Asset]:
+        return self.asset_aliases.known_assets()
+
+    def is_asset_alias(self, name: str) -> bool:
+        return self.asset_aliases.is_alias(name)
+
+    def add_asset_alias(self, asset: Asset, name: str):
+        self.asset_aliases.add(asset, name)
+
+    def asset_from_alias(self, name: str) -> Asset:
+        return self.asset_aliases.asset_from_alias(name)
+
+    def insert_seq_and_fee(self, txn: Transaction):
+        acc_info = self(AccountInfo(txn.account))
+        # TODO: set better fee (Hard code a fee of 15 for now)
+        txn.set_seq_and_fee(acc_info['account_data']['Sequence'], 15)
+
+    def get_client(self) -> RippleClient:
+        return self.client
+
+
+def balances_dataframe(chains: List[App],
+                       chain_names: List[str],
+                       account_ids: Optional[List[Account]] = None,
+                       assets: List[Asset] = None,
+                       in_drops: bool = False):
+    def _removesuffix(self: str, suffix: str) -> str:
+        if suffix and self.endswith(suffix):
+            return self[:-len(suffix)]
+        else:
+            return self[:]
+
+    def _balance_df(chain: App, acc: Optional[Account],
+                    asset: Union[Asset, List[Asset]], in_drops: bool):
+        b = chain.get_balances(acc, asset)
+        if not in_drops:
+            b.loc[b['currency'] == 'XRP', 'balance'] /= 1_000_000
+            b = chain.substitute_nicknames(b)
+            b = b.set_index('account')
+            return b
+
+    if account_ids is None:
+        account_ids = [None] * len(chains)
+
+    if assets is None:
+        # XRP and all assets in the assets alias list
+        assets = [[XRP(0)] + c.known_iou_assets() for c in chains]
+
+    dfs = []
+    keys = []
+    for chain, chain_name, acc, asset in zip(chains, chain_names, account_ids,
+                                             assets):
+        dfs.append(_balance_df(chain, acc, asset, in_drops))
+        keys.append(_removesuffix(chain_name, 'chain'))
+    df = pd.concat(dfs, keys=keys)
+    return df
+
+
+# Start an app with a single client
+@contextmanager
+def single_client_app(*,
+                      config: ConfigFile,
+                      command_log: Optional[str] = None,
+                      server_out=os.devnull,
+                      run_server: bool = True,
+                      exe: Optional[str] = None,
+                      extra_args: Optional[List[str]] = None,
+                      standalone=False):
+    '''Start a ripple server and return an app'''
+    try:
+        if extra_args is None:
+            extra_args = []
+        to_run = None
+        app = None
+        client = RippleClient(config=config, command_log=command_log, exe=exe)
+        if run_server:
+            to_run = [client.exe, '--conf', client.config_file_name]
+            if standalone:
+                to_run.append('-a')
+            fout = open(server_out, 'w')
+            p = subprocess.Popen(to_run + extra_args,
+                                 stdout=fout,
+                                 stderr=subprocess.STDOUT)
+            client.set_pid(p.pid)
+            print(
+                f'started rippled: config: {client.config_file_name} PID: {p.pid}',
+                flush=True)
+            time.sleep(1.5)  # give process time to startup
+        app = App(client=client, standalone=standalone)
+        yield app
+    finally:
+        if app:
+            app.shutdown()
+        if run_server and to_run:
+            subprocess.Popen(to_run + ['stop'],
+                             stdout=fout,
+                             stderr=subprocess.STDOUT)
+            p.wait()
+
+
+def configs_for_testnet(config_file_prefix: str) -> List[ConfigFile]:
+    configs = []
+    p = Path(config_file_prefix)
+    dir = p.parent
+    file = p.name
+    file_names = []
+    for f in os.listdir(dir):
+        cfg = os.path.join(dir, f, 'rippled.cfg')
+        if f.startswith(file) and os.path.exists(cfg):
+            file_names.append(cfg)
+    file_names.sort()
+    return [ConfigFile(file_name=f) for f in file_names]
+
+
+# Start an app for a network with the config files matched by `config_file_prefix*/rippled.cfg`
+
+
+# Undocumented feature: if the environment variable RIPPLED_SIDECHAIN_RR is set, it is
+# assumed to point to the rr executable. Sidechain server 0 will then be run under rr.
+@contextmanager
+def testnet_app(*,
+                exe: str,
+                configs: List[ConfigFile],
+                command_logs: Optional[List[str]] = None,
+                run_server: Optional[List[bool]] = None,
+                sidechain_rr: Optional[str] = None,
+                extra_args: Optional[List[List[str]]] = None):
+    '''Start a ripple testnet and return an app'''
+    try:
+        app = None
+        network = testnet.Network(exe,
+                                  configs,
+                                  command_logs=command_logs,
+                                  run_server=run_server,
+                                  with_rr=sidechain_rr,
+                                  extra_args=extra_args)
+        network.wait_for_validated_ledger()
+        app = App(network=network, standalone=False)
+        yield app
+    finally:
+        if app:
+            app.shutdown()

bin/sidechain/python/command.py

@@ -0,0 +1,563 @@
+import json
+from typing import List, Optional, Union
+
+from common import Account, Asset
+
+
+class Command:
+    '''Interface for all commands sent to the server'''
+
+    # command id useful for websocket messages
+    next_cmd_id = 1
+
+    def __init__(self):
+        self.cmd_id = Command.next_cmd_id
+        Command.next_cmd_id += 1
+
+    def cmd_name(self) -> str:
+        '''Return the command name for use in a command line'''
+        assert False
+        return ''
+
+    def get_command_line_list(self) -> List[str]:
+        '''Return a list of strings suitable for a command line command for a rippled server'''
+        return [self.cmd_name, json.dumps(self.to_cmd_obj())]
+
+    def get_websocket_dict(self) -> dict:
+        '''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket'''
+        result = self.to_cmd_obj()
+        return self.add_websocket_fields(result)
+
+    def to_cmd_obj(self) -> dict:
+        '''Return an object suitalbe for use in a command (input to json.dumps or similar)'''
+        assert False
+        return {}
+
+    def add_websocket_fields(self, cmd_dict: dict) -> dict:
+        cmd_dict['id'] = self.cmd_id
+        cmd_dict['command'] = self.cmd_name()
+        return cmd_dict
+
+    def _set_flag(self, flag_bit: int, value: bool = True):
+        '''Set or clear the flag bit'''
+        if value:
+            self.flags |= flag_bit
+        else:
+            self.flags &= ~flag_bit
+        return self
+
+
+class SubscriptionCommand(Command):
+    def __init__(self):
+        super().__init__()
+
+
+class PathFind(Command):
+    '''Rippled ripple_path_find command'''
+    def __init__(self,
+                 *,
+                 src: Account,
+                 dst: Account,
+                 amt: Asset,
+                 send_max: Optional[Asset] = None,
+                 src_currencies: Optional[List[Asset]] = None,
+                 ledger_hash: Optional[str] = None,
+                 ledger_index: Optional[Union[int, str]] = None):
+        super().__init__()
+        self.src = src
+        self.dst = dst
+        self.amt = amt
+        self.send_max = send_max
+        self.src_currencies = src_currencies
+        self.ledger_hash = ledger_hash
+        self.ledger_index = ledger_index
+
+    def cmd_name(self) -> str:
+        return 'ripple_path_find'
+
+    def add_websocket_fields(self, cmd_dict: dict) -> dict:
+        cmd_dict = super().add_websocket_fields(cmd_dict)
+        cmd_dict['subcommand'] = 'create'
+        return cmd_dict
+
+    def to_cmd_obj(self) -> dict:
+        '''convert to transaction form (suitable for using json.dumps or similar)'''
+        cmd = {
+            'source_account': self.src.account_id,
+            'destination_account': self.dst.account_id,
+            'destination_amount': self.amt.to_cmd_obj()
+        }
+        if self.send_max is not None:
+            cmd['send_max'] = self.send_max.to_cmd_obj()
+        if self.ledger_hash is not None:
+            cmd['ledger_hash'] = self.ledger_hash
+        if self.ledger_index is not None:
+            cmd['ledger_index'] = self.ledger_index
+        if self.src_currencies:
+            c = []
+            for sc in self.src_currencies:
+                d = {'currency': sc.currency, 'issuer': sc.issuer.account_id}
+                c.append(d)
+            cmd['source_currencies'] = c
+        return cmd
+
+
+class Sign(Command):
+    '''Rippled sign command'''
+    def __init__(self, secret: str, tx: dict):
+        super().__init__()
+        self.tx = tx
+        self.secret = secret
+
+    def cmd_name(self) -> str:
+        return 'sign'
+
+    def get_command_line_list(self) -> List[str]:
+        '''Return a list of strings suitable for a command line command for a rippled server'''
+        return [self.cmd_name(), self.secret, f'{json.dumps(self.tx)}']
+
+    def get_websocket_dict(self) -> dict:
+        '''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket'''
+        result = {'secret': self.secret, 'tx_json': self.tx}
+        return self.add_websocket_fields(result)
+
+
+class Submit(Command):
+    '''Rippled submit command'''
+    def __init__(self, tx_blob: str):
+        super().__init__()
+        self.tx_blob = tx_blob
+
+    def cmd_name(self) -> str:
+        return 'submit'
+
+    def get_command_line_list(self) -> List[str]:
+        '''Return a list of strings suitable for a command line command for a rippled server'''
+        return [self.cmd_name(), self.tx_blob]
+
+    def get_websocket_dict(self) -> dict:
+        '''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket'''
+        result = {'tx_blob': self.tx_blob}
+        return self.add_websocket_fields(result)
+
+
+class LedgerAccept(Command):
+    '''Rippled ledger_accept command'''
+    def __init__(self):
+        super().__init__()
+
+    def cmd_name(self) -> str:
+        return 'ledger_accept'
+
+    def get_command_line_list(self) -> List[str]:
+        '''Return a list of strings suitable for a command line command for a rippled server'''
+        return [self.cmd_name()]
+
+    def get_websocket_dict(self) -> dict:
+        '''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket'''
+        result = {}
+        return self.add_websocket_fields(result)
+
+
+class Stop(Command):
+    '''Rippled stop command'''
+    def __init__(self):
+        super().__init__()
+
+    def cmd_name(self) -> str:
+        return 'stop'
+
+    def get_command_line_list(self) -> List[str]:
+        '''Return a list of strings suitable for a command line command for a rippled server'''
+        return [self.cmd_name()]
+
+    def get_websocket_dict(self) -> dict:
+        '''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket'''
+        result = {}
+        return self.add_websocket_fields(result)
+
+
+class LogLevel(Command):
+    '''Rippled log_level command'''
+    def __init__(self, severity: str, *, partition: Optional[str] = None):
+        super().__init__()
+        self.severity = severity
+        self.partition = partition
+
+    def cmd_name(self) -> str:
+        return 'log_level'
+
+    def get_command_line_list(self) -> List[str]:
+        '''Return a list of strings suitable for a command line command for a rippled server'''
+        if self.partition is not None:
+            return [self.cmd_name(), self.partition, self.severity]
+        return [self.cmd_name(), self.severity]
+
+    def get_websocket_dict(self) -> dict:
+        '''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket'''
+        result = {'severity': self.severity}
+        if self.partition is not None:
+            result['partition'] = self.partition
+        return self.add_websocket_fields(result)
+
+
+class WalletPropose(Command):
+    '''Rippled wallet_propose command'''
+    def __init__(self,
+                 *,
+                 passphrase: Optional[str] = None,
+                 seed: Optional[str] = None,
+                 seed_hex: Optional[str] = None,
+                 key_type: Optional[str] = None):
+        super().__init__()
+        self.passphrase = passphrase
+        self.seed = seed
+        self.seed_hex = seed_hex
+        self.key_type = key_type
+
+    def cmd_name(self) -> str:
+        return 'wallet_propose'
+
+    def get_command_line_list(self) -> List[str]:
+        '''Return a list of strings suitable for a command line command for a rippled server'''
+        assert not self.seed and not self.seed_hex and (
+            not self.key_type or self.key_type == 'secp256k1')
+        if self.passphrase:
+            return [self.cmd_name(), self.passphrase]
+        return [self.cmd_name()]
+
+    def get_websocket_dict(self) -> dict:
+        '''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket'''
+        result = {}
+        if self.seed is not None:
+            result['seed'] = self.seed
+        if self.seed_hex is not None:
+            result['seed_hex'] = self.seed_hex
+        if self.passphrase is not None:
+            result['passphrase'] = self.passphrase
+        if self.key_type is not None:
+            result['key_type'] = self.key_type
+        return self.add_websocket_fields(result)
+
+
+class ValidationCreate(Command):
+    '''Rippled validation_create command'''
+    def __init__(self, *, secret: Optional[str] = None):
+        super().__init__()
+        self.secret = secret
+
+    def cmd_name(self) -> str:
+        return 'validation_create'
+
+    def get_command_line_list(self) -> List[str]:
+        '''Return a list of strings suitable for a command line command for a rippled server'''
+        if self.secret:
+            return [self.cmd_name(), self.secret]
+        return [self.cmd_name()]
+
+    def get_websocket_dict(self) -> dict:
+        '''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket'''
+        result = {}
+        if self.secret is not None:
+            result['secret'] = self.secret
+        return self.add_websocket_fields(result)
+
+
+class AccountInfo(Command):
+    '''Rippled account_info command'''
+    def __init__(self,
+                 account: Account,
+                 *,
+                 strict: Optional[bool] = None,
+                 ledger_hash: Optional[str] = None,
+                 ledger_index: Optional[Union[str, int]] = None,
+                 queue: Optional[bool] = None,
+                 signers_list: Optional[bool] = None):
+        super().__init__()
+        self.account = account
+        self.strict = strict
+        self.ledger_hash = ledger_hash
+        self.ledger_index = ledger_index
+        self.queue = queue
+        self.signers_list = signers_list
+        assert not ((ledger_hash is not None) and (ledger_index is not None))
+
+    def cmd_name(self) -> str:
+        return 'account_info'
+
+    def get_command_line_list(self) -> List[str]:
+        '''Return a list of strings suitable for a command line command for a rippled server'''
+        result = [self.cmd_name(), self.account.account_id]
+        if self.ledger_index is not None:
+            result.append(self.ledger_index)
+        if self.ledger_hash is not None:
+            result.append(self.ledger_hash)
+        if self.strict is not None:
+            result.append(self.strict)
+        return result
+
+    def get_websocket_dict(self) -> dict:
+        '''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket'''
+        result = {'account': self.account.account_id}
+        if self.ledger_index is not None:
+            result['ledger_index'] = self.ledger_index
+        if self.ledger_hash is not None:
+            result['ledger_hash'] = self.ledger_hash
+        if self.strict is not None:
+            result['strict'] = self.strict
+        if self.queue is not None:
+            result['queue'] = self.queue
+        return self.add_websocket_fields(result)
+
+
+class AccountLines(Command):
+    '''Rippled account_lines command'''
+    def __init__(self,
+                 account: Account,
+                 *,
+                 peer: Optional[Account] = None,
+                 ledger_hash: Optional[str] = None,
+                 ledger_index: Optional[Union[str, int]] = None,
+                 limit: Optional[int] = None,
+                 marker=None):
+        super().__init__()
+        self.account = account
+        self.peer = peer
+        self.ledger_hash = ledger_hash
+        self.ledger_index = ledger_index
+        self.limit = limit
+        self.marker = marker
+        assert not ((ledger_hash is not None) and (ledger_index is not None))
+
+    def cmd_name(self) -> str:
+        return 'account_lines'
+
+    def get_command_line_list(self) -> List[str]:
+        '''Return a list of strings suitable for a command line command for a rippled server'''
+        assert sum(x is None for x in [
+            self.ledger_index, self.ledger_hash, self.limit, self.marker
+        ]) == 4
+        result = [self.cmd_name(), self.account.account_id]
+        if self.peer is not None:
+            result.append(self.peer)
+        return result
+
+    def get_websocket_dict(self) -> dict:
+        '''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket'''
+        result = {'account': self.account.account_id}
+        if self.peer is not None:
+            result['peer'] = self.peer
+        if self.ledger_index is not None:
+            result['ledger_index'] = self.ledger_index
+        if self.ledger_hash is not None:
+            result['ledger_hash'] = self.ledger_hash
+        if self.limit is not None:
+            result['limit'] = self.limit
+        if self.marker is not None:
+            result['marker'] = self.marker
+        return self.add_websocket_fields(result)
+
+
+class AccountTx(Command):
+    def __init__(self,
+                 account: Account,
+                 *,
+                 limit: Optional[int] = None,
+                 marker=None):
+        super().__init__()
+        self.account = account
+        self.limit = limit
+        self.marker = marker
+
+    def cmd_name(self) -> str:
+        return 'account_tx'
+
+    def get_command_line_list(self) -> List[str]:
+        '''Return a list of strings suitable for a command line command for a rippled server'''
+        result = [self.cmd_name(), self.account.account_id]
+        return result
+
+    def get_websocket_dict(self) -> dict:
+        '''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket'''
+        result = {'account': self.account.account_id}
+        if self.limit is not None:
+            result['limit'] = self.limit
+        if self.marker is not None:
+            result['marker'] = self.marker
+        return self.add_websocket_fields(result)
+
+
+class BookOffers(Command):
+    '''Rippled book_offers command'''
+    def __init__(self,
+                 taker_pays: Asset,
+                 taker_gets: Asset,
+                 *,
+                 taker: Optional[Account] = None,
+                 ledger_hash: Optional[str] = None,
+                 ledger_index: Optional[Union[str, int]] = None,
+                 limit: Optional[int] = None,
+                 marker=None):
+        super().__init__()
+        self.taker_pays = taker_pays
+        self.taker_gets = taker_gets
+        self.taker = taker
+        self.ledger_hash = ledger_hash
+        self.ledger_index = ledger_index
+        self.limit = limit
+        self.marker = marker
+        assert not ((ledger_hash is not None) and (ledger_index is not None))
+
+    def cmd_name(self) -> str:
+        return 'book_offers'
+
+    def get_command_line_list(self) -> List[str]:
+        '''Return a list of strings suitable for a command line command for a rippled server'''
+        assert sum(x is None for x in [
+            self.ledger_index, self.ledger_hash, self.limit, self.marker
+        ]) == 4
+        return [
+            self.cmd_name(),
+            self.taker_pays.cmd_str(),
+            self.taker_gets.cmd_str()
+        ]
+
+    def get_websocket_dict(self) -> dict:
+        '''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket'''
+        result = {
+            'taker_pays': self.taker_pays.to_cmd_obj(),
+            'taker_gets': self.taker_gets.to_cmd_obj()
+        }
+        if self.taker is not None:
+            result['taker'] = self.taker.account_id
+        if self.ledger_index is not None:
+            result['ledger_index'] = self.ledger_index
+        if self.ledger_hash is not None:
+            result['ledger_hash'] = self.ledger_hash
+        if self.limit is not None:
+            result['limit'] = self.limit
+        if self.marker is not None:
+            result['marker'] = self.marker
+        return self.add_websocket_fields(result)
+
+
+class BookSubscription:
+    '''Spec for a book in a subscribe command'''
+    def __init__(self,
+                 taker_pays: Asset,
+                 taker_gets: Asset,
+                 *,
+                 taker: Optional[Account] = None,
+                 snapshot: Optional[bool] = None,
+                 both: Optional[bool] = None):
+        self.taker_pays = taker_pays
+        self.taker_gets = taker_gets
+        self.taker = taker
+        self.snapshot = snapshot
+        self.both = both
+
+    def to_cmd_obj(self) -> dict:
+        '''Return an object suitalbe for use in a command'''
+        result = {
+            'taker_pays': self.taker_pays.to_cmd_obj(),
+            'taker_gets': self.taker_gets.to_cmd_obj()
+        }
+        if self.taker is not None:
+            result['taker'] = self.taker.account_id
+        if self.snapshot is not None:
+            result['snapshot'] = self.snapshot
+        if self.both is not None:
+            result['both'] = self.both
+        return result
+
+
+class ServerInfo(Command):
+    '''Rippled server_info command'''
+    def __init__(self):
+        super().__init__()
+
+    def cmd_name(self) -> str:
+        return 'server_info'
+
+    def get_command_line_list(self) -> List[str]:
+        '''Return a list of strings suitable for a command line command for a rippled server'''
+        return [self.cmd_name()]
+
+    def get_websocket_dict(self) -> dict:
+        '''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket'''
+        result = {}
+        return self.add_websocket_fields(result)
+
+
+class FederatorInfo(Command):
+    '''Rippled federator_info command'''
+    def __init__(self):
+        super().__init__()
+
+    def cmd_name(self) -> str:
+        return 'federator_info'
+
+    def get_command_line_list(self) -> List[str]:
+        '''Return a list of strings suitable for a command line command for a rippled server'''
+        return [self.cmd_name()]
+
+    def get_websocket_dict(self) -> dict:
+        '''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket'''
+        result = {}
+        return self.add_websocket_fields(result)
+
+
+class Subscribe(SubscriptionCommand):
+    '''The subscribe method requests periodic notifications from the server
+    when certain events happen. See: https://developers.ripple.com/subscribe.html'''
+    def __init__(
+            self,
+            *,
+            streams: Optional[List[str]] = None,
+            accounts: Optional[List[Account]] = None,
+            accounts_proposed: Optional[List[Account]] = None,
+            account_history_account: Optional[Account] = None,
+            books: Optional[
+                List[BookSubscription]] = None,  # taker_pays, taker_gets
+            url: Optional[str] = None,
+            url_username: Optional[str] = None,
+            url_password: Optional[str] = None):
+        super().__init__()
+        self.streams = streams
+        self.accounts = accounts
+        self.account_history_account = account_history_account
+        self.accounts_proposed = accounts_proposed
+        self.books = books
+        self.url = url
+        self.url_username = url_username
+        self.url_password = url_password
+        self.websocket = None
+
+    def cmd_name(self) -> str:
+        if self.websocket:
+            return 'unsubscribe'
+        return 'subscribe'
+
+    def to_cmd_obj(self) -> dict:
+        d = {}
+        if self.streams is not None:
+            d['streams'] = self.streams
+        if self.accounts is not None:
+            d['accounts'] = [a.account_id for a in self.accounts]
+        if self.account_history_account is not None:
+            d['account_history_tx_stream'] = {
+                'account': self.account_history_account.account_id
+            }
+        if self.accounts_proposed is not None:
+            d['accounts_proposed'] = [
+                a.account_id for a in self.accounts_proposed
+            ]
+        if self.books is not None:
+            d['books'] = [b.to_cmd_obj() for b in self.books]
+        if self.url is not None:
+            d['url'] = self.url
+        if self.url_username is not None:
+            d['url_username'] = self.url_username
+        if self.url_password is not None:
+            d['url_password'] = self.url_password
+        return d

bin/sidechain/python/common.py

@@ -0,0 +1,256 @@
+import binascii
+import datetime
+import logging
+from typing import List, Optional, Union
+import pandas as pd
+import pytz
+import sys
+
+EPRINT_ENABLED = True
+
+
+def disable_eprint():
+    global EPRINT_ENABLED
+    EPRINT_ENABLED = False
+
+
+def enable_eprint():
+    global EPRINT_ENABLED
+    EPRINT_ENABLED = True
+
+
+def eprint(*args, **kwargs):
+    if not EPRINT_ENABLED:
+        return
+    logging.error(*args)
+    print(*args, file=sys.stderr, flush=True, **kwargs)
+
+
+def to_rippled_epoch(d: datetime.datetime) -> int:
+    '''Convert from a datetime to the number of seconds since Jan 1, 2000 (rippled epoch)'''
+    start = datetime.datetime(2000, 1, 1, tzinfo=pytz.utc)
+    return int((d - start).total_seconds())
+
+
+class Account:  # pylint: disable=too-few-public-methods
+    '''
+    Account in the ripple ledger
+    '''
+    def __init__(self,
+                 *,
+                 account_id: Optional[str] = None,
+                 nickname: Optional[str] = None,
+                 public_key: Optional[str] = None,
+                 public_key_hex: Optional[str] = None,
+                 secret_key: Optional[str] = None,
+                 result_dict: Optional[dict] = None):
+        self.account_id = account_id
+        self.nickname = nickname
+        self.public_key = public_key
+        self.public_key_hex = public_key_hex
+        self.secret_key = secret_key
+
+        if result_dict is not None:
+            self.account_id = result_dict['account_id']
+            self.public_key = result_dict['public_key']
+            self.public_key_hex = result_dict['public_key_hex']
+            self.secret_key = result_dict['master_seed']
+
+    # Accounts are equal if they represent the same account on the ledger
+    # I.e. only check the account_id field for equality.
+    def __eq__(self, lhs):
+        if not isinstance(lhs, self.__class__):
+            return False
+        return self.account_id == lhs.account_id
+
+    def __ne__(self, lhs):
+        return not self.__eq__(lhs)
+
+    def __str__(self) -> str:
+        if self.nickname is not None:
+            return self.nickname
+        return self.account_id
+
+    def alias_or_account_id(self) -> str:
+        '''
+        return the alias if it exists, otherwise return the id
+        '''
+        if self.nickname is not None:
+            return self.nickname
+        return self.account_id
+
+    def account_id_str_as_hex(self) -> str:
+        return binascii.hexlify(self.account_id.encode()).decode('utf-8')
+
+    def to_cmd_obj(self) -> dict:
+        return {
+            'account_id': self.account_id,
+            'nickname': self.nickname,
+            'public_key': self.public_key,
+            'public_key_hex': self.public_key_hex,
+            'secret_key': self.secret_key
+        }
+
+
+class Asset:
+    '''An XRP or IOU value'''
+    def __init__(
+        self,
+        *,
+        value: Union[int, float, None] = None,
+        currency: Optional[
+            str] = None,  # Will default to 'XRP' if not specified
+        issuer: Optional[Account] = None,
+        from_asset=None,  # asset is of type Optional[Asset]
+        # from_rpc_result is a python object resulting from an rpc command
+        from_rpc_result: Optional[Union[dict, str]] = None):
+
+        assert from_asset is None or from_rpc_result is None
+
+        self.value = value
+        self.issuer = issuer
+        self.currency = currency
+        if from_asset is not None:
+            if self.value is None:
+                self.value = from_asset.value
+            if self.issuer is None:
+                self.issuer = from_asset.issuer
+            if self.currency is None:
+                self.currency = from_asset.currency
+        if from_rpc_result is not None:
+            if isinstance(from_rpc_result, str):
+                self.value = int(from_rpc_result)
+                self.currency = 'XRP'
+            else:
+                self.value = from_rpc_result['value']
+                self.currency = float(from_rpc_result['currency'])
+                self.issuer = Account(account_id=from_rpc_result['issuer'])
+
+        if self.currency is None:
+            self.currency = 'XRP'
+
+        if isinstance(self.value, str):
+            if self.is_xrp():
+                self.value = int(value)
+            else:
+                self.value = float(value)
+
+    def __call__(self, value: Union[int, float]):
+        '''Call operator useful for a terse syntax for assets in tests. I.e. USD(100)'''
+        return Asset(value=value, from_asset=self)
+
+    def __add__(self, lhs):
+        assert (self.issuer == lhs.issuer and self.currency == lhs.currency)
+        return Asset(value=self.value + lhs.value,
+                     currency=self.currency,
+                     issuer=self.issuer)
+
+    def __sub__(self, lhs):
+        assert (self.issuer == lhs.issuer and self.currency == lhs.currency)
+        return Asset(value=self.value - lhs.value,
+                     currency=self.currency,
+                     issuer=self.issuer)
+
+    def __eq__(self, lhs):
+        if not isinstance(lhs, self.__class__):
+            return False
+        return (self.value == lhs.value and self.currency == lhs.currency
+                and self.issuer == lhs.issuer)
+
+    def __ne__(self, lhs):
+        return not self.__eq__(lhs)
+
+    def __str__(self) -> str:
+        value_part = '' if self.value is None else f'{self.value}/'
+        issuer_part = '' if self.issuer is None else f'/{self.issuer}'
+        return f'{value_part}{self.currency}{issuer_part}'
+
+    def __repr__(self) -> str:
+        return self.__str__()
+
+    def is_xrp(self) -> bool:
+        ''' return true if the asset represents XRP'''
+        return self.currency == 'XRP'
+
+    def cmd_str(self) -> str:
+        value_part = '' if self.value is None else f'{self.value}/'
+        issuer_part = '' if self.issuer is None else f'/{self.issuer.account_id}'
+        return f'{value_part}{self.currency}{issuer_part}'
+
+    def to_cmd_obj(self) -> dict:
+        '''Return an object suitalbe for use in a command'''
+        if self.currency == 'XRP':
+            if self.value is not None:
+                return f'{self.value}'  # must be a string
+            return {'currency': self.currency}
+        result = {'currency': self.currency, 'issuer': self.issuer.account_id}
+        if self.value is not None:
+            result['value'] = f'{self.value}'  # must be a string
+        return result
+
+
+def XRP(v: Union[int, float]) -> Asset:
+    return Asset(value=v * 1_000_000)
+
+
+def drops(v: int) -> Asset:
+    return Asset(value=v)
+
+
+class Path:
+    '''Payment Path'''
+    def __init__(self,
+                 nodes: Optional[List[Union[Account, Asset]]] = None,
+                 *,
+                 result_list: Optional[List[dict]] = None):
+        assert not (nodes and result_list)
+        if result_list is not None:
+            self.result_list = result_list
+            return
+        if nodes is None:
+            self.result_list = []
+            return
+        self.result_list = [
+            self._create_account_path_node(n)
+            if isinstance(n, Account) else self._create_currency_path_node(n)
+            for n in nodes
+        ]
+
+    def _create_account_path_node(self, account: Account) -> dict:
+        return {
+            'account': account.account_id,
+            'type': 1,
+            'type_hex': '0000000000000001'
+        }
+
+    def _create_currency_path_node(self, asset: Asset) -> dict:
+        result = {
+            'currency': asset.currency,
+            'type': 48,
+            'type_hex': '0000000000000030'
+        }
+        if not asset.is_xrp():
+            result['issuer'] = asset.issuer.account_id
+        return result
+
+    def to_cmd_obj(self) -> list:
+        '''Return an object suitalbe for use in a command'''
+        return self.result_list
+
+
+class PathList:
+    '''Collection of paths for use in payments'''
+    def __init__(self,
+                 path_list: Optional[List[Path]] = None,
+                 *,
+                 result_list: Optional[List[List[dict]]] = None):
+        # result_list can be the response from the rippled server
+        assert not (path_list and result_list)
+        if result_list is not None:
+            self.paths = [Path(result_list=l) for l in result_list]
+            return
+        self.paths = path_list
+
+    def to_cmd_obj(self) -> list:
+        '''Return an object suitalbe for use in a command'''
+        return [p.to_cmd_obj() for p in self.paths]

bin/sidechain/python/config_file.py

@@ -0,0 +1,101 @@
+from typing import List, Optional, Tuple
+
+
+class Section:
+    def section_header(l: str) -> Optional[str]:
+        '''
+        If the line is a section header, return the section name
+        otherwise return None
+        '''
+        if l.startswith('[') and l.endswith(']'):
+            return l[1:-1]
+        return None
+
+    def __init__(self, name: str):
+        super().__setattr__('_name', name)
+        # lines contains all non key-value pairs
+        super().__setattr__('_lines', [])
+        super().__setattr__('_kv_pairs', {})
+
+    def get_name(self):
+        return self._name
+
+    def add_line(self, l):
+        s = l.split('=')
+        if len(s) == 2:
+            self._kv_pairs[s[0].strip()] = s[1].strip()
+        else:
+            self._lines.append(l)
+
+    def get_lines(self):
+        return self._lines
+
+    def get_line(self) -> Optional[str]:
+        if len(self._lines) > 0:
+            return self._lines[0]
+        return None
+
+    def __getattr__(self, name):
+        try:
+            return self._kv_pairs[name]
+        except KeyError:
+            raise AttributeError(name)
+
+    def __setattr__(self, name, value):
+        if name in self.__dict__:
+            super().__setattr__(name, value)
+        else:
+            self._kv_pairs[name] = value
+
+    def __getstate__(self):
+        return vars(self)
+
+    def __setstate__(self, state):
+        vars(self).update(state)
+
+
+class ConfigFile:
+    def __init__(self, *, file_name: Optional[str] = None):
+        # parse the file
+        self._file_name = file_name
+        self._sections = {}
+        if not file_name:
+            return
+
+        cur_section = None
+        with open(file_name) as f:
+            for n, l in enumerate(f):
+                l = l.strip()
+                if l.startswith('#') or not l:
+                    continue
+                if section_name := Section.section_header(l):
+                    if cur_section:
+                        self.add_section(cur_section)
+                    cur_section = Section(section_name)
+                    continue
+                if not cur_section:
+                    raise ValueError(
+                        f'Error parsing config file: {file_name} line_num: {n} line: {l}'
+                    )
+                cur_section.add_line(l)
+
+        if cur_section:
+            self.add_section(cur_section)
+
+    def add_section(self, s: Section):
+        self._sections[s.get_name()] = s
+
+    def get_file_name(self):
+        return self._file_name
+
+    def __getstate__(self):
+        return vars(self)
+
+    def __setstate__(self, state):
+        vars(self).update(state)
+
+    def __getattr__(self, name):
+        try:
+            return self._sections[name]
+        except KeyError:
+            raise AttributeError(name)

bin/sidechain/python/create_config_files.py

@@ -0,0 +1,630 @@
+#!/usr/bin/env python3
+
+# Generate rippled config files, each with their own ports, database paths, and validation_seeds.
+# There will be configs for shards/no_shards, main/test nets, two config files for each combination
+# (so one can run in a dogfood mode while another is tested). To avoid confusion,The directory path
+# will be $data_dir/{main | test}.{shard | no_shard}.{dog | test}
+# The config file will reside in that directory with the name rippled.cfg
+# The validators file will reside in that directory with the name validators.txt
+'''
+Script to test and debug sidechains.
+
+The rippled exe location can be set through the command line or
+the environment variable RIPPLED_MAINCHAIN_EXE
+
+The configs_dir (where the config files will reside) can be set through the command line
+or the environment variable RIPPLED_SIDECHAIN_CFG_DIR
+'''
+
+import argparse
+from dataclasses import dataclass
+import json
+import os
+from pathlib import Path
+import sys
+from typing import Dict, List, Optional, Tuple, Union
+
+from config_file import ConfigFile
+from command import ValidationCreate, WalletPropose
+from common import Account, Asset, eprint, XRP
+from app import App, single_client_app
+
+mainnet_validators = """
+[validator_list_sites]
+https://vl.ripple.com
+
+[validator_list_keys]
+ED2677ABFFD1B33AC6FBC3062B71F1E8397C1505E1C42C64D11AD1B28FF73F4734
+"""
+
+altnet_validators = """
+[validator_list_sites]
+https://vl.altnet.rippletest.net
+
+[validator_list_keys]
+ED264807102805220DA0F312E71FC2C69E1552C9C5790F6C25E3729DEB573D5860
+"""
+
+node_size = 'medium'
+default_data_dir = '/home/swd/data/rippled'
+
+
+@dataclass
+class Keypair:
+    public_key: str
+    secret_key: str
+    account_id: Optional[str]
+
+
+def generate_node_keypairs(n: int, rip: App) -> List[Keypair]:
+    '''
+    generate keypairs suitable for validator keys
+    '''
+    result = []
+    for i in range(n):
+        keys = rip(ValidationCreate())
+        result.append(
+            Keypair(public_key=keys["validation_public_key"],
+                    secret_key=keys["validation_seed"],
+                    account_id=None))
+    return result
+
+
+def generate_federator_keypairs(n: int, rip: App) -> List[Keypair]:
+    '''
+    generate keypairs suitable for federator keys
+    '''
+    result = []
+    for i in range(n):
+        keys = rip(WalletPropose(key_type='ed25519'))
+        result.append(
+            Keypair(public_key=keys["public_key"],
+                    secret_key=keys["master_seed"],
+                    account_id=keys["account_id"]))
+    return result
+
+
+class Ports:
+    '''
+    Port numbers for various services.
+    Port numbers differ by cfg_index so different configs can run
+    at the same time without interfering with each other.
+    '''
+    peer_port_base = 51235
+    http_admin_port_base = 5005
+    ws_public_port_base = 6005
+
+    def __init__(self, cfg_index: int):
+        self.peer_port = Ports.peer_port_base + cfg_index
+        self.http_admin_port = Ports.http_admin_port_base + cfg_index
+        self.ws_public_port = Ports.ws_public_port_base + (2 * cfg_index)
+        # note admin port uses public port base
+        self.ws_admin_port = Ports.ws_public_port_base + (2 * cfg_index) + 1
+
+
+class Network:
+    def __init__(self, num_nodes: int, num_validators: int,
+                 start_cfg_index: int, rip: App):
+        self.validator_keypairs = generate_node_keypairs(num_validators, rip)
+        self.ports = [Ports(start_cfg_index + i) for i in range(num_nodes)]
+
+
+class SidechainNetwork(Network):
+    def __init__(self, num_nodes: int, num_federators: int,
+                 num_validators: int, start_cfg_index: int, rip: App):
+        super().__init__(num_nodes, num_validators, start_cfg_index, rip)
+        self.federator_keypairs = generate_federator_keypairs(
+            num_federators, rip)
+        self.main_account = rip(WalletPropose(key_type='secp256k1'))
+
+
+class XChainAsset:
+    def __init__(self, main_asset: Asset, side_asset: Asset,
+                 main_value: Union[int, float], side_value: Union[int, float],
+                 main_refund_penalty: Union[int, float],
+                 side_refund_penalty: Union[int, float]):
+        self.main_asset = main_asset(main_value)
+        self.side_asset = side_asset(side_value)
+        self.main_refund_penalty = main_asset(main_refund_penalty)
+        self.side_refund_penalty = side_asset(side_refund_penalty)
+
+
+def generate_asset_stanzas(
+        assets: Optional[Dict[str, XChainAsset]] = None) -> str:
+    if assets is None:
+        # default to xrp only at a 1:1 value
+        assets = {}
+        assets['xrp_xrp_sidechain_asset'] = XChainAsset(
+            XRP(0), XRP(0), 1, 1, 400, 400)
+
+    index_stanza = """
+[sidechain_assets]"""
+
+    asset_stanzas = []
+
+    for name, xchainasset in assets.items():
+        index_stanza += '\n' + name
+        new_stanza = f"""
+[{name}]
+mainchain_asset={json.dumps(xchainasset.main_asset.to_cmd_obj())}
+sidechain_asset={json.dumps(xchainasset.side_asset.to_cmd_obj())}
+mainchain_refund_penalty={json.dumps(xchainasset.main_refund_penalty.to_cmd_obj())}
+sidechain_refund_penalty={json.dumps(xchainasset.side_refund_penalty.to_cmd_obj())}"""
+        asset_stanzas.append(new_stanza)
+
+    return index_stanza + '\n' + '\n'.join(asset_stanzas)
+
+
+# First element of the returned tuple is the sidechain stanzas
+# second element is the bootstrap stanzas
+def generate_sidechain_stanza(
+        mainchain_ports: Ports,
+        main_account: dict,
+        federators: List[Keypair],
+        signing_key: str,
+        mainchain_cfg_file: str,
+        xchain_assets: Optional[Dict[str,
+                                     XChainAsset]] = None) -> Tuple[str, str]:
+    mainchain_ip = "127.0.0.1"
+
+    federators_stanza = """
+# federator signing public keys
+[sidechain_federators]
+"""
+    federators_secrets_stanza = """
+# federator signing secret keys (for standalone-mode testing only; Normally won't be in a config file)
+[sidechain_federators_secrets]
+"""
+    bootstrap_federators_stanza = """
+# first value is federator signing public key, second is the signing pk account
+[sidechain_federators]
+"""
+
+    assets_stanzas = generate_asset_stanzas(xchain_assets)
+
+    for fed in federators:
+        federators_stanza += f'{fed.public_key}\n'
+        federators_secrets_stanza += f'{fed.secret_key}\n'
+        bootstrap_federators_stanza += f'{fed.public_key} {fed.account_id}\n'
+
+    sidechain_stanzas = f"""
+[sidechain]
+signing_key={signing_key}
+mainchain_account={main_account["account_id"]}
+mainchain_ip={mainchain_ip}
+mainchain_port_ws={mainchain_ports.ws_public_port}
+# mainchain config file is: {mainchain_cfg_file}
+
+{assets_stanzas}
+
+{federators_stanza}
+
+{federators_secrets_stanza}
+"""
+    bootstrap_stanzas = f"""
+[sidechain]
+mainchain_secret={main_account["master_seed"]}
+
+{bootstrap_federators_stanza}
+"""
+    return (sidechain_stanzas, bootstrap_stanzas)
+
+
+# cfg_type will typically be either 'dog' or 'test', but can be any string. It is only used
+# to create the data directories.
+def generate_cfg_dir(*,
+                     ports: Ports,
+                     with_shards: bool,
+                     main_net: bool,
+                     cfg_type: str,
+                     sidechain_stanza: str,
+                     sidechain_bootstrap_stanza: str,
+                     validation_seed: Optional[str] = None,
+                     validators: Optional[List[str]] = None,
+                     fixed_ips: Optional[List[Ports]] = None,
+                     data_dir: str,
+                     full_history: bool = False,
+                     with_hooks: bool = False) -> str:
+    ips_stanza = ''
+    this_ip = '127.0.0.1'
+    if fixed_ips:
+        ips_stanza = '# Fixed ips for a testnet.\n'
+        ips_stanza += '[ips_fixed]\n'
+        for i, p in enumerate(fixed_ips):
+            if p.peer_port == ports.peer_port:
+                continue
+            # rippled limits the number of connects per ip. So use the other loopback devices
+            ips_stanza += f'127.0.0.{i+1} {p.peer_port}\n'
+    else:
+        ips_stanza = '# Where to find some other servers speaking the Ripple protocol.\n'
+        ips_stanza += '[ips]\n'
+        if main_net:
+            ips_stanza += 'r.ripple.com 51235\n'
+        else:
+            ips_stanza += 'r.altnet.rippletest.net 51235\n'
+    disable_shards = '' if with_shards else '# '
+    disable_delete = '#' if full_history else ''
+    history_line = 'full' if full_history else '256'
+    earliest_seq_line = ''
+    if sidechain_stanza:
+        earliest_seq_line = 'earliest_seq=1'
+    hooks_line = 'Hooks' if with_hooks else ''
+    validation_seed_stanza = ''
+    if validation_seed:
+        validation_seed_stanza = f'''
+[validation_seed]
+{validation_seed}
+        '''
+    node_size = 'medium'
+    shard_str = 'shards' if with_shards else 'no_shards'
+    net_str = 'main' if main_net else 'test'
+    if not fixed_ips:
+        sub_dir = data_dir + f'/{net_str}.{shard_str}.{cfg_type}'
+        if sidechain_stanza:
+            sub_dir += '.sidechain'
+    else:
+        sub_dir = data_dir + f'/{cfg_type}'
+    db_path = sub_dir + '/db'
+    debug_logfile = sub_dir + '/debug.log'
+    shard_db_path = sub_dir + '/shards'
+    node_db_path = db_path + '/nudb'
+
+    cfg_str = f"""
+[server]
+port_rpc_admin_local
+port_peer
+port_ws_admin_local
+port_ws_public
+#ssl_key = /etc/ssl/private/server.key
+#ssl_cert = /etc/ssl/certs/server.crt
+
+[port_rpc_admin_local]
+port = {ports.http_admin_port}
+ip = {this_ip}
+admin = {this_ip}
+protocol = http
+
+[port_peer]
+port = {ports.peer_port}
+ip = 0.0.0.0
+protocol = peer
+
+[port_ws_admin_local]
+port = {ports.ws_admin_port}
+ip = {this_ip}
+admin = {this_ip}
+protocol = ws
+
+[port_ws_public]
+port = {ports.ws_public_port}
+ip = {this_ip}
+protocol = ws
+# protocol = wss
+
+[node_size]
+{node_size}
+
+[ledger_history]
+{history_line}
+
+[node_db]
+type=NuDB
+path={node_db_path}
+open_files=2000
+filter_bits=12
+cache_mb=256
+file_size_mb=8
+file_size_mult=2
+{earliest_seq_line}
+{disable_delete}online_delete=256
+{disable_delete}advisory_delete=0
+
+[database_path]
+{db_path}
+
+# This needs to be an absolute directory reference, not a relative one.
+# Modify this value as required.
+[debug_logfile]
+{debug_logfile}
+
+[sntp_servers]
+time.windows.com
+time.apple.com
+time.nist.gov
+pool.ntp.org
+
+{ips_stanza}
+
+[validators_file]
+validators.txt
+
+[rpc_startup]
+{{ "command": "log_level", "severity": "fatal" }}
+{{ "command": "log_level", "partition": "SidechainFederator", "severity": "trace" }}
+
+[ssl_verify]
+1
+
+{validation_seed_stanza}
+
+{disable_shards}[shard_db]
+{disable_shards}type=NuDB
+{disable_shards}path={shard_db_path}
+{disable_shards}max_historical_shards=6
+
+{sidechain_stanza}
+
+[features]
+{hooks_line}
+PayChan
+Flow
+FlowCross
+TickSize
+fix1368
+Escrow
+fix1373
+EnforceInvariants
+SortedDirectories
+fix1201
+fix1512
+fix1513
+fix1523
+fix1528
+DepositAuth
+Checks
+fix1571
+fix1543
+fix1623
+DepositPreauth
+fix1515
+fix1578
+MultiSignReserve
+fixTakerDryOfferRemoval
+fixMasterKeyAsRegularKey
+fixCheckThreading
+fixPayChanRecipientOwnerDir
+DeletableAccounts
+fixQualityUpperBound
+RequireFullyCanonicalSig
+fix1781 
+HardenedValidations
+fixAmendmentMajorityCalc
+NegativeUNL
+TicketBatch
+FlowSortStrands
+fixSTAmountCanonicalize
+fixRmSmallIncreasedQOffers
+CheckCashMakesTrustLine
+"""
+
+    validators_str = ''
+    for p in [sub_dir, db_path, shard_db_path]:
+        Path(p).mkdir(parents=True, exist_ok=True)
+    # Add the validators.txt file
+    if validators:
+        validators_str = '[validators]\n'
+        for k in validators:
+            validators_str += f'{k}\n'
+    else:
+        validators_str = mainnet_validators if main_net else altnet_validators
+    with open(sub_dir + "/validators.txt", "w") as f:
+        f.write(validators_str)
+
+    # add the rippled.cfg file
+    with open(sub_dir + "/rippled.cfg", "w") as f:
+        f.write(cfg_str)
+
+    if sidechain_bootstrap_stanza:
+        # add the bootstrap file
+        with open(sub_dir + "/sidechain_bootstrap.cfg", "w") as f:
+            f.write(sidechain_bootstrap_stanza)
+
+    return sub_dir + "/rippled.cfg"
+
+
+def generate_multinode_net(out_dir: str,
+                           mainnet: Network,
+                           sidenet: SidechainNetwork,
+                           xchain_assets: Optional[Dict[str,
+                                                        XChainAsset]] = None):
+    mainnet_cfgs = []
+    for i in range(len(mainnet.ports)):
+        validator_kp = mainnet.validator_keypairs[i]
+        ports = mainnet.ports[i]
+        mainchain_cfg_file = generate_cfg_dir(
+            ports=ports,
+            with_shards=False,
+            main_net=True,
+            cfg_type=f'mainchain_{i}',
+            sidechain_stanza='',
+            sidechain_bootstrap_stanza='',
+            validation_seed=validator_kp.secret_key,
+            data_dir=out_dir)
+        mainnet_cfgs.append(mainchain_cfg_file)
+
+    for i in range(len(sidenet.ports)):
+        validator_kp = sidenet.validator_keypairs[i]
+        ports = sidenet.ports[i]
+
+        mainnet_i = i % len(mainnet.ports)
+        sidechain_stanza, sidechain_bootstrap_stanza = generate_sidechain_stanza(
+            mainnet.ports[mainnet_i], sidenet.main_account,
+            sidenet.federator_keypairs,
+            sidenet.federator_keypairs[i].secret_key, mainnet_cfgs[mainnet_i],
+            xchain_assets)
+
+        generate_cfg_dir(
+            ports=ports,
+            with_shards=False,
+            main_net=True,
+            cfg_type=f'sidechain_{i}',
+            sidechain_stanza=sidechain_stanza,
+            sidechain_bootstrap_stanza=sidechain_bootstrap_stanza,
+            validation_seed=validator_kp.secret_key,
+            validators=[kp.public_key for kp in sidenet.validator_keypairs],
+            fixed_ips=sidenet.ports,
+            data_dir=out_dir,
+            full_history=True,
+            with_hooks=False)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description=('Create config files for testing sidechains'))
+
+    parser.add_argument(
+        '--exe',
+        '-e',
+        help=('path to rippled executable'),
+    )
+
+    parser.add_argument(
+        '--usd',
+        '-u',
+        action='store_true',
+        help=('include a USD/root IOU asset for cross chain transfers'),
+    )
+
+    parser.add_argument(
+        '--cfgs_dir',
+        '-c',
+        help=
+        ('path to configuration file dir (where the output config files will be located)'
+         ),
+    )
+
+    return parser.parse_known_args()[0]
+
+
+class Params:
+    def __init__(self):
+        args = parse_args()
+
+        self.exe = None
+        if 'RIPPLED_MAINCHAIN_EXE' in os.environ:
+            self.exe = os.environ['RIPPLED_MAINCHAIN_EXE']
+        if args.exe:
+            self.exe = args.exe
+
+        self.configs_dir = None
+        if 'RIPPLED_SIDECHAIN_CFG_DIR' in os.environ:
+            self.configs_dir = os.environ['RIPPLED_SIDECHAIN_CFG_DIR']
+        if args.cfgs_dir:
+            self.configs_dir = args.cfgs_dir
+
+        self.usd = False
+        if args.usd:
+            self.usd = args.usd
+
+    def check_error(self) -> str:
+        '''
+        Check for errors. Return `None` if no errors,
+        otherwise return a string describing the error
+        '''
+        if not self.exe:
+            return 'Missing exe location. Either set the env variable RIPPLED_MAINCHAIN_EXE or use the --exe_mainchain command line switch'
+        if not self.configs_dir:
+            return 'Missing configs directory location. Either set the env variable RIPPLED_SIDECHAIN_CFG_DIR or use the --cfgs_dir command line switch'
+
+
+def main(params: Params,
+         xchain_assets: Optional[Dict[str, XChainAsset]] = None):
+
+    if err_str := params.check_error():
+        eprint(err_str)
+        sys.exit(1)
+    index = 0
+    nonvalidator_cfg_file_name = generate_cfg_dir(
+        ports=Ports(index),
+        with_shards=False,
+        main_net=True,
+        cfg_type='non_validator',
+        sidechain_stanza='',
+        sidechain_bootstrap_stanza='',
+        validation_seed=None,
+        data_dir=params.configs_dir)
+    index = index + 1
+
+    nonvalidator_config = ConfigFile(file_name=nonvalidator_cfg_file_name)
+    with single_client_app(exe=params.exe,
+                           config=nonvalidator_config,
+                           standalone=True) as rip:
+        mainnet = Network(num_nodes=1,
+                          num_validators=1,
+                          start_cfg_index=index,
+                          rip=rip)
+        sidenet = SidechainNetwork(num_nodes=5,
+                                   num_federators=5,
+                                   num_validators=5,
+                                   start_cfg_index=index + 1,
+                                   rip=rip)
+        generate_multinode_net(
+            out_dir=f'{params.configs_dir}/sidechain_testnet',
+            mainnet=mainnet,
+            sidenet=sidenet,
+            xchain_assets=xchain_assets)
+        index = index + 2
+
+        (Path(params.configs_dir) / 'logs').mkdir(parents=True, exist_ok=True)
+
+        for with_shards in [True, False]:
+            for is_main_net in [True, False]:
+                for cfg_type in ['dog', 'test', 'one', 'two']:
+                    if not is_main_net and cfg_type not in ['dog', 'test']:
+                        continue
+
+                    mainnet = Network(num_nodes=1,
+                                      num_validators=1,
+                                      start_cfg_index=index,
+                                      rip=rip)
+                    mainchain_cfg_file = generate_cfg_dir(
+                        data_dir=params.configs_dir,
+                        ports=mainnet.ports[0],
+                        with_shards=with_shards,
+                        main_net=is_main_net,
+                        cfg_type=cfg_type,
+                        sidechain_stanza='',
+                        sidechain_bootstrap_stanza='',
+                        validation_seed=mainnet.validator_keypairs[0].
+                        secret_key)
+
+                    sidenet = SidechainNetwork(num_nodes=1,
+                                               num_federators=5,
+                                               num_validators=1,
+                                               start_cfg_index=index + 1,
+                                               rip=rip)
+                    signing_key = sidenet.federator_keypairs[0].secret_key
+
+                    sidechain_stanza, sizechain_bootstrap_stanza = generate_sidechain_stanza(
+                        mainnet.ports[0], sidenet.main_account,
+                        sidenet.federator_keypairs, signing_key,
+                        mainchain_cfg_file, xchain_assets)
+
+                    generate_cfg_dir(
+                        data_dir=params.configs_dir,
+                        ports=sidenet.ports[0],
+                        with_shards=with_shards,
+                        main_net=is_main_net,
+                        cfg_type=cfg_type,
+                        sidechain_stanza=sidechain_stanza,
+                        sidechain_bootstrap_stanza=sizechain_bootstrap_stanza,
+                        validation_seed=sidenet.validator_keypairs[0].
+                        secret_key)
+                    index = index + 2
+
+
+if __name__ == '__main__':
+    params = Params()
+
+    xchain_assets = None
+    if params.usd:
+        xchain_assets = {}
+        xchain_assets['xrp_xrp_sidechain_asset'] = XChainAsset(
+            XRP(0), XRP(0), 1, 1, 200, 200)
+        root_account = Account(account_id="rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh")
+        main_iou_asset = Asset(value=0, currency='USD', issuer=root_account)
+        side_iou_asset = Asset(value=0, currency='USD', issuer=root_account)
+        xchain_assets['iou_iou_sidechain_asset'] = XChainAsset(
+            main_iou_asset, side_iou_asset, 1, 1, 0.02, 0.02)
+
+    main(params, xchain_assets)

bin/sidechain/python/log_analyzer.py

@@ -0,0 +1,198 @@
+#!/usr/bin/env python3
+
+import argparse
+import json
+import os
+import re
+import sys
+
+from common import eprint
+from typing import IO, Optional
+
+
+class LogLine:
+    UNSTRUCTURED_RE = re.compile(r'''(?x)
+                # The x flag enables insignificant whitespace mode (allowing comments)
+                ^(?P<timestamp>.*UTC)
+                [\ ]
+                (?P<module>[^:]*):(?P<level>[^\ ]*)
+                [\ ]
+                (?P<msg>.*$)
+                ''')
+
+    STRUCTURED_RE = re.compile(r'''(?x)
+                # The x flag enables insignificant whitespace mode (allowing comments)
+                ^(?P<timestamp>.*UTC)
+                [\ ]
+                (?P<module>[^:]*):(?P<level>[^\ ]*)
+                [\ ]
+                (?P<msg>[^{]*)
+                [\ ]
+                (?P<json_data>.*$)
+                ''')
+
+    def __init__(self, line: str):
+        self.raw_line = line
+        self.json_data = None
+
+        try:
+            if line.endswith('}'):
+                m = self.STRUCTURED_RE.match(line)
+                try:
+                    self.json_data = json.loads(m.group('json_data'))
+                except:
+                    m = self.UNSTRUCTURED_RE.match(line)
+            else:
+                m = self.UNSTRUCTURED_RE.match(line)
+
+            self.timestamp = m.group('timestamp')
+            self.level = m.group('level')
+            self.module = m.group('module')
+            self.msg = m.group('msg')
+        except Exception as e:
+            eprint(f'init exception: {e} line: {line}')
+
+    def to_mixed_json_str(self) -> str:
+        '''
+        return a pretty printed string as mixed json
+        '''
+        try:
+            r = f'{self.timestamp} {self.module}:{self.level} {self.msg}'
+            if self.json_data:
+                r += '\n' + json.dumps(self.json_data, indent=1)
+            return r
+        except:
+            eprint(f'Using raw line: {self.raw_line}')
+            return self.raw_line
+
+    def to_pure_json(self) -> dict:
+        '''
+        return a json dict
+        '''
+        dict = {}
+        dict['t'] = self.timestamp
+        dict['m'] = self.module
+        dict['l'] = self.level
+        dict['msg'] = self.msg
+        if self.json_data:
+            dict['data'] = self.json_data
+        return dict
+
+    def to_pure_json_str(self, f_id: Optional[str] = None) -> str:
+        '''
+        return a pretty printed string as pure json
+        '''
+        try:
+            dict = self.to_pure_json(f_id)
+            return json.dumps(dict, indent=1)
+        except:
+            return self.raw_line
+
+
+def convert_log(in_file_name: str,
+                out: str,
+                *,
+                as_list=False,
+                pure_json=False,
+                module: Optional[str] = 'SidechainFederator') -> list:
+    result = []
+    try:
+        prev_lines = None
+        with open(in_file_name) as input:
+            for l in input:
+                l = l.strip()
+                if not l:
+                    continue
+                if LogLine.UNSTRUCTURED_RE.match(l):
+                    if prev_lines:
+                        log_line = LogLine(prev_lines)
+                        if not module or log_line.module == module:
+                            if as_list:
+                                result.append(log_line.to_pure_json())
+                            else:
+                                if pure_json:
+                                    print(log_line.to_pure_json_str(),
+                                          file=out)
+                                else:
+                                    print(log_line.to_mixed_json_str(),
+                                          file=out)
+                    prev_lines = l
+                else:
+                    if not prev_lines:
+                        eprint(f'Error: Expected prev_lines. Cur line: {l}')
+                    assert prev_lines
+                    prev_lines += f' {l}'
+            if prev_lines:
+                log_line = LogLine(prev_lines)
+                if not module or log_line.module == module:
+                    if as_list:
+                        result.append(log_line.to_pure_json())
+                    else:
+                        if pure_json:
+                            print(log_line.to_pure_json_str(f_id),
+                                  file=out,
+                                  flush=True)
+                        else:
+                            print(log_line.to_mixed_json_str(),
+                                  file=out,
+                                  flush=True)
+    except Exception as e:
+        eprint(f'Excption: {e}')
+        raise e
+    return result
+
+
+def convert_all(in_dir_name: str, out: IO, *, pure_json=False):
+    '''
+    Convert all the "debug.log" log files in one directory level below the in_dir_name into a single json file.
+    There will be a field called 'f' for the director name that the origional log file came from.
+    This is useful when analyzing networks that run on the local machine.
+    '''
+    if not os.path.isdir(in_dir_name):
+        print(f'Error: {in_dir_name} is not a directory')
+    files = []
+    f_ids = []
+    for subdir in os.listdir(in_dir_name):
+        file = f'{in_dir_name}/{subdir}/debug.log'
+        if not os.path.isfile(file):
+            continue
+        files.append(file)
+        f_ids.append(subdir)
+
+    result = {}
+    for f, f_id in zip(files, f_ids):
+        l = convert_log(f, out, as_list=True, pure_json=pure_json, module=None)
+        result[f_id] = l
+    print(json.dumps(result, indent=1), file=out, flush=True)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description=('python script to convert log files to json'))
+
+    parser.add_argument(
+        '--input',
+        '-i',
+        help=('input log file or sidechain config directory structure'),
+    )
+
+    parser.add_argument(
+        '--output',
+        '-o',
+        help=('output log file'),
+    )
+
+    return parser.parse_known_args()[0]
+
+
+if __name__ == '__main__':
+    try:
+        args = parse_args()
+        with open(args.output, "w") as out:
+            if os.path.isdir(args.input):
+                convert_all(args.input, out, pure_json=True)
+            else:
+                convert_log(args.input, out, pure_json=True)
+    except Exception as e:
+        eprint(f'Excption: {e}')
+        raise e

bin/sidechain/python/log_report.py

@@ -0,0 +1,285 @@
+#!/usr/bin/env python3
+
+import argparse
+from collections import defaultdict
+import datetime
+import json
+import numpy as np
+import os
+import pandas as pd
+import string
+import sys
+from typing import Dict, Set
+
+from common import eprint
+import log_analyzer
+
+
+def _has_256bit_hex_field_other(data, result: Set[str]):
+    return
+
+
+_has_256bit_hex_field_overloads = defaultdict(
+    lambda: _has_256bit_hex_field_other)
+
+
+def _has_256bit_hex_field_str(data: str, result: Set[str]):
+    if len(data) != 64:
+        return
+    for c in data:
+        o = ord(c.upper())
+        if ord('A') <= o <= ord('F'):
+            continue
+        if ord('0') <= o <= ord('9'):
+            continue
+        return
+    result.add(data)
+
+
+_has_256bit_hex_field_overloads[str] = _has_256bit_hex_field_str
+
+
+def _has_256bit_hex_field_dict(data: dict, result: Set[str]):
+    for k, v in data.items():
+        if k in [
+                "meta", "index", "LedgerIndex", "ledger_index", "ledger_hash",
+                "SigningPubKey", "suppression"
+        ]:
+            continue
+        _has_256bit_hex_field_overloads[type(v)](v, result)
+
+
+_has_256bit_hex_field_overloads[dict] = _has_256bit_hex_field_dict
+
+
+def _has_256bit_hex_field_list(data: list, result: Set[str]):
+    for v in data:
+        _has_256bit_hex_field_overloads[type(v)](v, result)
+
+
+_has_256bit_hex_field_overloads[list] = _has_256bit_hex_field_list
+
+
+def has_256bit_hex_field(data: dict) -> Set[str]:
+    '''
+    Find all the fields that are strings 64 chars long with only hex digits
+    This is useful when grouping transactions by hex
+    '''
+    result = set()
+    _has_256bit_hex_field_dict(data, result)
+    return result
+
+
+def group_by_txn(data: dict) -> dict:
+    '''
+    return a dictionary where the key is the transaction hash, the value is another dictionary.
+    The second dictionary the key is the server id, and the values are a list of log items
+    '''
+    def _make_default():
+        return defaultdict(lambda: list())
+
+    result = defaultdict(_make_default)
+    for server_id, log_list in data.items():
+        for log_item in log_list:
+            if txn_hashes := has_256bit_hex_field(log_item):
+                for h in txn_hashes:
+                    result[h][server_id].append(log_item)
+    return result
+
+
+def _rekey_dict_by_txn_date(hash_to_timestamp: dict,
+                            grouped_by_txn: dict) -> dict:
+    '''
+    hash_to_timestamp is a dictionary with a key of the txn hash and a value of the timestamp.
+    grouped_by_txn is a dictionary with a key of the txn and an unspecified value.
+    the keys in hash_to_timestamp are a superset of the keys in grouped_by_txn
+    This function returns a new grouped_by_txn dictionary with the transactions sorted by date.
+    '''
+    known_txns = [
+        k for k, v in sorted(hash_to_timestamp.items(), key=lambda x: x[1])
+    ]
+    result = {}
+    for k, v in grouped_by_txn.items():
+        if k not in known_txns:
+            result[k] = v
+    for h in known_txns:
+        result[h] = grouped_by_txn[h]
+    return result
+
+
+def _to_timestamp(str_time: str) -> datetime.datetime:
+    return datetime.datetime.strptime(
+        str_time.split('.')[0], "%Y-%b-%d %H:%M:%S")
+
+
+class Report:
+    def __init__(self, in_dir, out_dir):
+        self.in_dir = in_dir
+        self.out_dir = out_dir
+
+        self.combined_logs_file_name = f'{self.out_dir}/combined_logs.json'
+        self.grouped_by_txn_file_name = f'{self.out_dir}/grouped_by_txn.json'
+        self.counts_by_txn_and_server_file_name = f'{self.out_dir}/counts_by_txn_and_server.org'
+        self.data = None  # combined logs
+
+        # grouped_by_txn is a dictionary where the key is the server id. mainchain servers
+        # have a key of `mainchain_#` and sidechain servers have a key of
+        # `sidechain_#`, where `#` is a number.
+        self.grouped_by_txn = None
+
+        if not os.path.isdir(in_dir):
+            eprint(f'The input {self.in_dir} must be an existing directory')
+            sys.exit(1)
+
+        if os.path.exists(self.out_dir):
+            if not os.path.isdir(self.out_dir):
+                eprint(
+                    f'The output: {self.out_dir} exists and is not a directory'
+                )
+                sys.exit(1)
+        else:
+            os.makedirs(self.out_dir)
+
+        self.combine_logs()
+        with open(self.combined_logs_file_name) as f:
+            self.data = json.load(f)
+        self.grouped_by_txn = group_by_txn(self.data)
+
+        # counts_by_txn_and_server is a dictionary where the key is the txn_hash
+        # and the value is a pandas df with a row for every server and a column for every message
+        # the value is a count of how many times that message appears for that server.
+        counts_by_txn_and_server = {}
+        # dict where the key is a transaction hash and the value is the transaction
+        hash_to_txn = {}
+        # dict where the key is a transaction hash and the value is earliest timestamp in a log file
+        hash_to_timestamp = {}
+        for txn_hash, server_dict in self.grouped_by_txn.items():
+            message_set = set()
+            # message list is ordered by when it appears in the log
+            message_list = []
+            for server_id, messages in server_dict.items():
+                for m in messages:
+                    try:
+                        d = m['data']
+                        if 'msg' in d and 'transaction' in d['msg']:
+                            t = d['msg']['transaction']
+                        elif 'tx_json' in d:
+                            t = d['tx_json']
+                        if t['hash'] == txn_hash:
+                            hash_to_txn[txn_hash] = t
+                    except:
+                        pass
+                    msg = m['msg']
+                    t = _to_timestamp(m['t'])
+                    if txn_hash not in hash_to_timestamp:
+                        hash_to_timestamp[txn_hash] = t
+                    elif hash_to_timestamp[txn_hash] > t:
+                        hash_to_timestamp[txn_hash] = t
+                    if msg not in message_set:
+                        message_set.add(msg)
+                        message_list.append(msg)
+            df = pd.DataFrame(0,
+                              index=server_dict.keys(),
+                              columns=message_list)
+            for server_id, messages in server_dict.items():
+                for m in messages:
+                    df[m['msg']][server_id] += 1
+            counts_by_txn_and_server[txn_hash] = df
+
+        # sort the transactions by timestamp, but the txns with unknown timestamp at the beginning
+        self.grouped_by_txn = _rekey_dict_by_txn_date(hash_to_timestamp,
+                                                      self.grouped_by_txn)
+        counts_by_txn_and_server = _rekey_dict_by_txn_date(
+            hash_to_timestamp, counts_by_txn_and_server)
+
+        with open(self.grouped_by_txn_file_name, 'w') as out:
+            print(json.dumps(self.grouped_by_txn, indent=1), file=out)
+
+        with open(self.counts_by_txn_and_server_file_name, 'w') as out:
+            for txn_hash, df in counts_by_txn_and_server.items():
+                print(f'\n\n* Txn: {txn_hash}', file=out)
+                if txn_hash in hash_to_txn:
+                    print(json.dumps(hash_to_txn[txn_hash], indent=1),
+                          file=out)
+                rename_dict = {}
+                for column, renamed_column in zip(df.columns.array,
+                                                  string.ascii_uppercase):
+                    print(f'{renamed_column} = {column}', file=out)
+                    rename_dict[column] = renamed_column
+                df.rename(columns=rename_dict, inplace=True)
+                print(f'\n{df}', file=out)
+
+    def combine_logs(self):
+        try:
+            with open(self.combined_logs_file_name, "w") as out:
+                log_analyzer.convert_all(args.input, out, pure_json=True)
+        except Exception as e:
+            eprint(f'Excption: {e}')
+            raise e
+
+
+def main(input_dir_name: str, output_dir_name: str):
+    r = Report(input_dir_name, output_dir_name)
+
+    # Values are a list of log lines formatted as json. There are five fields:
+    # `t` is the timestamp.
+    # `m` is the module.
+    # `l` is the log level.
+    # `msg` is the message.
+    # `data` is the data.
+    # For example:
+    #
+    # {
+    #  "t": "2021-Oct-08 21:33:41.731371562 UTC",
+    #  "m": "SidechainFederator",
+    #  "l": "TRC",
+    #  "msg": "no last xchain txn with result",
+    #  "data": {
+    #   "needsOtherChainLastXChainTxn": true,
+    #   "isMainchain": false,
+    #   "jlogId": 121
+    #  }
+    # },
+
+
+# Lifecycle of a transaction
+# For each federator record:
+# Transaction detected: amount, seq, destination, chain, hash
+# Signature received: hash, seq
+# Signature sent: hash, seq, federator dst
+# Transaction submitted
+# Result received, and detect if error
+# Detect any field that doesn't match
+
+# Lifecycle of initialization
+
+# Chain listener messages
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description=(
+        'python script to generate a log report from a sidechain config directory structure containing the logs'
+    ))
+
+    parser.add_argument(
+        '--input',
+        '-i',
+        help=('directory with sidechain config directory structure'),
+    )
+
+    parser.add_argument(
+        '--output',
+        '-o',
+        help=('output directory for report files'),
+    )
+
+    return parser.parse_known_args()[0]
+
+
+if __name__ == '__main__':
+    try:
+        args = parse_args()
+        main(args.input, args.output)
+    except Exception as e:
+        eprint(f'Excption: {e}')
+        raise e

bin/sidechain/python/requirements.txt

@@ -0,0 +1,14 @@
+attrs==21.2.0
+iniconfig==1.1.1
+numpy==1.21.2
+packaging==21.0
+pandas==1.3.3
+pluggy==1.0.0
+py==1.10.0
+pyparsing==2.4.7
+pytest==6.2.5
+python-dateutil==2.8.2
+pytz==2021.1
+six==1.16.0
+toml==0.10.2
+websockets==8.1

bin/sidechain/python/riplrepl.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+'''
+Script to run an interactive shell to test sidechains.
+'''
+
+import sys
+
+from common import disable_eprint, eprint
+import interactive
+import sidechain
+
+
+def main():
+    params = sidechain.Params()
+    params.interactive = True
+
+    interactive.set_hooks_dir(params.hooks_dir)
+
+    if err_str := params.check_error():
+        eprint(err_str)
+        sys.exit(1)
+
+    if params.verbose:
+        print("eprint enabled")
+    else:
+        disable_eprint()
+
+    if params.standalone:
+        sidechain.standalone_interactive_repl(params)
+    else:
+        sidechain.multinode_interactive_repl(params)
+
+
+if __name__ == '__main__':
+    main()