Improve sweephandler test flakiness (#499)

Fixes #498

.githooks/ensure_release_tag

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Pushing a release branch requires an annotated tag at the released commit
+branch=$(git rev-parse --abbrev-ref HEAD)
+
+if [[ $branch =~ master ]]; then
+  # check if HEAD commit is tagged
+  if ! git describe --exact-match HEAD; then
+    echo "Commits to master must be tagged"
+    exit 1
+  fi
+elif [[ $branch =~ release/* ]]; then
+  IFS=/ read -r branch rel_ver <<< ${branch}
+  tag=$(git describe --tags --abbrev=0)
+  if [[ "${rel_ver}" != "${tag}" ]]; then
+    echo "release/${rel_ver} branches must have annotated tag ${rel_ver}"
+    echo "git tag -am\"${rel_ver}\" ${rel_ver}"
+    exit 1
+  fi
+fi

.githooks/pre-commit

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+exec 1>&2
+
+# paths to check and re-format
+sources="src unittests"
+formatter="clang-format -i"
+
+first=$(git diff $sources)
+find $sources -type f \( -name '*.cpp' -o -name '*.h' -o -name '*.ipp' \) -print0 | xargs -0 $formatter
+second=$(git diff $sources)
+changes=$(diff <(echo "$first") <(echo "$second") | wc -l | sed -e 's/^[[:space:]]*//')
+
+if [ "$changes" != "0" ]; then
+    cat <<\EOF
+
+                                   WARNING
+-----------------------------------------------------------------------------
+  Automatically re-formatted code with `clang-format` - commit was aborted.
+  Please manually add any updated files and commit again.
+-----------------------------------------------------------------------------
+
+EOF
+    exit 1
+fi
+
+.githooks/ensure_release_tag

.github/actions/sign/action.yml

@@ -1,21 +0,0 @@
-name: 'Sign packages'
-runs:
-  using: "composite"
-
-  steps:
-    - name: Sign
-      shell: bash
-      run: |
-        set -ex -o pipefail
-        echo "$GPG_KEY_B64"| base64 -d | gpg --batch --no-tty --allow-secret-key-import --import -
-        unset GPG_KEY_B64
-        export GPG_PASSPHRASE=$(echo $GPG_KEY_PASS_B64 | base64 -di)
-        unset GPG_KEY_PASS_B64
-        export GPG_KEYID=$(gpg --with-colon --list-secret-keys | head -n1 | cut -d : -f 5)
-        for PKG in $(ls *.deb); do
-          dpkg-sig \
-            -g "--no-tty --digest-algo 'sha512' --passphrase '${GPG_PASSPHRASE}' --pinentry-mode=loopback" \
-                  -k "${GPG_KEYID}" \
-                  --sign builder \
-                  $PKG
-        done

.github/workflows/build.yml

@@ -1,9 +1,9 @@
 name: Build Clio
 on:
   push:
-    branches:  [master, release, develop, develop-next]
+    branches: [master, release/*, develop, develop-next]
   pull_request:
-    branches:  [master, release, develop, develop-next]
+    branches: [master, release/*, develop, develop-next]
   workflow_dispatch:
 
 jobs:
@@ -29,19 +29,22 @@ jobs:
           - suffix: rpm
             image: rippleci/clio-rpm-builder:2022-09-17
             script: rpm
+
     container:
       image: ${{ matrix.type.image }}
-
+      # options: --user 1001
     steps:
       - uses: actions/checkout@v3
         with:
           path: clio
+          fetch-depth: 0
 
       - name: Clone Clio packaging repo
         uses: actions/checkout@v3
         with:
           path: clio-packages
           repository: XRPLF/clio-packages
+          ref: main
 
       - name: Build
         shell: bash
@@ -54,7 +57,6 @@ jobs:
           cmake --build clio-packages/build --parallel $(nproc)
           cp ./clio-packages/build/clio-prefix/src/clio-build/clio_tests .
           mv ./clio-packages/build/*.${{ matrix.type.suffix }} .
-
       - name: Artifact packages
         uses: actions/upload-artifact@v3
         with:
@@ -67,59 +69,48 @@ jobs:
           name: clio_tests-${{ matrix.type.suffix }}
           path: ${{ github.workspace }}/clio_tests
 
-  sign:
-    name: Sign packages
-    needs: build_clio
-    runs-on: ubuntu-20.04
-    if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/release' || github.ref == 'refs/heads/develop'
-    env:
-      GPG_KEY_B64: ${{ secrets.GPG_KEY_B64 }}
-      GPG_KEY_PASS_B64: ${{ secrets.GPG_KEY_PASS_B64 }}
-    strategy:
-      fail-fast: false
-      matrix:
-        type:
-          - suffix: deb
-            image: ubuntu:20.04
-            script: dpkg
-          # - suffix: rpm
-          #   image: centos:7
-          #   script: rpm
-    container:
-      image: ${{ matrix.type.image }}
+  build_dev:
+    name: Build on Mac/Clang14 and run tests
+    needs: lint
+    continue-on-error: false
+    runs-on: macos-12
+
     steps:
       - uses: actions/checkout@v3
-      - name: Install dpkg-sig
-        run: |
-          apt-get update && apt-get install -y dpkg-sig gnupg
-      - name: Get package artifact
-        uses: actions/download-artifact@v3
         with:
-          name: clio_${{ matrix.type.suffix }}_packages
+          path: clio
 
-      - name: find packages
-        run: find . -name "*.${{ matrix.type.suffix }}"
-
-      - name: Sign packages
-        uses: ./.github/actions/sign
-
-
-      - name: Verify the signature
-        run: |
-          set -e
-          for PKG in $(ls *.deb); do
-            gpg --verify "${PKG}"
-          done
-
-      - name: Get short SHA
-        id: shortsha
-        run: echo "::set-output name=sha8::$(echo ${GITHUB_SHA} | cut -c1-8)"
-
-      - name: Artifact signed packages
-        uses: actions/upload-artifact@v2
+      - name: Check Boost cache
+        id: boost
+        uses: actions/cache@v3
         with:
-          name: signed-clio-deb-packages-${{ steps.shortsha.outputs.sha8 }}
-          path: ${{ github.workspace }}/*.deb
+          path: boost
+          key: ${{ runner.os }}-boost
+
+      - name: Build boost
+        if: steps.boost.outputs.cache-hit != 'true'
+        run: |
+          curl -s -OJL "https://boostorg.jfrog.io/artifactory/main/release/1.77.0/source/boost_1_77_0.tar.gz"
+          tar zxf boost_1_77_0.tar.gz
+          mv boost_1_77_0 boost
+          cd boost
+          ./bootstrap.sh
+          ./b2 cxxflags="-std=c++14"
+      - name: install deps
+        run: |
+          brew install pkg-config protobuf openssl ninja cassandra-cpp-driver bison
+      - name: Build clio
+        run: |
+          export BOOST_ROOT=$(pwd)/boost
+          cd clio
+          cmake -B build
+          if ! cmake --build build -j$(nproc); then
+            echo '# 🔥🔥 MacOS AppleClang build failed!💥' >> $GITHUB_STEP_SUMMARY
+          fi
+      - name: Run Test
+        run: |
+          cd clio/build
+          ./clio_tests --gtest_filter="-Backend*"
 
   test_clio:
     name: Test Clio
@@ -140,3 +131,65 @@ jobs:
       - name: Run tests
         timeout-minutes: 10
         uses: ./.github/actions/test
+
+  code_coverage:
+    name: Build on Linux and code coverage
+    needs: lint
+    continue-on-error: false
+    runs-on: ubuntu-22.04
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          path: clio
+
+      - name: Check Boost cache
+        id: boost
+        uses: actions/cache@v3
+        with:
+          path: boost
+          key: ${{ runner.os }}-boost
+
+      - name: Build boost
+        if: steps.boost.outputs.cache-hit != 'true'
+        run: |
+          curl -s -OJL "https://boostorg.jfrog.io/artifactory/main/release/1.77.0/source/boost_1_77_0.tar.gz"
+          tar zxf boost_1_77_0.tar.gz
+          mv boost_1_77_0 boost
+          cd boost
+          ./bootstrap.sh
+          ./b2
+
+      - name: install deps
+        run: |
+          sudo apt-get -y install git pkg-config protobuf-compiler libprotobuf-dev libssl-dev wget build-essential doxygen bison flex autoconf clang-format gcovr
+
+      - name: Build clio
+        run: |
+          export BOOST_ROOT=$(pwd)/boost
+          cd clio
+          cmake -B build -DCODE_COVERAGE=on -DTEST_PARAMETER='--gtest_filter="-Backend*"'
+          if ! cmake --build build -j$(nproc); then
+            echo '# 🔥Ubuntu build🔥 failed!💥' >> $GITHUB_STEP_SUMMARY
+          fi
+          cd build
+          make clio_tests-ccov
+
+      - name: Code Coverage Summary Report
+        uses: irongut/CodeCoverageSummary@v1.2.0
+        with:
+          filename: clio/build/clio_tests-gcc-cov/out.xml
+          badge: true
+          output: both
+          format: markdown
+
+      - name: Save PR number and ccov report
+        run: |
+          mkdir -p ./UnitTestCoverage
+          echo ${{ github.event.number }} > ./UnitTestCoverage/NR
+          cp clio/build/clio_tests-gcc-cov/report.html ./UnitTestCoverage/report.html
+          cp code-coverage-results.md ./UnitTestCoverage/out.md
+      - uses: actions/upload-artifact@v2
+        with:
+          name: UnitTestCoverage
+          path: UnitTestCoverage/

.gitignore

@@ -1,4 +1,6 @@
 *clio*.log
-build/
+build*/
 .vscode
 .python-version
+config.json
+src/main/impl/Build.cpp

CMake/Build.cpp.in

@@ -0,0 +1,39 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include <main/Build.h>
+
+namespace Build {
+static constexpr char versionString[] = "@VERSION@";
+
+std::string const&
+getClioVersionString()
+{
+    static std::string const value = versionString;
+    return value;
+}
+
+std::string const&
+getClioFullVersionString()
+{
+    static std::string const value = "clio-" + getClioVersionString();
+    return value;
+}
+
+}  // namespace Build

CMake/ClioVersion.cmake

@@ -1,15 +1,33 @@
 #[===================================================================[
-   read version from source
+   write version to source
 #]===================================================================]
 
-file (STRINGS src/main/impl/Build.cpp BUILD_INFO)
-foreach (line_ ${BUILD_INFO})
-  if (line_ MATCHES "versionString[ ]*=[ ]*\"(.+)\"")
-    set (clio_version ${CMAKE_MATCH_1})
-  endif ()
-endforeach ()
-if (clio_version)
-  message (STATUS "clio version: ${clio_version}")
-else ()
-  message (FATAL_ERROR "unable to determine clio version")
-endif ()
+find_package(Git REQUIRED)
+
+set(GIT_COMMAND rev-parse --short HEAD)
+execute_process(COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND} OUTPUT_VARIABLE REV OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+set(GIT_COMMAND branch --show-current)
+execute_process(COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND} OUTPUT_VARIABLE BRANCH OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+if(BRANCH STREQUAL "")
+  set(BRANCH "dev")
+endif()
+
+if(NOT (BRANCH MATCHES master OR BRANCH MATCHES release/*)) # for develop and any other branch name YYYYMMDDHMS-<branch>-<git-ref>
+  execute_process(COMMAND date +%Y%m%d%H%M%S OUTPUT_VARIABLE DATE OUTPUT_STRIP_TRAILING_WHITESPACE)
+  set(VERSION "${DATE}-${BRANCH}-${REV}")
+else()
+  set(GIT_COMMAND describe --tags)
+  execute_process(COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND} OUTPUT_VARIABLE TAG_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE)
+  set(VERSION "${TAG_VERSION}-${REV}")
+endif()
+
+if(CMAKE_BUILD_TYPE MATCHES Debug)
+  set(VERSION "${VERSION}+DEBUG")
+endif()
+
+message(STATUS "Build version: ${VERSION}")
+set(clio_version "${VERSION}")
+
+configure_file(CMake/Build.cpp.in ${CMAKE_SOURCE_DIR}/src/main/impl/Build.cpp)

CMake/coverage.cmake

@@ -0,0 +1,126 @@
+# call add_converage(module_name) to add coverage targets for the given module
+function(add_converage module)
+  if("${CMAKE_C_COMPILER_ID}" MATCHES "(Apple)?[Cc]lang"
+     OR "${CMAKE_CXX_COMPILER_ID}" MATCHES "(Apple)?[Cc]lang")
+    message("[Coverage] Building with llvm Code Coverage Tools")
+    # Using llvm gcov ; llvm install by xcode
+    set(LLVM_COV_PATH /Library/Developer/CommandLineTools/usr/bin)
+    if(NOT EXISTS ${LLVM_COV_PATH}/llvm-cov)
+      message(FATAL_ERROR "llvm-cov not found! Aborting.")
+    endif()
+
+    # set Flags
+    target_compile_options(${module} PRIVATE -fprofile-instr-generate
+                                             -fcoverage-mapping)
+    target_link_options(${module} PUBLIC -fprofile-instr-generate
+                        -fcoverage-mapping)
+
+    target_compile_options(clio PRIVATE -fprofile-instr-generate
+                                        -fcoverage-mapping)
+    target_link_options(clio PUBLIC -fprofile-instr-generate
+                        -fcoverage-mapping)
+
+    # llvm-cov
+    add_custom_target(
+      ${module}-ccov-preprocessing
+      COMMAND LLVM_PROFILE_FILE=${module}.profraw $<TARGET_FILE:${module}>
+      COMMAND ${LLVM_COV_PATH}/llvm-profdata merge -sparse ${module}.profraw -o
+              ${module}.profdata
+      DEPENDS ${module})
+
+    add_custom_target(
+      ${module}-ccov-show
+      COMMAND ${LLVM_COV_PATH}/llvm-cov show $<TARGET_FILE:${module}>
+              -instr-profile=${module}.profdata -show-line-counts-or-regions
+      DEPENDS ${module}-ccov-preprocessing)
+
+    # add summary for CI parse
+    add_custom_target(
+      ${module}-ccov-report
+      COMMAND
+        ${LLVM_COV_PATH}/llvm-cov report $<TARGET_FILE:${module}>
+        -instr-profile=${module}.profdata
+        -ignore-filename-regex=".*_makefiles|.*unittests"
+        -show-region-summary=false
+      DEPENDS ${module}-ccov-preprocessing)
+
+    # exclude libs and unittests self
+    add_custom_target(
+      ${module}-ccov
+      COMMAND
+        ${LLVM_COV_PATH}/llvm-cov show $<TARGET_FILE:${module}>
+        -instr-profile=${module}.profdata -show-line-counts-or-regions
+        -output-dir=${module}-llvm-cov -format="html"
+        -ignore-filename-regex=".*_makefiles|.*unittests" > /dev/null 2>&1
+      DEPENDS ${module}-ccov-preprocessing)
+
+    add_custom_command(
+      TARGET ${module}-ccov
+      POST_BUILD
+      COMMENT
+        "Open ${module}-llvm-cov/index.html in your browser to view the coverage report."
+    )
+  elseif("${CMAKE_C_COMPILER_ID}" MATCHES "GNU" OR "${CMAKE_CXX_COMPILER_ID}"
+                                                   MATCHES "GNU")
+    message("[Coverage] Building with Gcc Code Coverage Tools")
+
+    find_program(GCOV_PATH gcov)
+    if(NOT GCOV_PATH)
+      message(FATAL_ERROR "gcov not found! Aborting...")
+    endif() # NOT GCOV_PATH
+    find_program(GCOVR_PATH gcovr)
+    if(NOT GCOVR_PATH)
+      message(FATAL_ERROR "gcovr not found! Aborting...")
+    endif() # NOT GCOVR_PATH
+
+    set(COV_OUTPUT_PATH ${module}-gcc-cov)
+    target_compile_options(${module} PRIVATE -fprofile-arcs -ftest-coverage
+                                             -fPIC)
+    target_link_libraries(${module} PRIVATE gcov)
+
+    target_compile_options(clio PRIVATE -fprofile-arcs -ftest-coverage
+                                             -fPIC)
+    target_link_libraries(clio PRIVATE gcov)
+    # this target is used for CI as well generate the summary out.xml will send
+    # to github action to generate markdown, we can paste it to comments or
+    # readme
+    add_custom_target(
+      ${module}-ccov
+      COMMAND ${module} ${TEST_PARAMETER}
+      COMMAND rm -rf ${COV_OUTPUT_PATH}
+      COMMAND mkdir ${COV_OUTPUT_PATH}
+      COMMAND
+        gcovr -r ${CMAKE_SOURCE_DIR} --object-directory=${PROJECT_BINARY_DIR} -x
+        ${COV_OUTPUT_PATH}/out.xml --exclude='${CMAKE_SOURCE_DIR}/unittests/'
+        --exclude='${PROJECT_BINARY_DIR}/'
+      COMMAND
+        gcovr -r ${CMAKE_SOURCE_DIR} --object-directory=${PROJECT_BINARY_DIR}
+        --html ${COV_OUTPUT_PATH}/report.html
+        --exclude='${CMAKE_SOURCE_DIR}/unittests/'
+        --exclude='${PROJECT_BINARY_DIR}/'
+      WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
+      COMMENT "Running gcovr to produce Cobertura code coverage report.")
+
+    # generate the detail report
+    add_custom_target(
+      ${module}-ccov-report
+      COMMAND ${module} ${TEST_PARAMETER}
+      COMMAND rm -rf ${COV_OUTPUT_PATH}
+      COMMAND mkdir ${COV_OUTPUT_PATH}
+      COMMAND
+        gcovr -r ${CMAKE_SOURCE_DIR} --object-directory=${PROJECT_BINARY_DIR}
+        --html-details ${COV_OUTPUT_PATH}/index.html
+        --exclude='${CMAKE_SOURCE_DIR}/unittests/'
+        --exclude='${PROJECT_BINARY_DIR}/'
+      WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
+      COMMENT "Running gcovr to produce Cobertura code coverage report.")
+    add_custom_command(
+      TARGET ${module}-ccov-report
+      POST_BUILD
+      COMMENT
+        "Open ${COV_OUTPUT_PATH}/index.html in your browser to view the coverage report."
+    )
+  else()
+    message(FATAL_ERROR "Complier not support yet")
+  endif()
+endfunction()

CMake/deps/Postgres.cmake

@@ -1,31 +0,0 @@
-set(POSTGRES_INSTALL_DIR ${CMAKE_BINARY_DIR}/postgres)
-set(POSTGRES_LIBS pq pgcommon pgport)
-ExternalProject_Add(postgres
-    GIT_REPOSITORY https://github.com/postgres/postgres.git
-    GIT_TAG REL_14_1
-    GIT_SHALLOW 1
-    LOG_CONFIGURE 1
-    LOG_BUILD 1
-    CONFIGURE_COMMAND ./configure --prefix ${POSTGRES_INSTALL_DIR} --without-readline --verbose
-    BUILD_COMMAND ${CMAKE_COMMAND} -E env --unset=MAKELEVEL make VERBOSE=${CMAKE_VERBOSE_MAKEFILE} -j32
-    BUILD_IN_SOURCE 1
-    INSTALL_COMMAND  ${CMAKE_COMMAND} -E env make -s --no-print-directory install
-    UPDATE_COMMAND ""
-    BUILD_BYPRODUCTS
-        ${POSTGRES_INSTALL_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}pq${CMAKE_STATIC_LIBRARY_SUFFIX}}
-        ${POSTGRES_INSTALL_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}pgcommon${CMAKE_STATIC_LIBRARY_SUFFIX}}
-        ${POSTGRES_INSTALL_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}pgport${CMAKE_STATIC_LIBRARY_SUFFIX}}
-        )
-ExternalProject_Get_Property (postgres BINARY_DIR)
-
-foreach(_lib ${POSTGRES_LIBS})
-    add_library(${_lib} STATIC IMPORTED GLOBAL)
-    add_dependencies(${_lib} postgres)
-    set_target_properties(${_lib} PROPERTIES
-        IMPORTED_LOCATION ${POSTGRES_INSTALL_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${_lib}.a)
-    set_target_properties(${_lib} PROPERTIES
-        INTERFACE_INCLUDE_DIRECTORIES ${POSTGRES_INSTALL_DIR}/include)
-    target_link_libraries(clio PUBLIC ${POSTGRES_INSTALL_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${_lib}${CMAKE_STATIC_LIBRARY_SUFFIX})
-endforeach()
-add_dependencies(clio postgres)
-target_include_directories(clio PUBLIC ${POSTGRES_INSTALL_DIR}/include)

CMake/deps/SourceLocation.cmake

@@ -0,0 +1,11 @@
+include(CheckIncludeFileCXX)
+
+check_include_file_cxx("source_location" SOURCE_LOCATION_AVAILABLE)
+if(SOURCE_LOCATION_AVAILABLE) 
+    target_compile_definitions(clio PUBLIC "HAS_SOURCE_LOCATION")
+endif()
+
+check_include_file_cxx("experimental/source_location" EXPERIMENTAL_SOURCE_LOCATION_AVAILABLE)
+if(EXPERIMENTAL_SOURCE_LOCATION_AVAILABLE)     
+    target_compile_definitions(clio PUBLIC "HAS_EXPERIMENTAL_SOURCE_LOCATION")
+endif()

CMake/deps/gtest.cmake

@@ -10,7 +10,8 @@ if(NOT googletest_POPULATED)
   add_subdirectory(${googletest_SOURCE_DIR} ${googletest_BINARY_DIR} EXCLUDE_FROM_ALL)
 endif()
 
-target_link_libraries(clio_tests PUBLIC clio gtest_main)
+target_link_libraries(clio_tests PUBLIC clio gmock_main)
+target_include_directories(clio_tests PRIVATE unittests)
 
 enable_testing()
 

CMake/install/clio.service.in

@@ -11,6 +11,7 @@ ExecStart=@CLIO_INSTALL_DIR@/bin/clio_server @CLIO_INSTALL_DIR@/etc/config.json
 Restart=on-failure
 User=clio
 Group=clio
+LimitNOFILE=65536
 
 [Install]
-WantedBy=multi-user.target
+WantedBy=multi-user.target

CMake/settings.cmake

@@ -1 +1,6 @@
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-narrowing -Wall -Werror -Wno-dangling-else")
+target_compile_options(clio  
+  PUBLIC -Wall
+         -Werror 
+         -Wno-narrowing
+         -Wno-deprecated-declarations 
+         -Wno-dangling-else)

CMakeLists.txt

@@ -14,21 +14,10 @@ if(VERBOSE)
   set(FETCHCONTENT_QUIET FALSE CACHE STRING "Verbose FetchContent()")
 endif()
 
-if(NOT GIT_COMMIT_HASH)
-  if(VERBOSE)
-    message("GIT_COMMIT_HASH not provided...looking for git")
-  endif()
-  find_package(Git)
-  if(Git_FOUND)
-      execute_process(COMMAND ${GIT_EXECUTABLE} describe --always --abbrev=8
-          OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE gch)
-      if(gch)
-          set(GIT_COMMIT_HASH "${gch}")
-          message(STATUS "Git commit: ${GIT_COMMIT_HASH}")
-          add_definitions(-DCLIO_GIT_COMMIT_HASH="${GIT_COMMIT_HASH}")
-      endif()
-  endif()
-endif() #git
+
+if(PACKAGING)
+  add_definitions(-DPKG=1)
+endif()
 
 add_library(clio)
 target_compile_features(clio PUBLIC cxx_std_20)
@@ -41,7 +30,7 @@ include(CMake/ClioVersion.cmake)
 include(CMake/deps/rippled.cmake)
 include(CMake/deps/Boost.cmake)
 include(CMake/deps/cassandra.cmake)
-include(CMake/deps/Postgres.cmake)
+include(CMake/deps/SourceLocation.cmake)
 
 target_sources(clio PRIVATE
   ## Main
@@ -49,8 +38,6 @@ target_sources(clio PRIVATE
   ## Backend
   src/backend/BackendInterface.cpp
   src/backend/CassandraBackend.cpp
-  src/backend/Pg.cpp
-  src/backend/PostgresBackend.cpp
   src/backend/SimpleCache.cpp
   ## ETL
   src/etl/ETLSource.cpp
@@ -60,10 +47,14 @@ target_sources(clio PRIVATE
   ## Subscriptions
   src/subscriptions/SubscriptionManager.cpp
   ## RPC
+  src/rpc/Errors.cpp
   src/rpc/RPC.cpp
   src/rpc/RPCHelpers.cpp
   src/rpc/Counters.cpp
   src/rpc/WorkQueue.cpp
+  ## NextGen RPC
+  src/rpc/common/Specs.cpp
+  src/rpc/common/Validators.cpp
   ## RPC Methods
   # Account
   src/rpc/handlers/AccountChannels.cpp
@@ -75,7 +66,9 @@ target_sources(clio PRIVATE
   src/rpc/handlers/GatewayBalances.cpp
   src/rpc/handlers/NoRippleCheck.cpp
   # NFT
+  src/rpc/handlers/NFTHistory.cpp
   src/rpc/handlers/NFTInfo.cpp
+  src/rpc/handlers/NFTOffers.cpp
   # Ledger
   src/rpc/handlers/Ledger.cpp
   src/rpc/handlers/LedgerData.cpp
@@ -88,8 +81,6 @@ target_sources(clio PRIVATE
   # Dex
   src/rpc/handlers/BookChanges.cpp
   src/rpc/handlers/BookOffers.cpp
-  # NFT
-  src/rpc/handlers/NFTOffers.cpp
   # Payment Channel
   src/rpc/handlers/ChannelAuthorize.cpp
   src/rpc/handlers/ChannelVerify.cpp
@@ -97,15 +88,35 @@ target_sources(clio PRIVATE
   src/rpc/handlers/Subscribe.cpp
   # Server
   src/rpc/handlers/ServerInfo.cpp
-  # Utility
-  src/rpc/handlers/Random.cpp)
+  # Utilities
+  src/rpc/handlers/Random.cpp
+  src/config/Config.cpp
+  src/log/Logger.cpp
+  src/util/Taggable.cpp)
 
 add_executable(clio_server src/main/main.cpp)
 target_link_libraries(clio_server PUBLIC clio)
 
 if(BUILD_TESTS)
-  add_executable(clio_tests unittests/main.cpp)
+  set(TEST_TARGET clio_tests)
+  add_executable(${TEST_TARGET}
+    unittests/Playground.cpp
+    unittests/Backend.cpp
+    unittests/Logger.cpp
+    unittests/Config.cpp
+    unittests/ProfilerTest.cpp
+    unittests/DOSGuard.cpp
+    unittests/rpc/ErrorTests.cpp
+    unittests/rpc/BaseTests.cpp
+    unittests/rpc/handlers/TestHandlerTests.cpp
+    unittests/rpc/handlers/DefaultProcessorTests.cpp)
   include(CMake/deps/gtest.cmake)
+
+  # if CODE_COVERAGE enable, add clio_test-ccov
+  if(CODE_COVERAGE)
+    include(CMake/coverage.cmake)
+    add_converage(${TEST_TARGET})
+  endif()
 endif()
 
 include(CMake/install/install.cmake)

CONTRIBUTING.md

@@ -13,6 +13,13 @@ To contribute, please:
 
 > **Note:** Please make sure you read the [Style guide](#style-guide).
 
+## Install git hooks
+Please make sure to run the following command in order to use git hooks that are helpful for `clio` development.
+
+``` bash
+git config --local core.hooksPath .githooks
+```
+
 ## Git commands
 This sections offers a detailed look at the git commands you will need to use to get your PR submitted. 
 Please note that there are more than one way to do this and these commands are only  provided for your convenience.
@@ -117,7 +124,11 @@ Existing maintainers can resign, or be subject to a vote for removal at the behe
 ## Existing Maintainers
 
 * [cjcobb23](https://github.com/cjcobb23) (Ripple)
-* [natenichols](https://github.com/natenichols) (Ripple)
 * [legleux](https://github.com/legleux) (Ripple)
 * [undertome](https://github.com/undertome) (Ripple)
 * [godexsoft](https://github.com/godexsoft) (Ripple)
+* [officialfrancismendoza](https://github.com/officialfrancismendoza) (Ripple)
+
+## Honorable ex-Maintainers
+
+* [natenichols](https://github.com/natenichols) (ex-Ripple)

Doxyfile

@@ -0,0 +1,3 @@
+PROJECT_NAME           = "Clio"
+INPUT                  = src
+RECURSIVE              = YES

README.md

@@ -22,13 +22,15 @@ from which data can be extracted. The rippled node does not need to be running o
 
 ## Building
 
-Clio is built with CMake. Clio requires at least GCC-11 (C++20), and Boost 1.75.0 or later.
+Clio is built with CMake. Clio requires at least GCC-11/clang-14.0.0 (C++20), and Boost 1.75.0.
 
 Use these instructions to build a Clio executable from the source. These instructions were tested on Ubuntu 20.04 LTS.
 
 ```sh
 # Install dependencies
-  sudo apt-get -y install git pkg-config protobuf-compiler libprotobuf-dev libssl-dev wget build-essential bison flex autoconf cmake
+  sudo apt-get -y install git pkg-config protobuf-compiler libprotobuf-dev libssl-dev wget build-essential bison flex autoconf cmake clang-format
+# Install gcovr to run code coverage
+  sudo apt-get -y install gcovr
 
 # Compile Boost
   wget -O $HOME/boost_1_75_0.tar.gz https://boostorg.jfrog.io/artifactory/main/release/1.75.0/source/boost_1_75_0.tar.gz
@@ -72,6 +74,36 @@ server is running
 to the IP of your Clio server. This entry can take the form of a comma-separated list if
 you are running multiple Clio nodes.
 
+
+In addition, the parameter `start_sequence` can be included and configured within the top level of the config file. This parameter specifies the sequence of first ledger to extract if the database is empty. Note that ETL extracts ledgers in order and that no backfilling functionality currently exists, meaning Clio will not retroactively learn ledgers older than the one you specify. Choosing to specify this or not will yield the following behavior:
+- If this setting is absent and the database is empty, ETL will start with the next ledger validated by the network. 
+- If this setting is present and the database is not empty, an exception is thrown.
+
+In addition, the optional parameter `finish_sequence` can be added to the json file as well, specifying where the ledger can stop.
+
+To add `start_sequence` and/or `finish_sequence` to the config.json file appropriately, they will be on the same top level of precedence as other parameters (such as `database`, `etl_sources`, `read_only`, etc.) and be specified with an integer. Here is an example snippet from the config file:
+
+```json
+"start_sequence": 12345,
+"finish_sequence": 54321
+```
+
+The parameters `ssl_cert_file` and `ssl_key_file` can also be added to the top level of precedence of our Clio config. `ssl_cert_file` specifies the filepath for your SSL cert while `ssl_key_file` specifies the filepath for your SSL key. It is up to you how to change ownership of these folders for your designated Clio user. Your options include:
+- Copying the two files as root somewhere that's accessible by the Clio user, then running `sudo chown` to your user
+- Changing the permissions directly so it's readable by your Clio user
+- Running Clio as root (strongly discouraged)
+
+An example of how to specify `ssl_cert_file` and `ssl_key_file` in the config:
+
+```json
+"server":{
+    "ip": "0.0.0.0",
+    "port": 51233
+},
+"ssl_cert_file" : "/full/path/to/cert.file",
+"ssl_key_file" : "/full/path/to/key.file"
+```
+
 Once your config files are ready, start rippled and Clio. It doesn't matter which you
 start first, and it's fine to stop one or the other and restart at any given time.
 
@@ -152,9 +184,33 @@ You must:
 ## Logging
 Clio provides several logging options, all are configurable via the config file and are detailed below.
 
-`log_level`: The minimum level of severity at which the log message will be outputted.
+`log_level`: The minimum level of severity at which the log message will be outputted by default.
 Severity options are `trace`, `debug`, `info`, `warning`, `error`, `fatal`. Defaults to `info`.
 
+`log_format`: The format of log lines produced by clio. Defaults to `"%TimeStamp% (%SourceLocation%) [%ThreadID%] %Channel%:%Severity% %Message%"`.
+Each of the variables expands like so
+- `TimeStamp`: The full date and time of the log entry
+- `SourceLocation`: A partial path to the c++ file and the line number in said file (`source/file/path:linenumber`)  
+- `ThreadID`: The ID of the thread the log entry is written from
+- `Channel`: The channel that this log entry was sent to
+- `Severity`: The severity (aka log level) the entry was sent at
+- `Message`: The actual log message
+
+`log_channels`: An array of json objects, each overriding properties for a logging `channel`. 
+At the moment of writing, only `log_level` can be overriden using this mechanism.
+
+Each object is of this format:
+```json
+{
+    "channel": "Backend",
+    "log_level": "fatal"
+}
+``` 
+If no override is present for a given channel, that channel will log at the severity specified by the global `log_level`.
+Overridable log channels: `Backend`, `WebServer`, `Subscriptions`, `RPC`, `ETL` and `Performance`.
+
+> **Note:** See `example-config.json` for more details. 
+
 `log_to_console`: Enable/disable log output to console. Options are `true`/`false`. Defaults to true.
 
 `log_directory`: Path to the directory where log files are stored. If such directory doesn't exist, Clio will create it. If not specified, logs are not written to a file.
@@ -170,6 +226,11 @@ rotate the current log file. Defaults to 12 hours.
 Note, time-based log rotation occurs dependently on size-based log rotation, where if a
 size-based log rotation occurs, the timer for the time-based rotation will reset.
 
+`log_tag_style`: Tag implementation to use. Must be one of:
+- `uint`: Lock free and threadsafe but outputs just a simple unsigned integer 
+- `uuid`: Threadsafe and outputs a UUID tag
+- `none`: Don't use tagging at all
+
 ## Cassandra / Scylla Administration
 
 Since Clio relies on either Cassandra or Scylla for its database backend, here are some important considerations:

REVIEW.md

@@ -1,121 +0,0 @@
-# How to review clio
-Clio is a massive project, and thus I don't expect the code to be reviewed the
-way a normal PR would. So I put this guide together to help reviewers look at
-the relevant pieces of code without getting lost in the weeds.
-
-One thing reviewers should keep in mind is that most of clio is designed to be
-lightweight and simple. We try not to introduce any uneccessary complexity and
-keep the code as simple and straightforward as possible. Sometimes complexity is
-unavoidable, but simplicity is the goal.
-
-## Order of review
-The code is organized into 4 main components, each with their own folder. The
-code in each folder is as self contained as possible. A good way to approach
-the review would be to review one folder at a time.
-
-### backend
-The code in the backend folder is the heart of the project, and reviewers should
-start here. This is the most complex part of the code, as well as the most
-performance sensitive. clio does not keep any data in memory, so performance
-generally depends on the data model and the way we talk to the database.
-
-Reviewers should start with the README in this folder to get a high level idea
-of the data model and to review the data model itself. Then, reviewers should
-dive into the implementation. The table schemas and queries for Cassandra are 
-defined in `CassandraBackend::open()`. The table schemas for Postgres are defined
-in Pg.cpp. The queries for Postgres are defined in each of the functions of `PostgresBackend`.
-A good way to approach the implementation would be to look at the table schemas,
-and then go through the functions declared in `BackendInterface`. Reviewers could
-also branch out to the rest of the code by looking at where these functions are
-called from.
-
-### webserver
-The code in the webserver folder implements the web server for handling RPC requests.
-This code was mostly copied and pasted from boost beast example code, so I would
-really appreciate review here.
-
-### rpc
-The rpc folder contains all of the handlers and any helper functions they need.
-This code is not too complicated, so reviewers don't need to dwell long here.
-
-### etl
-The etl folder contains all of the code for extracting data from rippled. This
-code is complex and important, but most of this code was just copied from rippled
-reporting mode, and thus has already been reviewed and is being used in prod.
-
-## Design decisions that should be reviewed
-
-### Data model
-Reviewers should review the general data model. The data model itself is described
-at a high level in the README in the backend folder. The table schemas and queries
-for Cassandra are defined in the `open()` function of `CassandraBackend`. The table
-schemas for Postgres are defined in Pg.cpp. 
-
-Particular attention should be paid to the keys table, and the problem that solves
-(successor/upper bound). I originally was going to have a special table for book_offers,
-but then I decided that we could use the keys table itself for that and save space.
-This makes book_offers somewhat slow compared to rippled, though still very usable.
-
-### Large rows
-I did some tricks with Cassandra to deal with very large rows in the keys and account_tx
-tables. For each of these, the partition key (the first component of the primary
-key) is a compound key. This is meant to break large rows into smaller rows. This
-is done to avoid hotspots. Data is sharded in Cassandra, and if some rows get very
-large, some nodes can have a lot more data than others.
-
-For account_tx, this has performance implications when iterating very far back
-in time. Refer to the `fetchAccountTransactions()` function in `CassandraBackend`.
-
-It is unclear if this needs to be done for other tables.
-
-### Postgres table partitioning
-Originally, Postgres exhibited performance problems when the dataset approach 1
-TB. This was solved by table partitioning.
-
-### Threading
-I used asio for multithreading. There are a lot of different io_contexts lying
-around the code. This needs to be cleaned up a bit. Most of these are really
-just ways to submit an async job to a single thread. I don't think it makes
-sense to have one io_context for the whole application, but some of the threading
-is a bit opaque and could be cleaned up.
-
-### Boost Json
-I used boost json for serializing data to json.
-
-### No cache
-As of now, there is no cache. I am not sure if a cache is even worth it. A
-transaction cache would not be hard, but a cache for ledger data will be hard.
-While a cache would improve performance, it would increase memory usage. clio
-is designed to be lightweight. Also, I've reached thousands of requests per
-second with a single clio node, so I'm not sure performance is even an issue.
-
-## Things I'm less than happy about
-
-#### BackendIndexer
-This is a particularly hairy piece of code that handles writing to the keys table.
-I am not too happy with this code. Parts of it need to execute in real time as
-part of ETL, and other parts are allowed to run in the background. There is also
-code that detects if a previous background job failed to complete before the
-server shutdown, and thus tries to rerun that job. The code feels tacked on, and
-I would like it to be more cleanly integrated with the rest of the code.
-
-#### Shifting
-There is some bit shifting going on with the keys table and the account_tx table.
-The keys table is written to every 2^20 ledgers. Maybe it would be better to just
-write every 1 million ledgers. 
-
-#### performance of book_offers
-book_offers is a bit slow. It could be sped up in a variety of ways. One is to
-keep a separate book_offers table. However, this is not straightforward and will
-use more space. Another is to keep a cache of book_offers for the most recent ledger
-(or few ledgers). I am not sure if this is worth it
-
-#### account_tx in Cassandra
-After the fix to deal with large rows, account_tx can be slow at times when using
-Cassandra. Specifically, if there are large gaps in time where the account was
-not affected by any transactions, the code will be reading empty records. I would
-like to sidestep this issue if possible.
-
-#### Implementation of fetchLedgerPage
-`fetchLedgerPage()` is rather complex. Part of this seems unavoidable, since this
-code is dealing with the keys table.

example-config.json

@@ -1,40 +1,93 @@
 {
-    "database":
-    {
-        "type":"cassandra",
-        "cassandra":
-        {
-            "contact_points":"127.0.0.1",
-            "port":9042,
-            "keyspace":"clio",
-            "replication_factor":1,
-            "table_prefix":"",
-            "max_requests_outstanding":25000,
-            "threads":8
+    "database": {
+        "type": "cassandra",
+        "cassandra": {
+            "contact_points": "127.0.0.1",
+            "port": 9042,
+            "keyspace": "clio",
+            "replication_factor": 1,
+            "table_prefix": "",
+            "max_write_requests_outstanding": 25000,
+            "max_read_requests_outstanding": 30000,
+            "threads": 8
         }
     },
-    "etl_sources":
-    [
+    "etl_sources": [
         {
-            "ip":"127.0.0.1",
-            "ws_port":"6006",
-            "grpc_port":"50051"
+            "ip": "127.0.0.1",
+            "ws_port": "6006",
+            "grpc_port": "50051"
         }
     ],
-    "dos_guard":
-    {
-        "whitelist":["127.0.0.1"]
+    "dos_guard": {
+        "whitelist": [
+            "127.0.0.1"
+        ], // comma-separated list of ips to exclude from rate limiting
+        /* The below values are the default values and are only specified here
+         * for documentation purposes. The rate limiter currently limits
+         * connections and bandwidth per ip. The rate limiter looks at the raw
+         * ip of a client connection, and so requests routed through a load
+         * balancer will all have the same ip and be treated as a single client
+         */
+        "max_fetches": 1000000, // max bytes per ip per sweep interval
+        "max_connections": 20, // max connections per ip
+        "max_requests": 20, // max connections per ip
+        "sweep_interval": 1 // time in seconds before resetting bytes per ip count
     },
-    "server":{
-        "ip":"0.0.0.0",
-        "port":51233
+    "cache": {
+        "peers": [
+            {
+                "ip": "127.0.0.1",
+                "port": 51234
+            }
+        ]
     },
-    "log_level":"debug",
+    "server": {
+        "ip": "0.0.0.0",
+        "port": 51233,
+        /* Max number of requests to queue up before rejecting further requests.
+        * Defaults to 0, which disables the limit
+        */
+        "max_queue_size": 500
+    },
+    "log_channels": [
+        {
+            "channel": "Backend",
+            "log_level": "fatal"
+        },
+        {
+            "channel": "WebServer",
+            "log_level": "info"
+        },
+        {
+            "channel": "Subscriptions",
+            "log_level": "info"
+        },
+        {
+            "channel": "RPC",
+            "log_level": "error"
+        },
+        {
+            "channel": "ETL",
+            "log_level": "debug"
+        },
+        {
+            "channel": "Performance",
+            "log_level": "trace"
+        }
+    ],
+    "log_level": "info",
+    "log_format": "%TimeStamp% (%SourceLocation%) [%ThreadID%] %Channel%:%Severity% %Message%", // This is the default format
     "log_to_console": true,
-    "log_directory":"./clio_log",
+    "log_directory": "./clio_log",
     "log_rotation_size": 2048,
     "log_directory_max_size": 51200,
     "log_rotation_hour_interval": 12,
-    "extractor_threads":8,
-    "read_only":false
+    "log_tag_style": "uint",
+    "extractor_threads": 8,
+    "read_only": false,
+    //"start_sequence": [integer] the ledger index to start from,
+    //"finish_sequence": [integer] the ledger index to finish at,
+    //"ssl_cert_file" : "/full/path/to/cert.file",
+    //"ssl_key_file" : "/full/path/to/key.file"
 }

src/backend/BackendFactory.h

@@ -1,47 +1,47 @@
-#ifndef RIPPLE_APP_REPORTING_BACKENDFACTORY_H_INCLUDED
-#define RIPPLE_APP_REPORTING_BACKENDFACTORY_H_INCLUDED
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
 
-#include <boost/algorithm/string.hpp>
 #include <backend/BackendInterface.h>
 #include <backend/CassandraBackend.h>
-#include <backend/PostgresBackend.h>
+#include <config/Config.h>
+#include <log/Logger.h>
+
+#include <boost/algorithm/string.hpp>
 
 namespace Backend {
 std::shared_ptr<BackendInterface>
-make_Backend(boost::asio::io_context& ioc, boost::json::object const& config)
+make_Backend(boost::asio::io_context& ioc, clio::Config const& config)
 {
-    BOOST_LOG_TRIVIAL(info) << __func__ << ": Constructing BackendInterface";
-
-    boost::json::object dbConfig = config.at("database").as_object();
-
-    bool readOnly = false;
-    if (config.contains("read_only"))
-        readOnly = config.at("read_only").as_bool();
-
-    auto type = dbConfig.at("type").as_string();
+    static clio::Logger log{"Backend"};
+    log.info() << "Constructing BackendInterface";
 
+    auto readOnly = config.valueOr("read_only", false);
+    auto type = config.value<std::string>("database.type");
     std::shared_ptr<BackendInterface> backend = nullptr;
 
     if (boost::iequals(type, "cassandra"))
     {
-        if (config.contains("online_delete"))
-            dbConfig.at(type).as_object()["ttl"] =
-                config.at("online_delete").as_int64() * 4;
-        backend = std::make_shared<CassandraBackend>(
-            ioc, dbConfig.at(type).as_object());
-    }
-    else if (boost::iequals(type, "postgres"))
-    {
-        if (dbConfig.contains("experimental") &&
-            dbConfig.at("experimental").is_bool() &&
-            dbConfig.at("experimental").as_bool())
-            backend = std::make_shared<PostgresBackend>(
-                ioc, dbConfig.at(type).as_object());
-        else
-            BOOST_LOG_TRIVIAL(fatal)
-                << "Postgres support is experimental at this time. "
-                << "If you would really like to use Postgres, add "
-                   "\"experimental\":true to your database config";
+        auto cfg = config.section("database." + type);
+        auto ttl = config.valueOr<uint32_t>("online_delete", 0) * 4;
+        backend = std::make_shared<CassandraBackend>(ioc, cfg, ttl);
     }
 
     if (!backend)
@@ -55,11 +55,8 @@ make_Backend(boost::asio::io_context& ioc, boost::json::object const& config)
         backend->updateRange(rng->maxSequence);
     }
 
-    BOOST_LOG_TRIVIAL(info)
-        << __func__ << ": Constructed BackendInterface Successfully";
+    log.info() << "Constructed BackendInterface Successfully";
 
     return backend;
 }
 }  // namespace Backend
-
-#endif  // RIPPLE_REPORTING_BACKEND_FACTORY

src/backend/BackendInterface.cpp

@@ -1,6 +1,34 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
 #include <ripple/protocol/Indexes.h>
 #include <ripple/protocol/STLedgerEntry.h>
 #include <backend/BackendInterface.h>
+#include <log/Logger.h>
+
+using namespace clio;
+
+// local to compilation unit loggers
+namespace {
+clio::Logger gLog{"Backend"};
+}  // namespace
+
 namespace Backend {
 bool
 BackendInterface::finishWrites(std::uint32_t const ledgerSequence)
@@ -26,7 +54,7 @@ std::optional<LedgerRange>
 BackendInterface::hardFetchLedgerRangeNoThrow(
     boost::asio::yield_context& yield) const
 {
-    BOOST_LOG_TRIVIAL(debug) << __func__;
+    gLog.trace() << "called";
     while (true)
     {
         try
@@ -43,7 +71,7 @@ BackendInterface::hardFetchLedgerRangeNoThrow(
 std::optional<LedgerRange>
 BackendInterface::hardFetchLedgerRangeNoThrow() const
 {
-    BOOST_LOG_TRIVIAL(debug) << __func__;
+    gLog.trace() << "called";
     return retryOnTimeout([&]() { return hardFetchLedgerRange(); });
 }
 
@@ -57,21 +85,17 @@ BackendInterface::fetchLedgerObject(
     auto obj = cache_.get(key, sequence);
     if (obj)
     {
-        BOOST_LOG_TRIVIAL(trace)
-            << __func__ << " - cache hit - " << ripple::strHex(key);
+        gLog.trace() << "Cache hit - " << ripple::strHex(key);
         return *obj;
     }
     else
     {
-        BOOST_LOG_TRIVIAL(trace)
-            << __func__ << " - cache miss - " << ripple::strHex(key);
+        gLog.trace() << "Cache miss - " << ripple::strHex(key);
         auto dbObj = doFetchLedgerObject(key, sequence, yield);
         if (!dbObj)
-            BOOST_LOG_TRIVIAL(trace)
-                << __func__ << " - missed cache and missed in db";
+            gLog.trace() << "Missed cache and missed in db";
         else
-            BOOST_LOG_TRIVIAL(trace)
-                << __func__ << " - missed cache but found in db";
+            gLog.trace() << "Missed cache but found in db";
         return dbObj;
     }
 }
@@ -93,9 +117,8 @@ BackendInterface::fetchLedgerObjects(
         else
             misses.push_back(keys[i]);
     }
-    BOOST_LOG_TRIVIAL(trace)
-        << __func__ << " - cache hits = " << keys.size() - misses.size()
-        << " - cache misses = " << misses.size();
+    gLog.trace() << "Cache hits = " << keys.size() - misses.size()
+                 << " - cache misses = " << misses.size();
 
     if (misses.size())
     {
@@ -121,11 +144,9 @@ BackendInterface::fetchSuccessorKey(
 {
     auto succ = cache_.getSuccessor(key, ledgerSequence);
     if (succ)
-        BOOST_LOG_TRIVIAL(trace)
-            << __func__ << " - cache hit - " << ripple::strHex(key);
+        gLog.trace() << "Cache hit - " << ripple::strHex(key);
     else
-        BOOST_LOG_TRIVIAL(trace)
-            << __func__ << " - cache miss - " << ripple::strHex(key);
+        gLog.trace() << "Cache miss - " << ripple::strHex(key);
     return succ ? succ->key : doFetchSuccessorKey(key, ledgerSequence, yield);
 }
 
@@ -179,8 +200,8 @@ BackendInterface::fetchBookOffers(
         succMillis += getMillis(mid2 - mid1);
         if (!offerDir || offerDir->key >= bookEnd)
         {
-            BOOST_LOG_TRIVIAL(trace) << __func__ << " - offerDir.has_value() "
-                                     << offerDir.has_value() << " breaking";
+            gLog.trace() << "offerDir.has_value() " << offerDir.has_value()
+                         << " breaking";
             break;
         }
         uTipIndex = offerDir->key;
@@ -196,8 +217,7 @@ BackendInterface::fetchBookOffers(
             auto next = sle.getFieldU64(ripple::sfIndexNext);
             if (!next)
             {
-                BOOST_LOG_TRIVIAL(trace)
-                    << __func__ << " next is empty. breaking";
+                gLog.trace() << "Next is empty. breaking";
                 break;
             }
             auto nextKey = ripple::keylet::page(uTipIndex, next);
@@ -214,29 +234,27 @@ BackendInterface::fetchBookOffers(
     auto objs = fetchLedgerObjects(keys, ledgerSequence, yield);
     for (size_t i = 0; i < keys.size() && i < limit; ++i)
     {
-        BOOST_LOG_TRIVIAL(trace)
-            << __func__ << " key = " << ripple::strHex(keys[i])
-            << " blob = " << ripple::strHex(objs[i])
-            << " ledgerSequence = " << ledgerSequence;
+        gLog.trace() << "Key = " << ripple::strHex(keys[i])
+                     << " blob = " << ripple::strHex(objs[i])
+                     << " ledgerSequence = " << ledgerSequence;
         assert(objs[i].size());
         page.offers.push_back({keys[i], objs[i]});
     }
     auto end = std::chrono::system_clock::now();
-    BOOST_LOG_TRIVIAL(debug)
-        << __func__ << " "
-        << "Fetching " << std::to_string(keys.size()) << " offers took "
-        << std::to_string(getMillis(mid - begin))
-        << " milliseconds. Fetching next dir took "
-        << std::to_string(succMillis) << " milliseonds. Fetched next dir "
-        << std::to_string(numSucc) << " times"
-        << " Fetching next page of dir took " << std::to_string(pageMillis)
-        << " milliseconds"
-        << ". num pages = " << std::to_string(numPages)
-        << ". Fetching all objects took "
-        << std::to_string(getMillis(end - mid))
-        << " milliseconds. total time = "
-        << std::to_string(getMillis(end - begin)) << " milliseconds"
-        << " book = " << ripple::strHex(book);
+    gLog.debug() << "Fetching " << std::to_string(keys.size())
+                 << " offers took " << std::to_string(getMillis(mid - begin))
+                 << " milliseconds. Fetching next dir took "
+                 << std::to_string(succMillis)
+                 << " milliseonds. Fetched next dir " << std::to_string(numSucc)
+                 << " times"
+                 << " Fetching next page of dir took "
+                 << std::to_string(pageMillis) << " milliseconds"
+                 << ". num pages = " << std::to_string(numPages)
+                 << ". Fetching all objects took "
+                 << std::to_string(getMillis(end - mid))
+                 << " milliseconds. total time = "
+                 << std::to_string(getMillis(end - begin)) << " milliseconds"
+                 << " book = " << ripple::strHex(book);
 
     return page;
 }
@@ -274,16 +292,15 @@ BackendInterface::fetchLedgerPage(
             page.objects.push_back({std::move(keys[i]), std::move(objects[i])});
         else if (!outOfOrder)
         {
-            BOOST_LOG_TRIVIAL(error)
-                << __func__
-                << " deleted or non-existent object in successor table. key = "
+            gLog.error()
+                << "Deleted or non-existent object in successor table. key = "
                 << ripple::strHex(keys[i]) << " - seq = " << ledgerSequence;
             std::stringstream msg;
             for (size_t j = 0; j < objects.size(); ++j)
             {
                 msg << " - " << ripple::strHex(keys[j]);
             }
-            BOOST_LOG_TRIVIAL(error) << __func__ << msg.str();
+            gLog.error() << msg.str();
         }
     }
     if (keys.size() && !reachedEnd)
@@ -304,7 +321,7 @@ BackendInterface::fetchFees(
 
     if (!bytes)
     {
-        BOOST_LOG_TRIVIAL(error) << __func__ << " - could not find fees";
+        gLog.error() << "Could not find fees";
         return {};
     }
 

src/backend/BackendInterface.h

@@ -1,16 +1,48 @@
-#ifndef RIPPLE_APP_REPORTING_BACKENDINTERFACE_H_INCLUDED
-#define RIPPLE_APP_REPORTING_BACKENDINTERFACE_H_INCLUDED
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
 #include <ripple/ledger/ReadView.h>
-#include <boost/asio.hpp>
 #include <backend/DBHelpers.h>
 #include <backend/SimpleCache.h>
 #include <backend/Types.h>
+#include <config/Config.h>
+#include <log/Logger.h>
+
+#include <boost/asio/spawn.hpp>
+#include <boost/json.hpp>
+
 #include <thread>
 #include <type_traits>
+
 namespace Backend {
 
+/**
+ * @brief Throws an error when database read time limit is exceeded.
+ *
+ * This class is throws an error when read time limit is exceeded but
+ * is also paired with a separate class to retry the connection.
+ */
 class DatabaseTimeout : public std::exception
 {
+public:
     const char*
     what() const throw() override
     {
@@ -18,10 +50,20 @@ class DatabaseTimeout : public std::exception
     }
 };
 
+/**
+ * @brief Separate class that reattempts connection after time limit.
+ *
+ * @tparam F Represents a class of handlers for Cassandra database.
+ * @param func Instance of Cassandra database handler class.
+ * @param waitMs Is the arbitrary time limit of 500ms.
+ * @return auto
+ */
 template <class F>
 auto
 retryOnTimeout(F func, size_t waitMs = 500)
 {
+    static clio::Logger log{"Backend"};
+
     while (true)
     {
         try
@@ -30,26 +72,55 @@ retryOnTimeout(F func, size_t waitMs = 500)
         }
         catch (DatabaseTimeout& t)
         {
+            log.error()
+                << "Database request timed out. Sleeping and retrying ... ";
             std::this_thread::sleep_for(std::chrono::milliseconds(waitMs));
-            BOOST_LOG_TRIVIAL(error)
-                << __func__ << " function timed out. Retrying ... ";
         }
     }
 }
 
+/**
+ * @brief Passes in serialized handlers in an asynchronous fashion.
+ *
+ * Note that the synchronous auto passes handlers critical to supporting
+ * the Clio backend. The coroutine types are checked if same/different.
+ *
+ * @tparam F Represents a class of handlers for Cassandra database.
+ * @param f R-value instance of Cassandra handler class.
+ * @return auto
+ */
 template <class F>
 auto
 synchronous(F&& f)
 {
+    /** @brief Serialized handlers and their execution.
+     *
+     * The ctx class is converted into a serialized handler, also named
+     * ctx, and is used to pass a stream of data into the method.
+     */
     boost::asio::io_context ctx;
     boost::asio::io_context::strand strand(ctx);
     std::optional<boost::asio::io_context::work> work;
 
+    /*! @brief Place the ctx within the vector of serialized handlers. */
     work.emplace(ctx);
 
+    /**
+     * @brief If/else statements regarding coroutine type matching.
+     *
+     * R is the currently executing coroutine that is about to get passed.
+     * If corountine types do not match, the current one's type is stored.
+     */
     using R = typename std::result_of<F(boost::asio::yield_context&)>::type;
     if constexpr (!std::is_same<R, void>::value)
     {
+        /**
+         * @brief When the coroutine type is the same
+         *
+         * The spawn function enables programs to implement asynchronous logic
+         * in a synchronous manner. res stores the instance of the currently
+         * executing coroutine, yield. The different type is returned.
+         */
         R res;
         boost::asio::spawn(
             strand, [&f, &work, &res](boost::asio::yield_context yield) {
@@ -62,6 +133,7 @@ synchronous(F&& f)
     }
     else
     {
+        /*! @brief When the corutine type is different, run as normal. */
         boost::asio::spawn(
             strand, [&f, &work](boost::asio::yield_context yield) {
                 f(yield);
@@ -72,6 +144,13 @@ synchronous(F&& f)
     }
 }
 
+/**
+ * @brief Reestablishes synchronous connection on timeout.
+ *
+ * @tparam Represents a class of handlers for Cassandra database.
+ * @param f R-value instance of Cassandra database handler class.
+ * @return auto
+ */
 template <class F>
 auto
 synchronousAndRetryOnTimeout(F&& f)
@@ -79,32 +158,47 @@ synchronousAndRetryOnTimeout(F&& f)
     return retryOnTimeout([&]() { return synchronous(f); });
 }
 
+/*! @brief Handles ledger and transaction backend data. */
 class BackendInterface
 {
+    /**
+     * @brief Shared mutexes and a cache for the interface.
+     *
+     * rngMutex is a shared mutex. Shared mutexes prevent shared data
+     * from being accessed by multiple threads and has two levels of
+     * access: shared and exclusive.
+     */
 protected:
     mutable std::shared_mutex rngMtx_;
     std::optional<LedgerRange> range;
     SimpleCache cache_;
 
-    // mutex used for open() and close()
-    mutable std::mutex mutex_;
+    /**
+     * @brief Public read methods
+     *
+     * All of these reads methods can throw DatabaseTimeout. When writing
+     * code in an RPC handler, this exception does not need to be caught:
+     * when an RPC results in a timeout, an error is returned to the client.
+     */
 
 public:
-    BackendInterface(boost::json::object const& config)
+    BackendInterface(clio::Config const& config)
     {
     }
     virtual ~BackendInterface()
     {
     }
 
-    // *** public read methods ***
-    // All of these reads methods can throw DatabaseTimeout. When writing code
-    // in an RPC handler, this exception does not need to be caught: when an RPC
-    // results in a timeout, an error is returned to the client
+    /*! @brief LEDGER METHODS */
 public:
-    // *** ledger methods
-    //
-
+    /**
+     * @brief Cache that holds states of the ledger
+     *
+     * const version holds the original cache state; the other tracks
+     * historical changes.
+     *
+     * @return SimpleCache const&
+     */
     SimpleCache const&
     cache() const
     {
@@ -117,19 +211,23 @@ public:
         return cache_;
     }
 
+    /*! @brief Fetches a specific ledger by sequence number. */
     virtual std::optional<ripple::LedgerInfo>
     fetchLedgerBySequence(
         std::uint32_t const sequence,
         boost::asio::yield_context& yield) const = 0;
 
+    /*! @brief Fetches a specific ledger by hash. */
     virtual std::optional<ripple::LedgerInfo>
     fetchLedgerByHash(
         ripple::uint256 const& hash,
         boost::asio::yield_context& yield) const = 0;
 
+    /*! @brief Fetches the latest ledger sequence. */
     virtual std::optional<std::uint32_t>
     fetchLatestLedgerSequence(boost::asio::yield_context& yield) const = 0;
 
+    /*! @brief Fetches the current ledger range while locking that process */
     std::optional<LedgerRange>
     fetchLedgerRange() const
     {
@@ -137,10 +235,18 @@ public:
         return range;
     }
 
+    /**
+     * @brief Updates the range of sequences to be tracked.
+     *
+     * Function that continues updating the range sliding window or creates
+     * a new sliding window once the maxSequence limit has been reached.
+     *
+     * @param newMax Unsigned 32-bit integer representing new max of range.
+     */
     void
     updateRange(uint32_t newMax)
     {
-        std::unique_lock lck(rngMtx_);
+        std::scoped_lock lck(rngMtx_);
         assert(!range || newMax >= range->maxSequence);
         if (!range)
             range = {newMax, newMax};
@@ -148,20 +254,53 @@ public:
             range->maxSequence = newMax;
     }
 
+    /**
+     * @brief Returns the fees for specific transactions.
+     *
+     * @param seq Unsigned 32-bit integer reprsenting sequence.
+     * @param yield The currently executing coroutine.
+     * @return std::optional<ripple::Fees>
+     */
     std::optional<ripple::Fees>
     fetchFees(std::uint32_t const seq, boost::asio::yield_context& yield) const;
 
-    // *** transaction methods
+    /*! @brief TRANSACTION METHODS */
+    /**
+     * @brief Fetches a specific transaction.
+     *
+     * @param hash Unsigned 256-bit integer representing hash.
+     * @param yield The currently executing coroutine.
+     * @return std::optional<TransactionAndMetadata>
+     */
     virtual std::optional<TransactionAndMetadata>
     fetchTransaction(
         ripple::uint256 const& hash,
         boost::asio::yield_context& yield) const = 0;
 
+    /**
+     * @brief Fetches multiple transactions.
+     *
+     * @param hashes Unsigned integer value representing a hash.
+     * @param yield The currently executing coroutine.
+     * @return std::vector<TransactionAndMetadata>
+     */
     virtual std::vector<TransactionAndMetadata>
     fetchTransactions(
         std::vector<ripple::uint256> const& hashes,
         boost::asio::yield_context& yield) const = 0;
 
+    /**
+     * @brief Fetches all transactions for a specific account
+     *
+     * @param account A specific XRPL Account, speciifed by unique type
+     * accountID.
+     * @param limit Paging limit for how many transactions can be returned per
+     * page.
+     * @param forward Boolean whether paging happens forwards or backwards.
+     * @param cursor Important metadata returned every time paging occurs.
+     * @param yield Currently executing coroutine.
+     * @return TransactionsAndCursor
+     */
     virtual TransactionsAndCursor
     fetchAccountTransactions(
         ripple::AccountID const& account,
@@ -170,23 +309,56 @@ public:
         std::optional<TransactionsCursor> const& cursor,
         boost::asio::yield_context& yield) const = 0;
 
+    /**
+     * @brief Fetches all transactions from a specific ledger.
+     *
+     * @param ledgerSequence Unsigned 32-bit integer for latest total
+     * transactions.
+     * @param yield Currently executing coroutine.
+     * @return std::vector<TransactionAndMetadata>
+     */
     virtual std::vector<TransactionAndMetadata>
     fetchAllTransactionsInLedger(
         std::uint32_t const ledgerSequence,
         boost::asio::yield_context& yield) const = 0;
 
+    /**
+     * @brief Fetches all transaction hashes from a specific ledger.
+     *
+     * @param ledgerSequence Standard unsigned integer.
+     * @param yield Currently executing coroutine.
+     * @return std::vector<ripple::uint256>
+     */
     virtual std::vector<ripple::uint256>
     fetchAllTransactionHashesInLedger(
         std::uint32_t const ledgerSequence,
         boost::asio::yield_context& yield) const = 0;
 
-    // *** NFT methods
+    /*! @brief NFT methods */
+    /**
+     * @brief Fetches a specific NFT
+     *
+     * @param tokenID Unsigned 256-bit integer.
+     * @param ledgerSequence Standard unsigned integer.
+     * @param yield Currently executing coroutine.
+     * @return std::optional<NFT>
+     */
     virtual std::optional<NFT>
     fetchNFT(
         ripple::uint256 const& tokenID,
         std::uint32_t const ledgerSequence,
         boost::asio::yield_context& yield) const = 0;
 
+    /**
+     * @brief Fetches all transactions for a specific NFT.
+     *
+     * @param tokenID Unsigned 256-bit integer.
+     * @param limit Paging limit as to how many transactions return per page.
+     * @param forward Boolean whether paging happens forwards or backwards.
+     * @param cursorIn Represents transaction number and ledger sequence.
+     * @param yield Currently executing coroutine is passed in as input.
+     * @return TransactionsAndCursor
+     */
     virtual TransactionsAndCursor
     fetchNFTTransactions(
         ripple::uint256 const& tokenID,
@@ -195,38 +367,74 @@ public:
         std::optional<TransactionsCursor> const& cursorIn,
         boost::asio::yield_context& yield) const = 0;
 
-    // *** state data methods
+    /*! @brief STATE DATA METHODS */
+    /**
+     * @brief Fetches a specific ledger object: vector of unsigned chars
+     *
+     * @param key Unsigned 256-bit integer.
+     * @param sequence Unsigned 32-bit integer.
+     * @param yield Currently executing coroutine.
+     * @return std::optional<Blob>
+     */
     std::optional<Blob>
     fetchLedgerObject(
         ripple::uint256 const& key,
         std::uint32_t const sequence,
         boost::asio::yield_context& yield) const;
 
+    /**
+     * @brief Fetches all ledger objects: a vector of vectors of unsigned chars.
+     *
+     * @param keys Unsigned 256-bit integer.
+     * @param sequence Unsigned 32-bit integer.
+     * @param yield Currently executing coroutine.
+     * @return std::vector<Blob>
+     */
     std::vector<Blob>
     fetchLedgerObjects(
         std::vector<ripple::uint256> const& keys,
         std::uint32_t const sequence,
         boost::asio::yield_context& yield) const;
 
+    /*! @brief Virtual function version of fetchLedgerObject */
     virtual std::optional<Blob>
     doFetchLedgerObject(
         ripple::uint256 const& key,
         std::uint32_t const sequence,
         boost::asio::yield_context& yield) const = 0;
 
+    /*! @brief Virtual function version of fetchLedgerObjects */
     virtual std::vector<Blob>
     doFetchLedgerObjects(
         std::vector<ripple::uint256> const& keys,
         std::uint32_t const sequence,
         boost::asio::yield_context& yield) const = 0;
 
+    /**
+     * @brief Returns the difference between ledgers: vector of objects
+     *
+     * Objects are made of a key value, vector of unsigned chars (blob),
+     * and a boolean detailing whether keys and blob match.
+     *
+     * @param ledgerSequence Standard unsigned integer.
+     * @param yield Currently executing coroutine.
+     * @return std::vector<LedgerObject>
+     */
     virtual std::vector<LedgerObject>
     fetchLedgerDiff(
         std::uint32_t const ledgerSequence,
         boost::asio::yield_context& yield) const = 0;
 
-    // Fetches a page of ledger objects, ordered by key/index.
-    // Used by ledger_data
+    /**
+     * @brief Fetches a page of ledger objects, ordered by key/index.
+     *
+     * @param cursor Important metadata returned every time paging occurs.
+     * @param ledgerSequence Standard unsigned integer.
+     * @param limit Paging limit as to how many transactions returned per page.
+     * @param outOfOrder Boolean on whether ledger page is out of order.
+     * @param yield Currently executing coroutine.
+     * @return LedgerPage
+     */
     LedgerPage
     fetchLedgerPage(
         std::optional<ripple::uint256> const& cursor,
@@ -235,26 +443,37 @@ public:
         bool outOfOrder,
         boost::asio::yield_context& yield) const;
 
-    // Fetches the successor to key/index
+    /*! @brief Fetches successor object from key/index. */
     std::optional<LedgerObject>
     fetchSuccessorObject(
         ripple::uint256 key,
         std::uint32_t const ledgerSequence,
         boost::asio::yield_context& yield) const;
 
+    /*! @brief Fetches successor key from key/index. */
     std::optional<ripple::uint256>
     fetchSuccessorKey(
         ripple::uint256 key,
         std::uint32_t const ledgerSequence,
         boost::asio::yield_context& yield) const;
-    // Fetches the successor to key/index
 
+    /*! @brief Virtual function version of fetchSuccessorKey. */
     virtual std::optional<ripple::uint256>
     doFetchSuccessorKey(
         ripple::uint256 key,
         std::uint32_t const ledgerSequence,
         boost::asio::yield_context& yield) const = 0;
 
+    /**
+     * @brief Fetches book offers.
+     *
+     * @param book Unsigned 256-bit integer.
+     * @param ledgerSequence Standard unsigned integer.
+     * @param limit Pagaing limit as to how many transactions returned per page.
+     * @param cursor Important metadata returned every time paging occurs.
+     * @param yield Currently executing coroutine.
+     * @return BookOffersPage
+     */
     BookOffersPage
     fetchBookOffers(
         ripple::uint256 const& book,
@@ -263,6 +482,16 @@ public:
         std::optional<ripple::uint256> const& cursor,
         boost::asio::yield_context& yield) const;
 
+    /**
+     * @brief Returns a ledger range
+     *
+     * Ledger range is a struct of min and max sequence numbers). Due to
+     * the use of [&], which denotes a special case of a lambda expression
+     * where values found outside the scope are passed by reference, wrt the
+     * currently executing coroutine.
+     *
+     * @return std::optional<LedgerRange>
+     */
     std::optional<LedgerRange>
     hardFetchLedgerRange() const
     {
@@ -271,27 +500,52 @@ public:
         });
     }
 
+    /*! @brief Virtual function equivalent of hardFetchLedgerRange. */
     virtual std::optional<LedgerRange>
     hardFetchLedgerRange(boost::asio::yield_context& yield) const = 0;
 
-    // Doesn't throw DatabaseTimeout. Should be used with care.
+    /*! @brief Fetches ledger range but doesn't throw timeout. Use with care. */
     std::optional<LedgerRange>
     hardFetchLedgerRangeNoThrow() const;
-    // Doesn't throw DatabaseTimeout. Should be used with care.
+    /*! @brief Fetches ledger range but doesn't throw timeout. Use with care. */
     std::optional<LedgerRange>
     hardFetchLedgerRangeNoThrow(boost::asio::yield_context& yield) const;
 
+    /**
+     * @brief Writes to a specific ledger.
+     *
+     * @param ledgerInfo Const on ledger information.
+     * @param ledgerHeader r-value string representing ledger header.
+     */
     virtual void
     writeLedger(
         ripple::LedgerInfo const& ledgerInfo,
         std::string&& ledgerHeader) = 0;
 
+    /**
+     * @brief Writes a new ledger object.
+     *
+     * The key and blob are r-value references and do NOT have memory addresses.
+     *
+     * @param key String represented as an r-value.
+     * @param seq Unsigned integer representing a sequence.
+     * @param blob r-value vector of unsigned characters (blob).
+     */
     virtual void
     writeLedgerObject(
         std::string&& key,
         std::uint32_t const seq,
         std::string&& blob);
 
+    /**
+     * @brief Writes a new transaction.
+     *
+     * @param hash r-value reference. No memory address.
+     * @param seq Unsigned 32-bit integer.
+     * @param date Unsigned 32-bit integer.
+     * @param transaction r-value reference. No memory address.
+     * @param metadata r-value refrence. No memory address.
+     */
     virtual void
     writeTransaction(
         std::string&& hash,
@@ -300,50 +554,101 @@ public:
         std::string&& transaction,
         std::string&& metadata) = 0;
 
+    /**
+     * @brief Write a new NFT.
+     *
+     * @param data Passed in as an r-value reference.
+     */
     virtual void
     writeNFTs(std::vector<NFTsData>&& data) = 0;
 
+    /**
+     * @brief Write a new set of account transactions.
+     *
+     * @param data Passed in as an r-value reference.
+     */
     virtual void
     writeAccountTransactions(std::vector<AccountTransactionsData>&& data) = 0;
 
+    /**
+     * @brief Write a new transaction for a specific NFT.
+     *
+     * @param data Passed in as an r-value reference.
+     */
     virtual void
     writeNFTTransactions(std::vector<NFTTransactionsData>&& data) = 0;
 
+    /**
+     * @brief Write a new successor.
+     *
+     * @param key Passed in as an r-value reference.
+     * @param seq Unsigned 32-bit integer.
+     * @param successor Passed in as an r-value reference.
+     */
     virtual void
     writeSuccessor(
         std::string&& key,
         std::uint32_t const seq,
         std::string&& successor) = 0;
 
-    // Tell the database we are about to begin writing data for a particular
-    // ledger.
+    /*! @brief Tells database we will write data for a specific ledger. */
     virtual void
     startWrites() const = 0;
 
-    // Tell the database we have finished writing all data for a particular
-    // ledger
-    // TODO change the return value to represent different results. committed,
-    // write conflict, errored, successful but not committed
+    /**
+     * @brief Tells database we finished writing all data for a specific ledger.
+     *
+     * TODO: change the return value to represent different results:
+     * Committed, write conflict, errored, successful but not committed
+     *
+     * @param ledgerSequence Const unsigned 32-bit integer on ledger sequence.
+     * @return true
+     * @return false
+     */
     bool
     finishWrites(std::uint32_t const ledgerSequence);
 
+    /**
+     * @brief Selectively delets parts of the database.
+     *
+     * @param numLedgersToKeep Unsigned 32-bit integer on number of ledgers to
+     * keep.
+     * @param yield Currently executing coroutine.
+     * @return true
+     * @return false
+     */
     virtual bool
     doOnlineDelete(
         std::uint32_t numLedgersToKeep,
         boost::asio::yield_context& yield) const = 0;
 
-    // Open the database. Set up all of the necessary objects and
-    // datastructures. After this call completes, the database is ready for
-    // use.
+    /**
+     * @brief Opens the database
+     *
+     * Open the database. Set up all of the necessary objects and
+     * datastructures. After this call completes, the database is
+     * ready for use.
+     *
+     * @param readOnly Boolean whether ledger is read only.
+     */
     virtual void
     open(bool readOnly) = 0;
 
-    // Close the database, releasing any resources
+    /*! @brief Closes the database, releasing any resources. */
     virtual void
     close(){};
 
-    // *** private helper methods
+    virtual bool
+    isTooBusy() const = 0;
+
 private:
+    /**
+     * @brief Private helper method to write ledger object
+     *
+     * @param key r-value string representing key.
+     * @param seq Unsigned 32-bit integer representing sequence.
+     * @param blob r-value vector of unsigned chars.
+     */
     virtual void
     doWriteLedgerObject(
         std::string&& key,
@@ -356,4 +661,3 @@ private:
 
 }  // namespace Backend
 using BackendInterface = Backend::BackendInterface;
-#endif

src/backend/CassandraBackend.cpp

@@ -1,9 +1,33 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
 #include <ripple/app/tx/impl/details/NFTokenUtils.h>
 #include <backend/CassandraBackend.h>
 #include <backend/DBHelpers.h>
+#include <log/Logger.h>
+#include <util/Profiler.h>
+
 #include <functional>
 #include <unordered_map>
 
+using namespace clio;
+
 namespace Backend {
 
 // Type alias for async completion handlers
@@ -16,6 +40,8 @@ template <class T, class F>
 void
 processAsyncWriteResponse(T& requestParams, CassFuture* fut, F func)
 {
+    static clio::Logger log{"Backend"};
+
     CassandraBackend const& backend = *requestParams.backend;
     auto rc = cass_future_error_code(fut);
     if (rc != CASS_OK)
@@ -23,11 +49,11 @@ processAsyncWriteResponse(T& requestParams, CassFuture* fut, F func)
         // exponential backoff with a max wait of 2^10 ms (about 1 second)
         auto wait = std::chrono::milliseconds(
             lround(std::pow(2, std::min(10u, requestParams.currentRetries))));
-        BOOST_LOG_TRIVIAL(error)
-            << "ERROR!!! Cassandra write error: " << rc << ", "
-            << cass_error_desc(rc) << " id= " << requestParams.toString()
-            << ", current retries " << requestParams.currentRetries
-            << ", retrying in " << wait.count() << " milliseconds";
+        log.error() << "ERROR!!! Cassandra write error: " << rc << ", "
+                    << cass_error_desc(rc)
+                    << " id= " << requestParams.toString()
+                    << ", current retries " << requestParams.currentRetries
+                    << ", retrying in " << wait.count() << " milliseconds";
         ++requestParams.currentRetries;
         std::shared_ptr<boost::asio::steady_timer> timer =
             std::make_shared<boost::asio::steady_timer>(
@@ -40,11 +66,11 @@ processAsyncWriteResponse(T& requestParams, CassFuture* fut, F func)
     }
     else
     {
-        BOOST_LOG_TRIVIAL(trace)
-            << __func__ << " Succesfully inserted a record";
+        log.trace() << "Succesfully inserted a record";
         requestParams.finish();
     }
 }
+
 template <class T>
 void
 processAsyncWrite(CassFuture* fut, void* cbData)
@@ -155,6 +181,7 @@ makeAndExecuteAsyncWrite(
     auto* cb = new WriteCallbackData<T, B>(b, std::move(d), bind, id);
     cb->start();
 }
+
 template <class T, class B>
 std::shared_ptr<BulkWriteCallbackData<T, B>>
 makeAndExecuteBulkAsyncWrite(
@@ -170,13 +197,14 @@ makeAndExecuteBulkAsyncWrite(
     cb->start();
     return cb;
 }
+
 void
 CassandraBackend::doWriteLedgerObject(
     std::string&& key,
     std::uint32_t const seq,
     std::string&& blob)
 {
-    BOOST_LOG_TRIVIAL(trace) << "Writing ledger object to cassandra";
+    log_.trace() << "Writing ledger object to cassandra";
     if (range)
         makeAndExecuteAsyncWrite(
             this,
@@ -204,15 +232,16 @@ CassandraBackend::doWriteLedgerObject(
         },
         "ledger_object");
 }
+
 void
 CassandraBackend::writeSuccessor(
     std::string&& key,
     std::uint32_t const seq,
     std::string&& successor)
 {
-    BOOST_LOG_TRIVIAL(trace)
-        << "Writing successor. key = " << key
-        << " seq = " << std::to_string(seq) << " successor = " << successor;
+    log_.trace() << "Writing successor. key = " << key.size() << " bytes. "
+                 << " seq = " << std::to_string(seq)
+                 << " successor = " << successor.size() << " bytes.";
     assert(key.size() != 0);
     assert(successor.size() != 0);
     makeAndExecuteAsyncWrite(
@@ -319,7 +348,7 @@ CassandraBackend::writeTransaction(
     std::string&& transaction,
     std::string&& metadata)
 {
-    BOOST_LOG_TRIVIAL(trace) << "Writing txn to cassandra";
+    log_.trace() << "Writing txn to cassandra";
     std::string hashCpy = hash;
 
     makeAndExecuteAsyncWrite(
@@ -393,13 +422,13 @@ CassandraBackend::writeNFTs(std::vector<NFTsData>&& data)
 std::optional<LedgerRange>
 CassandraBackend::hardFetchLedgerRange(boost::asio::yield_context& yield) const
 {
-    BOOST_LOG_TRIVIAL(trace) << "Fetching from cassandra";
+    log_.trace() << "Fetching from cassandra";
     CassandraStatement statement{selectLedgerRange_};
     CassandraResult result = executeAsyncRead(statement, yield);
 
     if (!result)
     {
-        BOOST_LOG_TRIVIAL(error) << __func__ << " - no rows";
+        log_.error() << "No rows";
         return {};
     }
     LedgerRange range;
@@ -486,6 +515,7 @@ CassandraBackend::fetchTransactions(
 {
     if (hashes.size() == 0)
         return {};
+    numReadRequestsOutstanding_ += hashes.size();
 
     handler_type handler(std::forward<decltype(yield)>(yield));
     result_type result(handler);
@@ -495,42 +525,39 @@ CassandraBackend::fetchTransactions(
     std::vector<TransactionAndMetadata> results{numHashes};
     std::vector<std::shared_ptr<ReadCallbackData<result_type>>> cbs;
     cbs.reserve(numHashes);
-    auto start = std::chrono::system_clock::now();
+    auto timeDiff = util::timed([&]() {
+        for (std::size_t i = 0; i < hashes.size(); ++i)
+        {
+            CassandraStatement statement{selectTransaction_};
+            statement.bindNextBytes(hashes[i]);
 
-    for (std::size_t i = 0; i < hashes.size(); ++i)
-    {
-        CassandraStatement statement{selectTransaction_};
-        statement.bindNextBytes(hashes[i]);
+            cbs.push_back(std::make_shared<ReadCallbackData<result_type>>(
+                numOutstanding, handler, [i, &results](auto& result) {
+                    if (result.hasResult())
+                        results[i] = {
+                            result.getBytes(),
+                            result.getBytes(),
+                            result.getUInt32(),
+                            result.getUInt32()};
+                }));
 
-        cbs.push_back(std::make_shared<ReadCallbackData<result_type>>(
-            numOutstanding, handler, [i, &results](auto& result) {
-                if (result.hasResult())
-                    results[i] = {
-                        result.getBytes(),
-                        result.getBytes(),
-                        result.getUInt32(),
-                        result.getUInt32()};
-            }));
+            executeAsyncRead(statement, processAsyncRead, *cbs[i]);
+        }
+        assert(results.size() == cbs.size());
 
-        executeAsyncRead(statement, processAsyncRead, *cbs[i]);
-    }
-    assert(results.size() == cbs.size());
-
-    // suspend the coroutine until completion handler is called.
-    result.get();
-
-    auto end = std::chrono::system_clock::now();
+        // suspend the coroutine until completion handler is called.
+        result.get();
+        numReadRequestsOutstanding_ -= hashes.size();
+    });
     for (auto const& cb : cbs)
     {
         if (cb->errored)
             throw DatabaseTimeout();
     }
 
-    BOOST_LOG_TRIVIAL(debug)
-        << "Fetched " << numHashes << " transactions from Cassandra in "
-        << std::chrono::duration_cast<std::chrono::milliseconds>(end - start)
-               .count()
-        << " milliseconds";
+    log_.debug() << "Fetched " << numHashes
+                 << " transactions from Cassandra in " << timeDiff
+                 << " milliseconds";
     return results;
 }
 
@@ -548,9 +575,7 @@ CassandraBackend::fetchAllTransactionHashesInLedger(
     auto end = std::chrono::system_clock::now();
     if (!result)
     {
-        BOOST_LOG_TRIVIAL(error)
-            << __func__
-            << " - no rows . ledger = " << std::to_string(ledgerSequence);
+        log_.error() << "No rows. Ledger = " << std::to_string(ledgerSequence);
         return {};
     }
     std::vector<ripple::uint256> hashes;
@@ -558,12 +583,12 @@ CassandraBackend::fetchAllTransactionHashesInLedger(
     {
         hashes.push_back(result.getUInt256());
     } while (result.nextRow());
-    BOOST_LOG_TRIVIAL(debug)
-        << "Fetched " << hashes.size()
-        << " transaction hashes from Cassandra in "
-        << std::chrono::duration_cast<std::chrono::milliseconds>(end - start)
-               .count()
-        << " milliseconds";
+    log_.debug() << "Fetched " << hashes.size()
+                 << " transaction hashes from Cassandra in "
+                 << std::chrono::duration_cast<std::chrono::milliseconds>(
+                        end - start)
+                        .count()
+                 << " milliseconds";
     return hashes;
 }
 
@@ -611,9 +636,9 @@ CassandraBackend::fetchNFTTransactions(
     {
         statement.bindNextIntTuple(
             cursor->ledgerSequence, cursor->transactionIndex);
-        BOOST_LOG_TRIVIAL(debug) << " token_id = " << ripple::strHex(tokenID)
-                                 << " tuple = " << cursor->ledgerSequence
-                                 << " : " << cursor->transactionIndex;
+        log_.debug() << "token_id = " << ripple::strHex(tokenID)
+                     << " tuple = " << cursor->ledgerSequence
+                     << cursor->transactionIndex;
     }
     else
     {
@@ -622,9 +647,8 @@ CassandraBackend::fetchNFTTransactions(
             forward ? 0 : std::numeric_limits<std::uint32_t>::max();
 
         statement.bindNextIntTuple(placeHolder, placeHolder);
-        BOOST_LOG_TRIVIAL(debug)
-            << " token_id = " << ripple::strHex(tokenID) << " idx = " << seq
-            << " tuple = " << placeHolder;
+        log_.debug() << "token_id = " << ripple::strHex(tokenID)
+                     << " idx = " << seq << " tuple = " << placeHolder;
     }
 
     statement.bindNextUInt(limit);
@@ -633,35 +657,38 @@ CassandraBackend::fetchNFTTransactions(
 
     if (!result.hasResult())
     {
-        BOOST_LOG_TRIVIAL(debug) << __func__ << " - no rows returned";
+        log_.debug() << "No rows returned";
         return {};
     }
 
     std::vector<ripple::uint256> hashes = {};
     auto numRows = result.numRows();
-    BOOST_LOG_TRIVIAL(info) << "num_rows = " << numRows;
+    log_.info() << "num_rows = " << numRows;
     do
     {
         hashes.push_back(result.getUInt256());
         if (--numRows == 0)
         {
-            BOOST_LOG_TRIVIAL(debug) << __func__ << " setting cursor";
+            log_.debug() << "Setting cursor";
             auto const [lgrSeq, txnIdx] = result.getInt64Tuple();
             cursor = {
                 static_cast<std::uint32_t>(lgrSeq),
                 static_cast<std::uint32_t>(txnIdx)};
 
+            // Only modify if forward because forward query
+            // (selectNFTTxForward_) orders by ledger/tx sequence >= whereas
+            // reverse query (selectNFTTx_) orders by ledger/tx sequence <.
             if (forward)
                 ++cursor->transactionIndex;
         }
     } while (result.nextRow());
 
     auto txns = fetchTransactions(hashes, yield);
-    BOOST_LOG_TRIVIAL(debug) << __func__ << " txns = " << txns.size();
+    log_.debug() << "Txns = " << txns.size();
 
     if (txns.size() == limit)
     {
-        BOOST_LOG_TRIVIAL(debug) << __func__ << " returning cursor";
+        log_.debug() << "Returning cursor";
         return {txns, cursor};
     }
 
@@ -693,9 +720,9 @@ CassandraBackend::fetchAccountTransactions(
     {
         statement.bindNextIntTuple(
             cursor->ledgerSequence, cursor->transactionIndex);
-        BOOST_LOG_TRIVIAL(debug) << " account = " << ripple::strHex(account)
-                                 << " tuple = " << cursor->ledgerSequence
-                                 << " : " << cursor->transactionIndex;
+        log_.debug() << "account = " << ripple::strHex(account)
+                     << " tuple = " << cursor->ledgerSequence
+                     << cursor->transactionIndex;
     }
     else
     {
@@ -704,9 +731,8 @@ CassandraBackend::fetchAccountTransactions(
             forward ? 0 : std::numeric_limits<std::uint32_t>::max();
 
         statement.bindNextIntTuple(placeHolder, placeHolder);
-        BOOST_LOG_TRIVIAL(debug)
-            << " account = " << ripple::strHex(account) << " idx = " << seq
-            << " tuple = " << placeHolder;
+        log_.debug() << "account = " << ripple::strHex(account)
+                     << " idx = " << seq << " tuple = " << placeHolder;
     }
     statement.bindNextUInt(limit);
 
@@ -714,35 +740,38 @@ CassandraBackend::fetchAccountTransactions(
 
     if (!result.hasResult())
     {
-        BOOST_LOG_TRIVIAL(debug) << __func__ << " - no rows returned";
+        log_.debug() << "No rows returned";
         return {};
     }
 
     std::vector<ripple::uint256> hashes = {};
     auto numRows = result.numRows();
-    BOOST_LOG_TRIVIAL(info) << "num_rows = " << std::to_string(numRows);
+    log_.info() << "num_rows = " << std::to_string(numRows);
     do
     {
         hashes.push_back(result.getUInt256());
         if (--numRows == 0)
         {
-            BOOST_LOG_TRIVIAL(debug) << __func__ << " setting cursor";
+            log_.debug() << "Setting cursor";
             auto [lgrSeq, txnIdx] = result.getInt64Tuple();
             cursor = {
                 static_cast<std::uint32_t>(lgrSeq),
                 static_cast<std::uint32_t>(txnIdx)};
 
+            // Only modify if forward because forward query
+            // (selectAccountTxForward_) orders by ledger/tx sequence >= whereas
+            // reverse query (selectAccountTx_) orders by ledger/tx sequence <.
             if (forward)
                 ++cursor->transactionIndex;
         }
     } while (result.nextRow());
 
     auto txns = fetchTransactions(hashes, yield);
-    BOOST_LOG_TRIVIAL(debug) << __func__ << "txns = " << txns.size();
+    log_.debug() << "Txns = " << txns.size();
 
     if (txns.size() == limit)
     {
-        BOOST_LOG_TRIVIAL(debug) << __func__ << " returning cursor";
+        log_.debug() << "Returning cursor";
         return {txns, cursor};
     }
 
@@ -755,7 +784,7 @@ CassandraBackend::doFetchSuccessorKey(
     std::uint32_t const ledgerSequence,
     boost::asio::yield_context& yield) const
 {
-    BOOST_LOG_TRIVIAL(trace) << "Fetching from cassandra";
+    log_.trace() << "Fetching from cassandra";
     CassandraStatement statement{selectSuccessor_};
     statement.bindNextBytes(key);
     statement.bindNextInt(ledgerSequence);
@@ -764,7 +793,7 @@ CassandraBackend::doFetchSuccessorKey(
 
     if (!result)
     {
-        BOOST_LOG_TRIVIAL(debug) << __func__ << " - no rows";
+        log_.debug() << "No rows";
         return {};
     }
     auto next = result.getUInt256();
@@ -779,7 +808,7 @@ CassandraBackend::doFetchLedgerObject(
     std::uint32_t const sequence,
     boost::asio::yield_context& yield) const
 {
-    BOOST_LOG_TRIVIAL(trace) << "Fetching from cassandra";
+    log_.trace() << "Fetching from cassandra";
     CassandraStatement statement{selectObject_};
     statement.bindNextBytes(key);
     statement.bindNextInt(sequence);
@@ -788,7 +817,7 @@ CassandraBackend::doFetchLedgerObject(
 
     if (!result)
     {
-        BOOST_LOG_TRIVIAL(debug) << __func__ << " - no rows";
+        log_.debug() << "No rows";
         return {};
     }
     auto res = result.getBytes();
@@ -806,12 +835,13 @@ CassandraBackend::doFetchLedgerObjects(
     if (keys.size() == 0)
         return {};
 
+    numReadRequestsOutstanding_ += keys.size();
+
     handler_type handler(std::forward<decltype(yield)>(yield));
     result_type result(handler);
 
     std::size_t const numKeys = keys.size();
-    BOOST_LOG_TRIVIAL(trace)
-        << "Fetching " << numKeys << " records from Cassandra";
+    log_.trace() << "Fetching " << numKeys << " records from Cassandra";
     std::atomic_int numOutstanding = numKeys;
     std::vector<Blob> results{numKeys};
     std::vector<std::shared_ptr<ReadCallbackData<result_type>>> cbs;
@@ -832,6 +862,7 @@ CassandraBackend::doFetchLedgerObjects(
 
     // suspend the coroutine until completion handler is called.
     result.get();
+    numReadRequestsOutstanding_ -= keys.size();
 
     for (auto const& cb : cbs)
     {
@@ -839,8 +870,7 @@ CassandraBackend::doFetchLedgerObjects(
             throw DatabaseTimeout();
     }
 
-    BOOST_LOG_TRIVIAL(trace)
-        << "Fetched " << numKeys << " records from Cassandra";
+    log_.trace() << "Fetched " << numKeys << " records from Cassandra";
     return results;
 }
 
@@ -859,9 +889,7 @@ CassandraBackend::fetchLedgerDiff(
 
     if (!result)
     {
-        BOOST_LOG_TRIVIAL(error)
-            << __func__
-            << " - no rows . ledger = " << std::to_string(ledgerSequence);
+        log_.error() << "No rows. Ledger = " << std::to_string(ledgerSequence);
         return {};
     }
     std::vector<ripple::uint256> keys;
@@ -869,11 +897,12 @@ CassandraBackend::fetchLedgerDiff(
     {
         keys.push_back(result.getUInt256());
     } while (result.nextRow());
-    BOOST_LOG_TRIVIAL(debug)
-        << "Fetched " << keys.size() << " diff hashes from Cassandra in "
-        << std::chrono::duration_cast<std::chrono::milliseconds>(end - start)
-               .count()
-        << " milliseconds";
+    log_.debug() << "Fetched " << keys.size()
+                 << " diff hashes from Cassandra in "
+                 << std::chrono::duration_cast<std::chrono::milliseconds>(
+                        end - start)
+                        .count()
+                 << " milliseconds";
     auto objs = fetchLedgerObjects(keys, ledgerSequence, yield);
     std::vector<LedgerObject> results;
     std::transform(
@@ -940,12 +969,12 @@ CassandraBackend::doOnlineDelete(
                 cv));
 
             std::unique_lock<std::mutex> lck(mtx);
-            BOOST_LOG_TRIVIAL(trace) << __func__ << "Got the mutex";
+            log_.trace() << "Got the mutex";
             cv.wait(lck, [&numOutstanding, concurrentLimit]() {
                 return numOutstanding < concurrentLimit;
             });
         }
-        BOOST_LOG_TRIVIAL(debug) << __func__ << " fetched a page";
+        log_.debug() << "Fetched a page";
         cursor = curCursor;
         if (!cursor)
             break;
@@ -959,37 +988,30 @@ CassandraBackend::doOnlineDelete(
     return true;
 }
 
+bool
+CassandraBackend::isTooBusy() const
+{
+    return numReadRequestsOutstanding_ >= maxReadRequestsOutstanding;
+}
+
 void
 CassandraBackend::open(bool readOnly)
 {
-    auto getString = [this](std::string const& field) -> std::string {
-        if (config_.contains(field))
-        {
-            auto jsonStr = config_[field].as_string();
-            return {jsonStr.c_str(), jsonStr.size()};
-        }
-        return {""};
-    };
-    auto getInt = [this](std::string const& field) -> std::optional<int> {
-        if (config_.contains(field) && config_.at(field).is_int64())
-            return config_[field].as_int64();
-        return {};
-    };
     if (open_)
     {
         assert(false);
-        BOOST_LOG_TRIVIAL(error) << "database is already open";
+        log_.error() << "Database is already open";
         return;
     }
 
-    BOOST_LOG_TRIVIAL(info) << "Opening Cassandra Backend";
+    log_.info() << "Opening Cassandra Backend";
 
-    std::lock_guard<std::mutex> lock(mutex_);
     CassCluster* cluster = cass_cluster_new();
     if (!cluster)
         throw std::runtime_error("nodestore:: Failed to create CassCluster");
 
-    std::string secureConnectBundle = getString("secure_connect_bundle");
+    std::string secureConnectBundle =
+        config_.valueOr<std::string>("secure_connect_bundle", "");
 
     if (!secureConnectBundle.empty())
     {
@@ -998,9 +1020,9 @@ CassandraBackend::open(bool readOnly)
         if (cass_cluster_set_cloud_secure_connection_bundle(
                 cluster, secureConnectBundle.c_str()) != CASS_OK)
         {
-            BOOST_LOG_TRIVIAL(error) << "Unable to configure cloud using the "
-                                        "secure connection bundle: "
-                                     << secureConnectBundle;
+            log_.error() << "Unable to configure cloud using the "
+                            "secure connection bundle: "
+                         << secureConnectBundle;
             throw std::runtime_error(
                 "nodestore: Failed to connect using secure connection "
                 "bundle");
@@ -1009,12 +1031,9 @@ CassandraBackend::open(bool readOnly)
     }
     else
     {
-        std::string contact_points = getString("contact_points");
-        if (contact_points.empty())
-        {
-            throw std::runtime_error(
-                "nodestore: Missing contact_points in Cassandra config");
-        }
+        std::string contact_points = config_.valueOrThrow<std::string>(
+            "contact_points",
+            "nodestore: Missing contact_points in Cassandra config");
         CassError rc =
             cass_cluster_set_contact_points(cluster, contact_points.c_str());
         if (rc != CASS_OK)
@@ -1027,7 +1046,7 @@ CassandraBackend::open(bool readOnly)
             throw std::runtime_error(ss.str());
         }
 
-        auto port = getInt("port");
+        auto port = config_.maybeValue<int>("port");
         if (port)
         {
             rc = cass_cluster_set_port(cluster, *port);
@@ -1053,15 +1072,16 @@ CassandraBackend::open(bool readOnly)
         throw std::runtime_error(ss.str());
     }
 
-    std::string username = getString("username");
-    if (username.size())
+    auto username = config_.maybeValue<std::string>("username");
+    if (username)
     {
-        BOOST_LOG_TRIVIAL(debug) << "user = " << username.c_str();
+        log_.debug() << "user = " << *username;
+        auto password = config_.value<std::string>("password");
         cass_cluster_set_credentials(
-            cluster, username.c_str(), getString("password").c_str());
+            cluster, username->c_str(), password.c_str());
     }
-    int threads =
-        getInt("threads").value_or(std::thread::hardware_concurrency());
+    auto threads =
+        config_.valueOr<int>("threads", std::thread::hardware_concurrency());
 
     rc = cass_cluster_set_num_threads_io(cluster, threads);
     if (rc != CASS_OK)
@@ -1071,40 +1091,44 @@ CassandraBackend::open(bool readOnly)
            << ", result: " << rc << ", " << cass_error_desc(rc);
         throw std::runtime_error(ss.str());
     }
-    if (getInt("max_requests_outstanding"))
-        maxRequestsOutstanding = *getInt("max_requests_outstanding");
 
-    if (getInt("sync_interval"))
-        syncInterval_ = *getInt("sync_interval");
-    BOOST_LOG_TRIVIAL(info)
-        << __func__ << " sync interval is " << syncInterval_
-        << ". max requests outstanding is " << maxRequestsOutstanding;
+    maxWriteRequestsOutstanding = config_.valueOr<int>(
+        "max_write_requests_outstanding", maxWriteRequestsOutstanding);
+    maxReadRequestsOutstanding = config_.valueOr<int>(
+        "max_read_requests_outstanding", maxReadRequestsOutstanding);
+    syncInterval_ = config_.valueOr<int>("sync_interval", syncInterval_);
+
+    log_.info() << "Sync interval is " << syncInterval_
+                << ". max write requests outstanding is "
+                << maxWriteRequestsOutstanding
+                << ". max read requests outstanding is "
+                << maxReadRequestsOutstanding;
 
     cass_cluster_set_request_timeout(cluster, 10000);
 
     rc = cass_cluster_set_queue_size_io(
         cluster,
-        maxRequestsOutstanding);  // This number needs to scale w/ the
-                                  // number of request per sec
+        maxWriteRequestsOutstanding +
+            maxReadRequestsOutstanding);  // This number needs to scale w/ the
+                                          // number of request per sec
     if (rc != CASS_OK)
     {
         std::stringstream ss;
         ss << "nodestore: Error setting Cassandra max core connections per "
               "host"
            << ", result: " << rc << ", " << cass_error_desc(rc);
-        BOOST_LOG_TRIVIAL(error) << ss.str();
+        log_.error() << ss.str();
         throw std::runtime_error(ss.str());
     }
 
-    std::string certfile = getString("certfile");
-    if (certfile.size())
+    if (auto certfile = config_.maybeValue<std::string>("certfile"); certfile)
     {
         std::ifstream fileStream(
-            boost::filesystem::path(certfile).string(), std::ios::in);
+            boost::filesystem::path(*certfile).string(), std::ios::in);
         if (!fileStream)
         {
             std::stringstream ss;
-            ss << "opening config file " << certfile;
+            ss << "opening config file " << *certfile;
             throw std::system_error(errno, std::generic_category(), ss.str());
         }
         std::string cert(
@@ -1113,7 +1137,7 @@ CassandraBackend::open(bool readOnly)
         if (fileStream.bad())
         {
             std::stringstream ss;
-            ss << "reading config file " << certfile;
+            ss << "reading config file " << *certfile;
             throw std::system_error(errno, std::generic_category(), ss.str());
         }
 
@@ -1132,27 +1156,24 @@ CassandraBackend::open(bool readOnly)
         cass_ssl_free(context);
     }
 
-    std::string keyspace = getString("keyspace");
+    auto keyspace = config_.valueOr<std::string>("keyspace", "");
     if (keyspace.empty())
     {
-        BOOST_LOG_TRIVIAL(warning)
-            << "No keyspace specified. Using keyspace clio";
+        log_.warn() << "No keyspace specified. Using keyspace clio";
         keyspace = "clio";
     }
 
-    int rf = getInt("replication_factor") ? *getInt("replication_factor") : 3;
-
-    std::string tablePrefix = getString("table_prefix");
+    auto rf = config_.valueOr<int>("replication_factor", 3);
+    auto tablePrefix = config_.valueOr<std::string>("table_prefix", "");
     if (tablePrefix.empty())
     {
-        BOOST_LOG_TRIVIAL(warning) << "Table prefix is empty";
+        log_.warn() << "Table prefix is empty";
     }
 
     cass_cluster_set_connect_timeout(cluster, 10000);
 
-    int ttl = getInt("ttl") ? *getInt("ttl") * 2 : 0;
-    BOOST_LOG_TRIVIAL(info)
-        << __func__ << " setting ttl to " << std::to_string(ttl);
+    auto ttl = ttl_ * 2;
+    log_.info() << "Setting ttl to " << std::to_string(ttl);
 
     auto executeSimpleStatement = [this](std::string const& query) {
         CassStatement* statement = makeStatement(query.c_str(), 0);
@@ -1165,7 +1186,7 @@ CassandraBackend::open(bool readOnly)
             std::stringstream ss;
             ss << "nodestore: Error executing simple statement: " << rc << ", "
                << cass_error_desc(rc) << " - " << query;
-            BOOST_LOG_TRIVIAL(error) << ss.str();
+            log_.error() << ss.str();
             return false;
         }
         return true;
@@ -1188,7 +1209,7 @@ CassandraBackend::open(bool readOnly)
             ss << "nodestore: Error connecting Cassandra session keyspace: "
                << rc << ", " << cass_error_desc(rc)
                << ", trying to create it ourselves";
-            BOOST_LOG_TRIVIAL(error) << ss.str();
+            log_.error() << ss.str();
             // if the keyspace doesn't exist, try to create it
             session_.reset(cass_session_new());
             fut = cass_session_connect(session_.get(), cluster);
@@ -1199,7 +1220,7 @@ CassandraBackend::open(bool readOnly)
                 std::stringstream ss;
                 ss << "nodestore: Error connecting Cassandra session at all: "
                    << rc << ", " << cass_error_desc(rc);
-                BOOST_LOG_TRIVIAL(error) << ss.str();
+                log_.error() << ss.str();
             }
             else
             {
@@ -1622,6 +1643,6 @@ CassandraBackend::open(bool readOnly)
 
     open_ = true;
 
-    BOOST_LOG_TRIVIAL(info) << "Opened CassandraBackend successfully";
+    log_.info() << "Opened CassandraBackend successfully";
 }
 }  // namespace Backend

src/backend/CassandraBackend.h

@@ -1,28 +1,52 @@
-#ifndef RIPPLE_APP_REPORTING_CASSANDRABACKEND_H_INCLUDED
-#define RIPPLE_APP_REPORTING_CASSANDRABACKEND_H_INCLUDED
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
 
 #include <ripple/basics/base_uint.h>
+#include <backend/BackendInterface.h>
+#include <backend/DBHelpers.h>
+#include <log/Logger.h>
+
+#include <cassandra.h>
+
 #include <boost/asio.hpp>
 #include <boost/asio/async_result.hpp>
 #include <boost/asio/spawn.hpp>
 #include <boost/filesystem.hpp>
 #include <boost/json.hpp>
-#include <boost/log/trivial.hpp>
+
 #include <atomic>
-#include <backend/BackendInterface.h>
-#include <backend/DBHelpers.h>
-#include <cassandra.h>
 #include <cstddef>
 #include <iostream>
 #include <memory>
 #include <mutex>
 #include <thread>
 
+#include <config/Config.h>
+
 namespace Backend {
 
 class CassandraPreparedStatement
 {
 private:
+    clio::Logger log_{"Backend"};
     CassPrepared const* prepared_ = nullptr;
 
 public:
@@ -63,7 +87,7 @@ public:
             std::stringstream ss;
             ss << "nodestore: Error preparing statement : " << rc << ", "
                << cass_error_desc(rc) << ". query : " << query;
-            BOOST_LOG_TRIVIAL(error) << ss.str();
+            log_.error() << ss.str();
         }
         cass_future_free(prepareFuture);
         return rc == CASS_OK;
@@ -71,7 +95,7 @@ public:
 
     ~CassandraPreparedStatement()
     {
-        BOOST_LOG_TRIVIAL(trace) << __func__;
+        log_.trace() << "called";
         if (prepared_)
         {
             cass_prepared_free(prepared_);
@@ -84,6 +108,7 @@ class CassandraStatement
 {
     CassStatement* statement_ = nullptr;
     size_t curBindingIndex_ = 0;
+    clio::Logger log_{"Backend"};
 
 public:
     CassandraStatement(CassandraPreparedStatement const& prepared)
@@ -121,7 +146,7 @@ public:
             std::stringstream ss;
             ss << "Error binding boolean to statement: " << rc << ", "
                << cass_error_desc(rc);
-            BOOST_LOG_TRIVIAL(error) << __func__ << " : " << ss.str();
+            log_.error() << ss.str();
             throw std::runtime_error(ss.str());
         }
         curBindingIndex_++;
@@ -177,7 +202,7 @@ public:
             std::stringstream ss;
             ss << "Error binding bytes to statement: " << rc << ", "
                << cass_error_desc(rc);
-            BOOST_LOG_TRIVIAL(error) << __func__ << " : " << ss.str();
+            log_.error() << ss.str();
             throw std::runtime_error(ss.str());
         }
         curBindingIndex_++;
@@ -189,8 +214,8 @@ public:
         if (!statement_)
             throw std::runtime_error(
                 "CassandraStatement::bindNextUInt - statement_ is null");
-        BOOST_LOG_TRIVIAL(trace)
-            << std::to_string(curBindingIndex_) << " " << std::to_string(value);
+        log_.trace() << std::to_string(curBindingIndex_) << " "
+                     << std::to_string(value);
         CassError rc =
             cass_statement_bind_int32(statement_, curBindingIndex_, value);
         if (rc != CASS_OK)
@@ -198,7 +223,7 @@ public:
             std::stringstream ss;
             ss << "Error binding uint to statement: " << rc << ", "
                << cass_error_desc(rc);
-            BOOST_LOG_TRIVIAL(error) << __func__ << " : " << ss.str();
+            log_.error() << ss.str();
             throw std::runtime_error(ss.str());
         }
         curBindingIndex_++;
@@ -223,7 +248,7 @@ public:
             std::stringstream ss;
             ss << "Error binding int to statement: " << rc << ", "
                << cass_error_desc(rc);
-            BOOST_LOG_TRIVIAL(error) << __func__ << " : " << ss.str();
+            log_.error() << ss.str();
             throw std::runtime_error(ss.str());
         }
         curBindingIndex_++;
@@ -239,7 +264,7 @@ public:
             std::stringstream ss;
             ss << "Error binding int to tuple: " << rc << ", "
                << cass_error_desc(rc);
-            BOOST_LOG_TRIVIAL(error) << __func__ << " : " << ss.str();
+            log_.error() << ss.str();
             throw std::runtime_error(ss.str());
         }
         rc = cass_tuple_set_int64(tuple, 1, second);
@@ -248,7 +273,7 @@ public:
             std::stringstream ss;
             ss << "Error binding int to tuple: " << rc << ", "
                << cass_error_desc(rc);
-            BOOST_LOG_TRIVIAL(error) << __func__ << " : " << ss.str();
+            log_.error() << ss.str();
             throw std::runtime_error(ss.str());
         }
         rc = cass_statement_bind_tuple(statement_, curBindingIndex_, tuple);
@@ -257,7 +282,7 @@ public:
             std::stringstream ss;
             ss << "Error binding tuple to statement: " << rc << ", "
                << cass_error_desc(rc);
-            BOOST_LOG_TRIVIAL(error) << __func__ << " : " << ss.str();
+            log_.error() << ss.str();
             throw std::runtime_error(ss.str());
         }
         cass_tuple_free(tuple);
@@ -273,6 +298,7 @@ public:
 
 class CassandraResult
 {
+    clio::Logger log_{"Backend"};
     CassResult const* result_ = nullptr;
     CassRow const* row_ = nullptr;
     CassIterator* iter_ = nullptr;
@@ -363,7 +389,7 @@ public:
             std::stringstream msg;
             msg << "CassandraResult::getBytes - error getting value: " << rc
                 << ", " << cass_error_desc(rc);
-            BOOST_LOG_TRIVIAL(error) << msg.str();
+            log_.error() << msg.str();
             throw std::runtime_error(msg.str());
         }
         curGetIndex_++;
@@ -384,7 +410,7 @@ public:
             std::stringstream msg;
             msg << "CassandraResult::getuint256 - error getting value: " << rc
                 << ", " << cass_error_desc(rc);
-            BOOST_LOG_TRIVIAL(error) << msg.str();
+            log_.error() << msg.str();
             throw std::runtime_error(msg.str());
         }
         curGetIndex_++;
@@ -404,7 +430,7 @@ public:
             std::stringstream msg;
             msg << "CassandraResult::getInt64 - error getting value: " << rc
                 << ", " << cass_error_desc(rc);
-            BOOST_LOG_TRIVIAL(error) << msg.str();
+            log_.error() << msg.str();
             throw std::runtime_error(msg.str());
         }
         ++curGetIndex_;
@@ -488,10 +514,9 @@ public:
     {
         if (!row_)
         {
-            std::stringstream msg;
-            msg << __func__ << " - no result";
-            BOOST_LOG_TRIVIAL(error) << msg.str();
-            throw std::runtime_error(msg.str());
+            std::string msg{"No result"};
+            log_.error() << msg;
+            throw std::runtime_error(msg);
         }
         cass_bool_t val;
         CassError rc =
@@ -499,9 +524,8 @@ public:
         if (rc != CASS_OK)
         {
             std::stringstream msg;
-            msg << __func__ << " - error getting value: " << rc << ", "
-                << cass_error_desc(rc);
-            BOOST_LOG_TRIVIAL(error) << msg.str();
+            msg << "Error getting value: " << rc << ", " << cass_error_desc(rc);
+            log_.error() << msg.str();
             throw std::runtime_error(msg.str());
         }
         ++curGetIndex_;
@@ -516,6 +540,7 @@ public:
             cass_iterator_free(iter_);
     }
 };
+
 inline bool
 isTimeout(CassError rc)
 {
@@ -595,6 +620,7 @@ private:
         return ret;
     }
 
+    clio::Logger log_{"Backend"};
     std::atomic<bool> open_{false};
 
     std::unique_ptr<CassSession, void (*)(CassSession*)> session_{
@@ -645,13 +671,18 @@ private:
     uint32_t syncInterval_ = 1;
     uint32_t lastSync_ = 0;
 
-    // maximum number of concurrent in flight requests. New requests will wait
-    // for earlier requests to finish if this limit is exceeded
-    std::uint32_t maxRequestsOutstanding = 10000;
-    mutable std::atomic_uint32_t numRequestsOutstanding_ = 0;
+    // maximum number of concurrent in flight write requests. New requests will
+    // wait for earlier requests to finish if this limit is exceeded
+    std::uint32_t maxWriteRequestsOutstanding = 10000;
+    mutable std::atomic_uint32_t numWriteRequestsOutstanding_ = 0;
+
+    // maximum number of concurrent in flight read requests. isTooBusy() will
+    // return true if the number of in flight read requests exceeds this limit
+    std::uint32_t maxReadRequestsOutstanding = 100000;
+    mutable std::atomic_uint32_t numReadRequestsOutstanding_ = 0;
 
     // mutex and condition_variable to limit the number of concurrent in flight
-    // requests
+    // write requests
     mutable std::mutex throttleMutex_;
     mutable std::condition_variable throttleCv_;
 
@@ -665,15 +696,17 @@ private:
     std::optional<boost::asio::io_context::work> work_;
     std::thread ioThread_;
 
-    boost::json::object config_;
+    clio::Config config_;
+    uint32_t ttl_ = 0;
 
     mutable std::uint32_t ledgerSequence_ = 0;
 
 public:
     CassandraBackend(
         boost::asio::io_context& ioc,
-        boost::json::object const& config)
-        : BackendInterface(config), config_(config)
+        clio::Config const& config,
+        uint32_t ttl)
+        : BackendInterface(config), config_(config), ttl_(ttl)
     {
         work_.emplace(ioContext_);
         ioThread_ = std::thread([this]() { ioContext_.run(); });
@@ -742,13 +775,11 @@ public:
         statement.bindNextInt(ledgerSequence_ - 1);
         if (!executeSyncUpdate(statement))
         {
-            BOOST_LOG_TRIVIAL(warning)
-                << __func__ << " Update failed for ledger "
-                << std::to_string(ledgerSequence_) << ". Returning";
+            log_.warn() << "Update failed for ledger "
+                        << std::to_string(ledgerSequence_) << ". Returning";
             return false;
         }
-        BOOST_LOG_TRIVIAL(info) << __func__ << " Committed ledger "
-                                << std::to_string(ledgerSequence_);
+        log_.info() << "Committed ledger " << std::to_string(ledgerSequence_);
         return true;
     }
 
@@ -782,22 +813,20 @@ public:
                 statement.bindNextInt(lastSync_);
             if (!executeSyncUpdate(statement))
             {
-                BOOST_LOG_TRIVIAL(warning)
-                    << __func__ << " Update failed for ledger "
-                    << std::to_string(ledgerSequence_) << ". Returning";
+                log_.warn() << "Update failed for ledger "
+                            << std::to_string(ledgerSequence_) << ". Returning";
                 return false;
             }
-            BOOST_LOG_TRIVIAL(info) << __func__ << " Committed ledger "
-                                    << std::to_string(ledgerSequence_);
+            log_.info() << "Committed ledger "
+                        << std::to_string(ledgerSequence_);
             lastSync_ = ledgerSequence_;
         }
         else
         {
-            BOOST_LOG_TRIVIAL(info)
-                << __func__ << " Skipping commit. sync interval is "
-                << std::to_string(syncInterval_) << " - last sync is "
-                << std::to_string(lastSync_) << " - ledger sequence is "
-                << std::to_string(ledgerSequence_);
+            log_.info() << "Skipping commit. sync interval is "
+                        << std::to_string(syncInterval_) << " - last sync is "
+                        << std::to_string(lastSync_) << " - ledger sequence is "
+                        << std::to_string(ledgerSequence_);
         }
         return true;
     }
@@ -817,12 +846,12 @@ public:
     std::optional<std::uint32_t>
     fetchLatestLedgerSequence(boost::asio::yield_context& yield) const override
     {
-        BOOST_LOG_TRIVIAL(trace) << __func__;
+        log_.trace() << "called";
         CassandraStatement statement{selectLatestLedger_};
         CassandraResult result = executeAsyncRead(statement, yield);
         if (!result.hasResult())
         {
-            BOOST_LOG_TRIVIAL(error)
+            log_.error()
                 << "CassandraBackend::fetchLatestLedgerSequence - no rows";
             return {};
         }
@@ -834,13 +863,13 @@ public:
         std::uint32_t const sequence,
         boost::asio::yield_context& yield) const override
     {
-        BOOST_LOG_TRIVIAL(trace) << __func__;
+        log_.trace() << "called";
         CassandraStatement statement{selectLedgerBySeq_};
         statement.bindNextInt(sequence);
         CassandraResult result = executeAsyncRead(statement, yield);
         if (!result)
         {
-            BOOST_LOG_TRIVIAL(error) << __func__ << " - no rows";
+            log_.error() << "No rows";
             return {};
         }
         std::vector<unsigned char> header = result.getBytes();
@@ -860,7 +889,7 @@ public:
 
         if (!result.hasResult())
         {
-            BOOST_LOG_TRIVIAL(debug) << __func__ << " - no rows returned";
+            log_.debug() << "No rows returned";
             return {};
         }
 
@@ -907,7 +936,7 @@ public:
     std::optional<int64_t>
     getToken(void const* key, boost::asio::yield_context& yield) const
     {
-        BOOST_LOG_TRIVIAL(trace) << "Fetching from cassandra";
+        log_.trace() << "Fetching from cassandra";
         CassandraStatement statement{getToken_};
         statement.bindNextBytes(key, 32);
 
@@ -915,7 +944,7 @@ public:
 
         if (!result)
         {
-            BOOST_LOG_TRIVIAL(error) << __func__ << " - no rows";
+            log_.error() << "No rows";
             return {};
         }
         int64_t token = result.getInt64();
@@ -930,14 +959,14 @@ public:
         ripple::uint256 const& hash,
         boost::asio::yield_context& yield) const override
     {
-        BOOST_LOG_TRIVIAL(trace) << __func__;
+        log_.trace() << "called";
         CassandraStatement statement{selectTransaction_};
         statement.bindNextBytes(hash);
         CassandraResult result = executeAsyncRead(statement, yield);
 
         if (!result)
         {
-            BOOST_LOG_TRIVIAL(error) << __func__ << " - no rows";
+            log_.error() << "No rows";
             return {};
         }
         return {
@@ -1017,33 +1046,34 @@ public:
         std::uint32_t const numLedgersToKeep,
         boost::asio::yield_context& yield) const override;
 
+    bool
+    isTooBusy() const override;
+
     inline void
-    incremementOutstandingRequestCount() const
+    incrementOutstandingRequestCount() const
     {
         {
             std::unique_lock<std::mutex> lck(throttleMutex_);
             if (!canAddRequest())
             {
-                BOOST_LOG_TRIVIAL(info)
-                    << __func__ << " : "
-                    << "Max outstanding requests reached. "
-                    << "Waiting for other requests to finish";
+                log_.debug() << "Max outstanding requests reached. "
+                             << "Waiting for other requests to finish";
                 throttleCv_.wait(lck, [this]() { return canAddRequest(); });
             }
         }
-        ++numRequestsOutstanding_;
+        ++numWriteRequestsOutstanding_;
     }
 
     inline void
     decrementOutstandingRequestCount() const
     {
         // sanity check
-        if (numRequestsOutstanding_ == 0)
+        if (numWriteRequestsOutstanding_ == 0)
         {
             assert(false);
             throw std::runtime_error("decrementing num outstanding below 0");
         }
-        size_t cur = (--numRequestsOutstanding_);
+        size_t cur = (--numWriteRequestsOutstanding_);
         {
             // mutex lock required to prevent race condition around spurious
             // wakeup
@@ -1062,13 +1092,13 @@ public:
     inline bool
     canAddRequest() const
     {
-        return numRequestsOutstanding_ < maxRequestsOutstanding;
+        return numWriteRequestsOutstanding_ < maxWriteRequestsOutstanding;
     }
 
     inline bool
     finishedAllRequests() const
     {
-        return numRequestsOutstanding_ == 0;
+        return numWriteRequestsOutstanding_ == 0;
     }
 
     void
@@ -1101,7 +1131,7 @@ public:
         bool isRetry) const
     {
         if (!isRetry)
-            incremementOutstandingRequestCount();
+            incrementOutstandingRequestCount();
         executeAsyncHelper(statement, callback, callbackData);
     }
 
@@ -1130,7 +1160,7 @@ public:
                 ss << "Cassandra sync write error";
                 ss << ", retrying";
                 ss << ": " << cass_error_desc(rc);
-                BOOST_LOG_TRIVIAL(warning) << ss.str();
+                log_.warn() << ss.str();
                 std::this_thread::sleep_for(std::chrono::milliseconds(5));
             }
         } while (rc != CASS_OK);
@@ -1154,7 +1184,7 @@ public:
                 ss << "Cassandra sync update error";
                 ss << ", retrying";
                 ss << ": " << cass_error_desc(rc);
-                BOOST_LOG_TRIVIAL(warning) << ss.str();
+                log_.warn() << ss.str();
                 std::this_thread::sleep_for(std::chrono::milliseconds(5));
             }
         } while (rc != CASS_OK);
@@ -1164,7 +1194,7 @@ public:
         CassRow const* row = cass_result_first_row(res);
         if (!row)
         {
-            BOOST_LOG_TRIVIAL(error) << "executeSyncUpdate - no rows";
+            log_.error() << "executeSyncUpdate - no rows";
             cass_result_free(res);
             return false;
         }
@@ -1173,16 +1203,14 @@ public:
         if (rc != CASS_OK)
         {
             cass_result_free(res);
-            BOOST_LOG_TRIVIAL(error)
-                << "executeSyncUpdate - error getting result " << rc << ", "
-                << cass_error_desc(rc);
+            log_.error() << "executeSyncUpdate - error getting result " << rc
+                         << ", " << cass_error_desc(rc);
             return false;
         }
         cass_result_free(res);
         if (success != cass_true && timedOut)
         {
-            BOOST_LOG_TRIVIAL(warning)
-                << __func__ << " Update failed, but timedOut is true";
+            log_.warn() << "Update failed, but timedOut is true";
             // if there was a timeout, the update may have succeeded in the
             // background on the first attempt. To determine if this happened,
             // we query the range from the db, making sure the range is what
@@ -1209,22 +1237,23 @@ public:
         CassError rc;
         do
         {
+            ++numReadRequestsOutstanding_;
             fut = cass_session_execute(session_.get(), statement.get());
 
             boost::system::error_code ec;
             rc = cass_future_error_code(fut, yield[ec]);
+            --numReadRequestsOutstanding_;
 
             if (ec)
             {
-                BOOST_LOG_TRIVIAL(error)
-                    << "Cannot read async cass_future_error_code";
+                log_.error() << "Cannot read async cass_future_error_code";
             }
             if (rc != CASS_OK)
             {
                 std::stringstream ss;
                 ss << "Cassandra executeAsyncRead error";
                 ss << ": " << cass_error_desc(rc);
-                BOOST_LOG_TRIVIAL(error) << ss.str();
+                log_.error() << ss.str();
             }
             if (isTimeout(rc))
             {
@@ -1247,4 +1276,3 @@ public:
 };
 
 }  // namespace Backend
-#endif

src/backend/DBHelpers.h

@@ -1,12 +1,33 @@
-#ifndef CLIO_BACKEND_DBHELPERS_H_INCLUDED
-#define CLIO_BACKEND_DBHELPERS_H_INCLUDED
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
 
 #include <ripple/basics/Log.h>
+#include <ripple/basics/StringUtilities.h>
+#include <ripple/ledger/ReadView.h>
 #include <ripple/protocol/SField.h>
 #include <ripple/protocol/STAccount.h>
 #include <ripple/protocol/TxMeta.h>
+
 #include <boost/container/flat_set.hpp>
-#include <backend/Pg.h>
+
 #include <backend/Types.h>
 
 /// Struct used to keep track of what to write to
@@ -178,4 +199,3 @@ uint256ToString(ripple::uint256 const& uint)
 }
 
 static constexpr std::uint32_t rippleEpochStart = 946684800;
-#endif

src/backend/Pg.h

@@ -1,564 +0,0 @@
-#ifndef RIPPLE_CORE_PG_H_INCLUDED
-#define RIPPLE_CORE_PG_H_INCLUDED
-
-#include <ripple/basics/StringUtilities.h>
-#include <ripple/basics/chrono.h>
-#include <ripple/ledger/ReadView.h>
-#include <boost/asio/io_context.hpp>
-#include <boost/asio/ip/tcp.hpp>
-#include <boost/asio/spawn.hpp>
-#include <boost/icl/closed_interval.hpp>
-#include <boost/json.hpp>
-#include <boost/lexical_cast.hpp>
-#include <boost/log/trivial.hpp>
-#include <atomic>
-#include <chrono>
-#include <condition_variable>
-#include <functional>
-#include <libpq-fe.h>
-#include <memory>
-#include <mutex>
-#include <optional>
-#include <string>
-#include <string_view>
-#include <utility>
-#include <vector>
-
-// These postgres structs must be freed only by the postgres API.
-using pg_result_type = std::unique_ptr<PGresult, void (*)(PGresult*)>;
-using pg_connection_type = std::unique_ptr<PGconn, void (*)(PGconn*)>;
-using asio_socket_type = std::unique_ptr<
-    boost::asio::ip::tcp::socket,
-    void (*)(boost::asio::ip::tcp::socket*)>;
-
-/** first: command
- * second: parameter values
- *
- * The 2nd member takes an optional string to
- * distinguish between NULL parameters and empty strings. An empty
- * item corresponds to a NULL parameter.
- *
- * Postgres reads each parameter as a c-string, regardless of actual type.
- * Binary types (bytea) need to be converted to hex and prepended with
- * \x ("\\x").
- */
-using pg_params =
-    std::pair<char const*, std::vector<std::optional<std::string>>>;
-
-/** Parameter values for pg API. */
-using pg_formatted_params = std::vector<char const*>;
-
-/** Parameters for managing postgres connections. */
-struct PgConfig
-{
-    /** Maximum connections allowed to db. */
-    std::size_t max_connections{1000};
-    /** Close idle connections past this duration. */
-    std::chrono::seconds timeout{600};
-
-    /** Index of DB connection parameter names. */
-    std::vector<char const*> keywordsIdx;
-    /** DB connection parameter names. */
-    std::vector<std::string> keywords;
-    /** Index of DB connection parameter values. */
-    std::vector<char const*> valuesIdx;
-    /** DB connection parameter values. */
-    std::vector<std::string> values;
-};
-
-//-----------------------------------------------------------------------------
-
-/** Class that operates on postgres query results.
- *
- * The functions that return results do not check first whether the
- * expected results are actually there. Therefore, the caller first needs
- * to check whether or not a valid response was returned using the operator
- * bool() overload. If number of tuples or fields are unknown, then check
- * those. Each result field should be checked for null before attempting
- * to return results. Finally, the caller must know the type of the field
- * before calling the corresponding function to return a field. Postgres
- * internally stores each result field as null-terminated strings.
- */
-class PgResult
-{
-    // The result object must be freed using the libpq API PQclear() call.
-    pg_result_type result_{nullptr, [](PGresult* result) { PQclear(result); }};
-    std::optional<std::pair<ExecStatusType, std::string>> error_;
-
-public:
-    /** Constructor for when the process is stopping.
-     *
-     */
-    PgResult()
-    {
-    }
-
-    /** Constructor for successful query results.
-     *
-     * @param result Query result.
-     */
-    explicit PgResult(pg_result_type&& result) : result_(std::move(result))
-    {
-    }
-
-    /** Constructor for failed query results.
-     *
-     * @param result Query result that contains error information.
-     * @param conn Postgres connection that contains error information.
-     */
-    PgResult(PGresult* result, PGconn* conn)
-        : error_({PQresultStatus(result), PQerrorMessage(conn)})
-    {
-    }
-
-    /** Return field as a null-terminated string pointer.
-     *
-     * Note that this function does not guarantee that the result struct
-     * exists, or that the row and fields exist, or that the field is
-     * not null.
-     *
-     * @param ntuple Row number.
-     * @param nfield Field number.
-     * @return Field contents.
-     */
-    char const*
-    c_str(int ntuple = 0, int nfield = 0) const
-    {
-        return PQgetvalue(result_.get(), ntuple, nfield);
-    }
-
-    std::vector<unsigned char>
-    asUnHexedBlob(int ntuple = 0, int nfield = 0) const
-    {
-        std::string_view view{c_str(ntuple, nfield) + 2};
-        auto res = ripple::strUnHex(view.size(), view.cbegin(), view.cend());
-        if (res)
-            return *res;
-        return {};
-    }
-
-    ripple::uint256
-    asUInt256(int ntuple = 0, int nfield = 0) const
-    {
-        ripple::uint256 val;
-        if (!val.parseHex(c_str(ntuple, nfield) + 2))
-            throw std::runtime_error("Pg - failed to parse hex into uint256");
-        return val;
-    }
-
-    /** Return field as equivalent to Postgres' INT type (32 bit signed).
-     *
-     * Note that this function does not guarantee that the result struct
-     * exists, or that the row and fields exist, or that the field is
-     * not null, or that the type is that requested.
-
-     * @param ntuple Row number.
-     * @param nfield Field number.
-     * @return Field contents.
-     */
-    std::int32_t
-    asInt(int ntuple = 0, int nfield = 0) const
-    {
-        return boost::lexical_cast<std::int32_t>(
-            PQgetvalue(result_.get(), ntuple, nfield));
-    }
-
-    /** Return field as equivalent to Postgres' BIGINT type (64 bit signed).
-     *
-     * Note that this function does not guarantee that the result struct
-     * exists, or that the row and fields exist, or that the field is
-     * not null, or that the type is that requested.
-
-     * @param ntuple Row number.
-     * @param nfield Field number.
-     * @return Field contents.
-     */
-    std::int64_t
-    asBigInt(int ntuple = 0, int nfield = 0) const
-    {
-        return boost::lexical_cast<std::int64_t>(
-            PQgetvalue(result_.get(), ntuple, nfield));
-    }
-
-    /** Returns whether the field is NULL or not.
-     *
-     * Note that this function does not guarantee that the result struct
-     * exists, or that the row and fields exist.
-     *
-     * @param ntuple Row number.
-     * @param nfield Field number.
-     * @return Whether field is NULL.
-     */
-    bool
-    isNull(int ntuple = 0, int nfield = 0) const
-    {
-        return PQgetisnull(result_.get(), ntuple, nfield);
-    }
-
-    /** Check whether a valid response occurred.
-     *
-     * @return Whether or not the query returned a valid response.
-     */
-    operator bool() const
-    {
-        return result_ != nullptr;
-    }
-
-    /** Message describing the query results suitable for diagnostics.
-     *
-     * If error, then the postgres error type and message are returned.
-     * Otherwise, "ok"
-     *
-     * @return Query result message.
-     */
-    std::string
-    msg() const;
-
-    /** Get number of rows in result.
-     *
-     * Note that this function does not guarantee that the result struct
-     * exists.
-     *
-     * @return Number of result rows.
-     */
-    int
-    ntuples() const
-    {
-        return PQntuples(result_.get());
-    }
-
-    /** Get number of fields in result.
-     *
-     * Note that this function does not guarantee that the result struct
-     * exists.
-     *
-     * @return Number of result fields.
-     */
-    int
-    nfields() const
-    {
-        return PQnfields(result_.get());
-    }
-
-    /** Return result status of the command.
-     *
-     * Note that this function does not guarantee that the result struct
-     * exists.
-     *
-     * @return
-     */
-    ExecStatusType
-    status() const
-    {
-        return PQresultStatus(result_.get());
-    }
-};
-
-/* Class that contains and operates upon a postgres connection. */
-class Pg
-{
-    friend class PgPool;
-    friend class PgQuery;
-
-    PgConfig const& config_;
-    boost::asio::io_context::strand strand_;
-
-    asio_socket_type socket_{nullptr, [](boost::asio::ip::tcp::socket*) {}};
-
-    // The connection object must be freed using the libpq API PQfinish() call.
-    pg_connection_type conn_{nullptr, [](PGconn* conn) { PQfinish(conn); }};
-
-    inline asio_socket_type
-    getSocket(boost::asio::yield_context& strand);
-
-    inline PgResult
-    waitForStatus(boost::asio::yield_context& yield, ExecStatusType expected);
-
-    inline void
-    flush(boost::asio::yield_context& yield);
-
-    /** Clear results from the connection.
-     *
-     * Results from previous commands must be cleared before new commands
-     * can be processed. This function should be called on connections
-     * that weren't processed completely before being reused, such as
-     * when being checked-in.
-     *
-     * @return whether or not connection still exists.
-     */
-    bool
-    clear();
-
-    /** Connect to postgres.
-     *
-     * Idempotently connects to postgres by first checking whether an
-     * existing connection is already present. If connection is not present
-     * or in an errored state, reconnects to the database.
-     */
-    void
-    connect(boost::asio::yield_context& yield);
-
-    /** Disconnect from postgres. */
-    void
-    disconnect()
-    {
-        conn_.reset();
-        socket_.reset();
-    }
-
-    /** Execute postgres query.
-     *
-     * If parameters are included, then the command should contain only a
-     * single SQL statement. If no parameters, then multiple SQL statements
-     * delimited by semi-colons can be processed. The response is from
-     * the last command executed.
-     *
-     * @param command postgres API command string.
-     * @param nParams postgres API number of parameters.
-     * @param values postgres API array of parameter.
-     * @return Query result object.
-     */
-    PgResult
-    query(
-        char const* command,
-        std::size_t const nParams,
-        char const* const* values,
-        boost::asio::yield_context& yield);
-
-    /** Execute postgres query with no parameters.
-     *
-     * @param command Query string.
-     * @return Query result object;
-     */
-    PgResult
-    query(char const* command, boost::asio::yield_context& yield)
-    {
-        return query(command, 0, nullptr, yield);
-    }
-
-    /** Execute postgres query with parameters.
-     *
-     * @param dbParams Database command and parameter values.
-     * @return Query result object.
-     */
-    PgResult
-    query(pg_params const& dbParams, boost::asio::yield_context& yield);
-
-    /** Insert multiple records into a table using Postgres' bulk COPY.
-     *
-     * Throws upon error.
-     *
-     * @param table Name of table for import.
-     * @param records Records in the COPY IN format.
-     */
-    void
-    bulkInsert(
-        char const* table,
-        std::string const& records,
-        boost::asio::yield_context& yield);
-
-public:
-    /** Constructor for Pg class.
-     *
-     * @param config Config parameters.
-     * @param j Logger object.
-     */
-    Pg(PgConfig const& config, boost::asio::io_context& ctx)
-        : config_(config), strand_(ctx)
-    {
-    }
-};
-
-//-----------------------------------------------------------------------------
-
-/** Database connection pool.
- *
- * Allow re-use of postgres connections. Postgres connections are created
- * as needed until configurable limit is reached. After use, each connection
- * is placed in a container ordered by time of use. Each request for
- * a connection grabs the most recently used connection from the container.
- * If none are available, a new connection is used (up to configured limit).
- * Idle connections are destroyed periodically after configurable
- * timeout duration.
- *
- * This should be stored as a shared pointer so PgQuery objects can safely
- * outlive it.
- */
-class PgPool
-{
-    friend class PgQuery;
-
-    using clock_type = std::chrono::steady_clock;
-
-    boost::asio::io_context& ioc_;
-    PgConfig config_;
-    std::mutex mutex_;
-    std::condition_variable cond_;
-    std::size_t connections_{};
-    bool stop_{false};
-
-    /** Idle database connections ordered by timestamp to allow timing out. */
-    std::multimap<std::chrono::time_point<clock_type>, std::unique_ptr<Pg>>
-        idle_;
-
-    /** Get a postgres connection object.
-     *
-     * Return the most recent idle connection in the pool, if available.
-     * Otherwise, return a new connection unless we're at the threshold.
-     * If so, then wait until a connection becomes available.
-     *
-     * @return Postgres object.
-     */
-    std::unique_ptr<Pg>
-    checkout();
-
-    /** Return a postgres object to the pool for reuse.
-     *
-     * If connection is healthy, place in pool for reuse. After calling this,
-     * the container no longer have a connection unless checkout() is called.
-     *
-     * @param pg Pg object.
-     */
-    void
-    checkin(std::unique_ptr<Pg>& pg);
-
-public:
-    /** Connection pool constructor.
-     *
-     * @param pgConfig Postgres config.
-     * @param j Logger object.
-     * @param parent Stoppable parent.
-     */
-    PgPool(boost::asio::io_context& ioc, boost::json::object const& config);
-
-    ~PgPool()
-    {
-        onStop();
-    }
-
-    PgConfig&
-    config()
-    {
-        return config_;
-    }
-
-    /** Initiate idle connection timer.
-     *
-     * The PgPool object needs to be fully constructed to support asynchronous
-     * operations.
-     */
-    void
-    setup();
-
-    /** Prepare for process shutdown. (Stoppable) */
-    void
-    onStop();
-};
-
-//-----------------------------------------------------------------------------
-
-/** Class to query postgres.
- *
- * This class should be used by functions outside of this
- * compilation unit for querying postgres. It automatically acquires and
- * relinquishes a database connection to handle each query.
- */
-class PgQuery
-{
-private:
-    std::shared_ptr<PgPool> pool_;
-    std::unique_ptr<Pg> pg_;
-
-public:
-    PgQuery() = delete;
-
-    PgQuery(std::shared_ptr<PgPool> const& pool)
-        : pool_(pool), pg_(pool->checkout())
-    {
-    }
-
-    ~PgQuery()
-    {
-        pool_->checkin(pg_);
-    }
-
-    // TODO. add sendQuery and getResult, for sending the query and getting the
-    // result asynchronously. This could be useful for sending a bunch of
-    // requests concurrently
-
-    /** Execute postgres query with parameters.
-     *
-     * @param dbParams Database command with parameters.
-     * @return Result of query, including errors.
-     */
-    PgResult
-    operator()(pg_params const& dbParams, boost::asio::yield_context& yield)
-    {
-        if (!pg_)  // It means we're stopping. Return empty result.
-            return PgResult();
-        return pg_->query(dbParams, yield);
-    }
-
-    /** Execute postgres query with only command statement.
-     *
-     * @param command Command statement.
-     * @return Result of query, including errors.
-     */
-    PgResult
-    operator()(char const* command, boost::asio::yield_context& yield)
-    {
-        return operator()(pg_params{command, {}}, yield);
-    }
-
-    /** Insert multiple records into a table using Postgres' bulk COPY.
-     *
-     * Throws upon error.
-     *
-     * @param table Name of table for import.
-     * @param records Records in the COPY IN format.
-     */
-    void
-    bulkInsert(
-        char const* table,
-        std::string const& records,
-        boost::asio::yield_context& yield)
-    {
-        pg_->bulkInsert(table, records, yield);
-    }
-};
-
-//-----------------------------------------------------------------------------
-
-/** Create Postgres connection pool manager.
- *
- * @param pgConfig Configuration for Postgres.
- * @param j Logger object.
- * @param parent Stoppable parent object.
- * @return Postgres connection pool manager
- */
-std::shared_ptr<PgPool>
-make_PgPool(boost::asio::io_context& ioc, boost::json::object const& pgConfig);
-
-/** Initialize the Postgres schema.
- *
- * This function ensures that the database is running the latest version
- * of the schema.
- *
- * @param pool Postgres connection pool manager.
- */
-void
-initSchema(std::shared_ptr<PgPool> const& pool);
-void
-initAccountTx(std::shared_ptr<PgPool> const& pool);
-
-// Load the ledger info for the specified ledger/s from the database
-// @param whichLedger specifies the ledger to load via ledger sequence, ledger
-// hash or std::monostate (which loads the most recent)
-// @return vector of LedgerInfos
-std::optional<ripple::LedgerInfo>
-getLedger(
-    std::variant<std::monostate, ripple::uint256, std::uint32_t> const&
-        whichLedger,
-    std::shared_ptr<PgPool>& pgPool);
-
-#endif  // RIPPLE_CORE_PG_H_INCLUDED

src/backend/PostgresBackend.cpp

@@ -1,895 +0,0 @@
-#include <boost/asio.hpp>
-#include <boost/format.hpp>
-#include <backend/PostgresBackend.h>
-#include <thread>
-
-namespace Backend {
-
-// Type alias for async completion handlers
-using completion_token = boost::asio::yield_context;
-using function_type = void(boost::system::error_code);
-using result_type = boost::asio::async_result<completion_token, function_type>;
-using handler_type = typename result_type::completion_handler_type;
-
-struct HandlerWrapper
-{
-    handler_type handler;
-
-    HandlerWrapper(handler_type&& handler_) : handler(std::move(handler_))
-    {
-    }
-};
-
-PostgresBackend::PostgresBackend(
-    boost::asio::io_context& ioc,
-    boost::json::object const& config)
-    : BackendInterface(config)
-    , pgPool_(make_PgPool(ioc, config))
-    , writeConnection_(pgPool_)
-{
-    if (config.contains("write_interval"))
-    {
-        writeInterval_ = config.at("write_interval").as_int64();
-    }
-}
-void
-PostgresBackend::writeLedger(
-    ripple::LedgerInfo const& ledgerInfo,
-    std::string&& ledgerHeader)
-{
-    synchronous([&](boost::asio::yield_context yield) {
-        auto cmd = boost::format(
-            R"(INSERT INTO ledgers
-            VALUES (%u,'\x%s', '\x%s',%u,%u,%u,%u,%u,'\x%s','\x%s'))");
-
-        auto ledgerInsert = boost::str(
-            cmd % ledgerInfo.seq % ripple::strHex(ledgerInfo.hash) %
-            ripple::strHex(ledgerInfo.parentHash) % ledgerInfo.drops.drops() %
-            ledgerInfo.closeTime.time_since_epoch().count() %
-            ledgerInfo.parentCloseTime.time_since_epoch().count() %
-            ledgerInfo.closeTimeResolution.count() % ledgerInfo.closeFlags %
-            ripple::strHex(ledgerInfo.accountHash) %
-            ripple::strHex(ledgerInfo.txHash));
-
-        auto res = writeConnection_(ledgerInsert.data(), yield);
-        abortWrite_ = !res;
-        inProcessLedger = ledgerInfo.seq;
-    });
-}
-
-void
-PostgresBackend::writeAccountTransactions(
-    std::vector<AccountTransactionsData>&& data)
-{
-    if (abortWrite_)
-        return;
-    PgQuery pg(pgPool_);
-    for (auto const& record : data)
-    {
-        for (auto const& a : record.accounts)
-        {
-            std::string acct = ripple::strHex(a);
-            accountTxBuffer_ << "\\\\x" << acct << '\t'
-                             << std::to_string(record.ledgerSequence) << '\t'
-                             << std::to_string(record.transactionIndex) << '\t'
-                             << "\\\\x" << ripple::strHex(record.txHash)
-                             << '\n';
-        }
-    }
-}
-
-void
-PostgresBackend::writeNFTTransactions(std::vector<NFTTransactionsData>&& data)
-{
-    throw std::runtime_error("Not implemented");
-}
-
-void
-PostgresBackend::doWriteLedgerObject(
-    std::string&& key,
-    std::uint32_t const seq,
-    std::string&& blob)
-{
-    synchronous([&](boost::asio::yield_context yield) {
-        if (abortWrite_)
-            return;
-        objectsBuffer_ << "\\\\x" << ripple::strHex(key) << '\t'
-                       << std::to_string(seq) << '\t' << "\\\\x"
-                       << ripple::strHex(blob) << '\n';
-        numRowsInObjectsBuffer_++;
-        // If the buffer gets too large, the insert fails. Not sure why. So we
-        // insert after 1 million records
-        if (numRowsInObjectsBuffer_ % writeInterval_ == 0)
-        {
-            BOOST_LOG_TRIVIAL(info)
-                << __func__ << " Flushing large buffer. num objects = "
-                << numRowsInObjectsBuffer_;
-            writeConnection_.bulkInsert("objects", objectsBuffer_.str(), yield);
-            BOOST_LOG_TRIVIAL(info) << __func__ << " Flushed large buffer";
-            objectsBuffer_.str("");
-        }
-    });
-}
-
-void
-PostgresBackend::writeSuccessor(
-    std::string&& key,
-    std::uint32_t const seq,
-    std::string&& successor)
-{
-    synchronous([&](boost::asio::yield_context yield) {
-        if (range)
-        {
-            if (successors_.count(key) > 0)
-                return;
-            successors_.insert(key);
-        }
-        successorBuffer_ << "\\\\x" << ripple::strHex(key) << '\t'
-                         << std::to_string(seq) << '\t' << "\\\\x"
-                         << ripple::strHex(successor) << '\n';
-        BOOST_LOG_TRIVIAL(trace)
-            << __func__ << ripple::strHex(key) << " - " << std::to_string(seq);
-        numRowsInSuccessorBuffer_++;
-        if (numRowsInSuccessorBuffer_ % writeInterval_ == 0)
-        {
-            BOOST_LOG_TRIVIAL(info)
-                << __func__ << " Flushing large buffer. num successors = "
-                << numRowsInSuccessorBuffer_;
-            writeConnection_.bulkInsert(
-                "successor", successorBuffer_.str(), yield);
-            BOOST_LOG_TRIVIAL(info) << __func__ << " Flushed large buffer";
-            successorBuffer_.str("");
-        }
-    });
-}
-
-void
-PostgresBackend::writeTransaction(
-    std::string&& hash,
-    std::uint32_t const seq,
-    std::uint32_t const date,
-    std::string&& transaction,
-    std::string&& metadata)
-{
-    if (abortWrite_)
-        return;
-    transactionsBuffer_ << "\\\\x" << ripple::strHex(hash) << '\t'
-                        << std::to_string(seq) << '\t' << std::to_string(date)
-                        << '\t' << "\\\\x" << ripple::strHex(transaction)
-                        << '\t' << "\\\\x" << ripple::strHex(metadata) << '\n';
-}
-
-void
-PostgresBackend::writeNFTs(std::vector<NFTsData>&& data)
-{
-    throw std::runtime_error("Not implemented");
-}
-
-std::uint32_t
-checkResult(PgResult const& res, std::uint32_t const numFieldsExpected)
-{
-    if (!res)
-    {
-        auto msg = res.msg();
-        BOOST_LOG_TRIVIAL(error) << __func__ << " - " << msg;
-        if (msg.find("statement timeout"))
-            throw DatabaseTimeout();
-        assert(false);
-        throw DatabaseTimeout();
-    }
-    if (res.status() != PGRES_TUPLES_OK)
-    {
-        std::stringstream msg;
-        msg << " : Postgres response should have been "
-               "PGRES_TUPLES_OK but instead was "
-            << res.status() << " - msg  = " << res.msg();
-        BOOST_LOG_TRIVIAL(error) << __func__ << " - " << msg.str();
-        assert(false);
-        throw DatabaseTimeout();
-    }
-
-    BOOST_LOG_TRIVIAL(trace)
-        << __func__ << " Postgres result msg  : " << res.msg();
-    if (res.isNull() || res.ntuples() == 0)
-    {
-        return 0;
-    }
-    else if (res.ntuples() > 0)
-    {
-        if (res.nfields() != numFieldsExpected)
-        {
-            std::stringstream msg;
-            msg << "Wrong number of fields in Postgres "
-                   "response. Expected "
-                << numFieldsExpected << ", but got " << res.nfields();
-            throw std::runtime_error(msg.str());
-            assert(false);
-        }
-    }
-    return res.ntuples();
-}
-
-ripple::LedgerInfo
-parseLedgerInfo(PgResult const& res)
-{
-    std::int64_t ledgerSeq = res.asBigInt(0, 0);
-    ripple::uint256 hash = res.asUInt256(0, 1);
-    ripple::uint256 prevHash = res.asUInt256(0, 2);
-    std::int64_t totalCoins = res.asBigInt(0, 3);
-    std::int64_t closeTime = res.asBigInt(0, 4);
-    std::int64_t parentCloseTime = res.asBigInt(0, 5);
-    std::int64_t closeTimeRes = res.asBigInt(0, 6);
-    std::int64_t closeFlags = res.asBigInt(0, 7);
-    ripple::uint256 accountHash = res.asUInt256(0, 8);
-    ripple::uint256 txHash = res.asUInt256(0, 9);
-
-    using time_point = ripple::NetClock::time_point;
-    using duration = ripple::NetClock::duration;
-
-    ripple::LedgerInfo info;
-    info.seq = ledgerSeq;
-    info.hash = hash;
-    info.parentHash = prevHash;
-    info.drops = totalCoins;
-    info.closeTime = time_point{duration{closeTime}};
-    info.parentCloseTime = time_point{duration{parentCloseTime}};
-    info.closeFlags = closeFlags;
-    info.closeTimeResolution = duration{closeTimeRes};
-    info.accountHash = accountHash;
-    info.txHash = txHash;
-    info.validated = true;
-    return info;
-}
-std::optional<std::uint32_t>
-PostgresBackend::fetchLatestLedgerSequence(
-    boost::asio::yield_context& yield) const
-{
-    PgQuery pgQuery(pgPool_);
-    pgQuery(set_timeout, yield);
-
-    auto const query =
-        "SELECT ledger_seq FROM ledgers ORDER BY ledger_seq DESC LIMIT 1";
-
-    if (auto res = pgQuery(query, yield); checkResult(res, 1))
-        return res.asBigInt(0, 0);
-
-    return {};
-}
-
-std::optional<ripple::LedgerInfo>
-PostgresBackend::fetchLedgerBySequence(
-    std::uint32_t const sequence,
-    boost::asio::yield_context& yield) const
-{
-    PgQuery pgQuery(pgPool_);
-    pgQuery(set_timeout, yield);
-
-    std::stringstream sql;
-    sql << "SELECT * FROM ledgers WHERE ledger_seq = "
-        << std::to_string(sequence);
-
-    if (auto res = pgQuery(sql.str().data(), yield); checkResult(res, 10))
-        return parseLedgerInfo(res);
-
-    return {};
-}
-
-std::optional<ripple::LedgerInfo>
-PostgresBackend::fetchLedgerByHash(
-    ripple::uint256 const& hash,
-    boost::asio::yield_context& yield) const
-{
-    PgQuery pgQuery(pgPool_);
-    pgQuery(set_timeout, yield);
-
-    std::stringstream sql;
-    sql << "SELECT * FROM ledgers WHERE ledger_hash = \'\\x"
-        << ripple::to_string(hash) << "\'";
-
-    if (auto res = pgQuery(sql.str().data(), yield); checkResult(res, 10))
-        return parseLedgerInfo(res);
-
-    return {};
-}
-
-std::optional<LedgerRange>
-PostgresBackend::hardFetchLedgerRange(boost::asio::yield_context& yield) const
-{
-    auto range = PgQuery(pgPool_)("SELECT complete_ledgers()", yield);
-    if (!range)
-        return {};
-
-    std::string res{range.c_str()};
-    BOOST_LOG_TRIVIAL(debug) << "range is = " << res;
-    try
-    {
-        size_t minVal = 0;
-        size_t maxVal = 0;
-        if (res == "empty" || res == "error" || res.empty())
-            return {};
-        else if (size_t delim = res.find('-'); delim != std::string::npos)
-        {
-            minVal = std::stol(res.substr(0, delim));
-            maxVal = std::stol(res.substr(delim + 1));
-        }
-        else
-        {
-            minVal = maxVal = std::stol(res);
-        }
-        return LedgerRange{minVal, maxVal};
-    }
-    catch (std::exception&)
-    {
-        BOOST_LOG_TRIVIAL(error)
-            << __func__ << " : "
-            << "Error parsing result of getCompleteLedgers()";
-    }
-    return {};
-}
-
-std::optional<Blob>
-PostgresBackend::doFetchLedgerObject(
-    ripple::uint256 const& key,
-    std::uint32_t const sequence,
-    boost::asio::yield_context& yield) const
-{
-    PgQuery pgQuery(pgPool_);
-    pgQuery(set_timeout, yield);
-
-    std::stringstream sql;
-    sql << "SELECT object FROM objects WHERE key = "
-        << "\'\\x" << ripple::strHex(key) << "\'"
-        << " AND ledger_seq <= " << std::to_string(sequence)
-        << " ORDER BY ledger_seq DESC LIMIT 1";
-
-    if (auto res = pgQuery(sql.str().data(), yield); checkResult(res, 1))
-    {
-        auto blob = res.asUnHexedBlob(0, 0);
-        if (blob.size())
-            return blob;
-    }
-
-    return {};
-}
-
-// returns a transaction, metadata pair
-std::optional<TransactionAndMetadata>
-PostgresBackend::fetchTransaction(
-    ripple::uint256 const& hash,
-    boost::asio::yield_context& yield) const
-{
-    PgQuery pgQuery(pgPool_);
-    pgQuery(set_timeout, yield);
-
-    std::stringstream sql;
-    sql << "SELECT transaction,metadata,ledger_seq,date FROM transactions "
-           "WHERE hash = "
-        << "\'\\x" << ripple::strHex(hash) << "\'";
-
-    if (auto res = pgQuery(sql.str().data(), yield); checkResult(res, 4))
-    {
-        return {
-            {res.asUnHexedBlob(0, 0),
-             res.asUnHexedBlob(0, 1),
-             res.asBigInt(0, 2),
-             res.asBigInt(0, 3)}};
-    }
-
-    return {};
-}
-std::vector<TransactionAndMetadata>
-PostgresBackend::fetchAllTransactionsInLedger(
-    std::uint32_t const ledgerSequence,
-    boost::asio::yield_context& yield) const
-{
-    PgQuery pgQuery(pgPool_);
-    pgQuery(set_timeout, yield);
-
-    std::stringstream sql;
-    sql << "SELECT transaction, metadata, ledger_seq,date FROM transactions "
-           "WHERE "
-        << "ledger_seq = " << std::to_string(ledgerSequence);
-
-    auto res = pgQuery(sql.str().data(), yield);
-    if (size_t numRows = checkResult(res, 4))
-    {
-        std::vector<TransactionAndMetadata> txns;
-        for (size_t i = 0; i < numRows; ++i)
-        {
-            txns.push_back(
-                {res.asUnHexedBlob(i, 0),
-                 res.asUnHexedBlob(i, 1),
-                 res.asBigInt(i, 2),
-                 res.asBigInt(i, 3)});
-        }
-        return txns;
-    }
-    return {};
-}
-std::vector<ripple::uint256>
-PostgresBackend::fetchAllTransactionHashesInLedger(
-    std::uint32_t const ledgerSequence,
-    boost::asio::yield_context& yield) const
-{
-    PgQuery pgQuery(pgPool_);
-    pgQuery(set_timeout, yield);
-
-    std::stringstream sql;
-    sql << "SELECT hash FROM transactions WHERE "
-        << "ledger_seq = " << std::to_string(ledgerSequence);
-
-    auto res = pgQuery(sql.str().data(), yield);
-    if (size_t numRows = checkResult(res, 1))
-    {
-        std::vector<ripple::uint256> hashes;
-        for (size_t i = 0; i < numRows; ++i)
-        {
-            hashes.push_back(res.asUInt256(i, 0));
-        }
-        return hashes;
-    }
-
-    return {};
-}
-
-std::optional<NFT>
-PostgresBackend::fetchNFT(
-    ripple::uint256 const& tokenID,
-    std::uint32_t const ledgerSequence,
-    boost::asio::yield_context& yield) const
-{
-    throw std::runtime_error("Not implemented");
-}
-
-std::optional<ripple::uint256>
-PostgresBackend::doFetchSuccessorKey(
-    ripple::uint256 key,
-    std::uint32_t const ledgerSequence,
-    boost::asio::yield_context& yield) const
-{
-    PgQuery pgQuery(pgPool_);
-    pgQuery(set_timeout, yield);
-
-    std::stringstream sql;
-    sql << "SELECT next FROM successor WHERE key = "
-        << "\'\\x" << ripple::strHex(key) << "\'"
-        << " AND ledger_seq <= " << std::to_string(ledgerSequence)
-        << " ORDER BY ledger_seq DESC LIMIT 1";
-
-    if (auto res = pgQuery(sql.str().data(), yield); checkResult(res, 1))
-    {
-        auto next = res.asUInt256(0, 0);
-        if (next == lastKey)
-            return {};
-        return next;
-    }
-
-    return {};
-}
-
-std::vector<TransactionAndMetadata>
-PostgresBackend::fetchTransactions(
-    std::vector<ripple::uint256> const& hashes,
-    boost::asio::yield_context& yield) const
-{
-    if (!hashes.size())
-        return {};
-
-    std::vector<TransactionAndMetadata> results;
-    results.resize(hashes.size());
-
-    handler_type handler(std::forward<decltype(yield)>(yield));
-    result_type result(handler);
-
-    auto hw = new HandlerWrapper(std::move(handler));
-
-    auto start = std::chrono::system_clock::now();
-
-    std::atomic_uint numRemaining = hashes.size();
-    std::atomic_bool errored = false;
-
-    for (size_t i = 0; i < hashes.size(); ++i)
-    {
-        auto const& hash = hashes[i];
-        boost::asio::spawn(
-            get_associated_executor(yield),
-            [this, &hash, &results, hw, &numRemaining, &errored, i](
-                boost::asio::yield_context yield) {
-                BOOST_LOG_TRIVIAL(trace) << __func__ << " getting txn = " << i;
-
-                PgQuery pgQuery(pgPool_);
-
-                std::stringstream sql;
-                sql << "SELECT transaction,metadata,ledger_seq,date FROM "
-                       "transactions "
-                       "WHERE HASH = \'\\x"
-                    << ripple::strHex(hash) << "\'";
-
-                try
-                {
-                    if (auto const res = pgQuery(sql.str().data(), yield);
-                        checkResult(res, 4))
-                    {
-                        results[i] = {
-                            res.asUnHexedBlob(0, 0),
-                            res.asUnHexedBlob(0, 1),
-                            res.asBigInt(0, 2),
-                            res.asBigInt(0, 3)};
-                    }
-                }
-                catch (DatabaseTimeout const&)
-                {
-                    errored = true;
-                }
-
-                if (--numRemaining == 0)
-                {
-                    handler_type h(std::move(hw->handler));
-                    h(boost::system::error_code{});
-                }
-            });
-    }
-
-    // Yields the worker to the io_context until handler is called.
-    result.get();
-
-    delete hw;
-
-    auto end = std::chrono::system_clock::now();
-    auto duration =
-        std::chrono::duration_cast<std::chrono::milliseconds>(end - start);
-
-    BOOST_LOG_TRIVIAL(info)
-        << __func__ << " fetched " << std::to_string(hashes.size())
-        << " transactions asynchronously. took "
-        << std::to_string(duration.count());
-    if (errored)
-    {
-        BOOST_LOG_TRIVIAL(error) << __func__ << " Database fetch timed out";
-        throw DatabaseTimeout();
-    }
-
-    return results;
-}
-
-std::vector<Blob>
-PostgresBackend::doFetchLedgerObjects(
-    std::vector<ripple::uint256> const& keys,
-    std::uint32_t const sequence,
-    boost::asio::yield_context& yield) const
-{
-    if (!keys.size())
-        return {};
-
-    PgQuery pgQuery(pgPool_);
-    pgQuery(set_timeout, yield);
-
-    std::vector<Blob> results;
-    results.resize(keys.size());
-
-    handler_type handler(std::forward<decltype(yield)>(yield));
-    result_type result(handler);
-
-    auto hw = new HandlerWrapper(std::move(handler));
-
-    std::atomic_uint numRemaining = keys.size();
-    std::atomic_bool errored = false;
-    auto start = std::chrono::system_clock::now();
-    for (size_t i = 0; i < keys.size(); ++i)
-    {
-        auto const& key = keys[i];
-        boost::asio::spawn(
-            boost::asio::get_associated_executor(yield),
-            [this, &key, &results, &numRemaining, &errored, hw, i, sequence](
-                boost::asio::yield_context yield) {
-                PgQuery pgQuery(pgPool_);
-
-                std::stringstream sql;
-                sql << "SELECT object FROM "
-                       "objects "
-                       "WHERE key = \'\\x"
-                    << ripple::strHex(key) << "\'"
-                    << " AND ledger_seq <= " << std::to_string(sequence)
-                    << " ORDER BY ledger_seq DESC LIMIT 1";
-
-                try
-                {
-                    if (auto const res = pgQuery(sql.str().data(), yield);
-                        checkResult(res, 1))
-                        results[i] = res.asUnHexedBlob();
-                }
-                catch (DatabaseTimeout const& ex)
-                {
-                    errored = true;
-                }
-
-                if (--numRemaining == 0)
-                {
-                    handler_type h(std::move(hw->handler));
-                    h(boost::system::error_code{});
-                }
-            });
-    }
-
-    // Yields the worker to the io_context until handler is called.
-    result.get();
-
-    delete hw;
-
-    auto end = std::chrono::system_clock::now();
-    auto duration =
-        std::chrono::duration_cast<std::chrono::milliseconds>(end - start);
-
-    BOOST_LOG_TRIVIAL(info)
-        << __func__ << " fetched " << std::to_string(keys.size())
-        << " objects asynchronously. ms = " << std::to_string(duration.count());
-    if (errored)
-    {
-        BOOST_LOG_TRIVIAL(error) << __func__ << " Database fetch timed out";
-        throw DatabaseTimeout();
-    }
-
-    return results;
-}
-
-std::vector<LedgerObject>
-PostgresBackend::fetchLedgerDiff(
-    std::uint32_t const ledgerSequence,
-    boost::asio::yield_context& yield) const
-{
-    PgQuery pgQuery(pgPool_);
-    pgQuery(set_timeout, yield);
-
-    std::stringstream sql;
-    sql << "SELECT key,object FROM objects "
-           "WHERE "
-        << "ledger_seq = " << std::to_string(ledgerSequence);
-
-    auto res = pgQuery(sql.str().data(), yield);
-    if (size_t numRows = checkResult(res, 2))
-    {
-        std::vector<LedgerObject> objects;
-        for (size_t i = 0; i < numRows; ++i)
-        {
-            objects.push_back({res.asUInt256(i, 0), res.asUnHexedBlob(i, 1)});
-        }
-        return objects;
-    }
-
-    return {};
-}
-
-// TODO this implementation and fetchAccountTransactions should be
-// generalized
-TransactionsAndCursor
-PostgresBackend::fetchNFTTransactions(
-    ripple::uint256 const& tokenID,
-    std::uint32_t const limit,
-    bool forward,
-    std::optional<TransactionsCursor> const& cursor,
-    boost::asio::yield_context& yield) const
-{
-    throw std::runtime_error("Not implemented");
-}
-
-TransactionsAndCursor
-PostgresBackend::fetchAccountTransactions(
-    ripple::AccountID const& account,
-    std::uint32_t const limit,
-    bool forward,
-    std::optional<TransactionsCursor> const& cursor,
-    boost::asio::yield_context& yield) const
-{
-    PgQuery pgQuery(pgPool_);
-    pgQuery(set_timeout, yield);
-    pg_params dbParams;
-
-    char const*& command = dbParams.first;
-    std::vector<std::optional<std::string>>& values = dbParams.second;
-    command =
-        "SELECT account_tx($1::bytea, $2::bigint, $3::bool, "
-        "$4::bigint, $5::bigint)";
-    values.resize(5);
-    values[0] = "\\x" + strHex(account);
-
-    values[1] = std::to_string(limit);
-
-    values[2] = std::to_string(forward);
-
-    if (cursor)
-    {
-        values[3] = std::to_string(cursor->ledgerSequence);
-        values[4] = std::to_string(cursor->transactionIndex);
-    }
-    for (size_t i = 0; i < values.size(); ++i)
-    {
-        BOOST_LOG_TRIVIAL(debug) << "value " << std::to_string(i) << " = "
-                                 << (values[i] ? values[i].value() : "null");
-    }
-
-    auto start = std::chrono::system_clock::now();
-    auto res = pgQuery(dbParams, yield);
-    auto end = std::chrono::system_clock::now();
-
-    auto duration = ((end - start).count()) / 1000000000.0;
-    BOOST_LOG_TRIVIAL(info)
-        << __func__ << " : executed stored_procedure in "
-        << std::to_string(duration)
-        << " num records = " << std::to_string(checkResult(res, 1));
-
-    checkResult(res, 1);
-
-    char const* resultStr = res.c_str();
-    BOOST_LOG_TRIVIAL(debug) << __func__ << " : "
-                             << "postgres result = " << resultStr
-                             << " : account = " << strHex(account);
-
-    boost::json::value raw = boost::json::parse(resultStr);
-    boost::json::object responseObj = raw.as_object();
-    BOOST_LOG_TRIVIAL(debug) << " parsed = " << responseObj;
-    if (responseObj.contains("transactions"))
-    {
-        auto txns = responseObj.at("transactions").as_array();
-        std::vector<ripple::uint256> hashes;
-        for (auto& hashHex : txns)
-        {
-            ripple::uint256 hash;
-            if (hash.parseHex(hashHex.at("hash").as_string().c_str() + 2))
-                hashes.push_back(hash);
-        }
-        if (responseObj.contains("cursor"))
-        {
-            return {
-                fetchTransactions(hashes, yield),
-                {{responseObj.at("cursor").at("ledger_sequence").as_int64(),
-                  responseObj.at("cursor")
-                      .at("transaction_index")
-                      .as_int64()}}};
-        }
-        return {fetchTransactions(hashes, yield), {}};
-    }
-    return {{}, {}};
-}  // namespace Backend
-
-void
-PostgresBackend::open(bool readOnly)
-{
-    initSchema(pgPool_);
-    initAccountTx(pgPool_);
-}
-
-void
-PostgresBackend::close()
-{
-}
-
-void
-PostgresBackend::startWrites() const
-{
-    synchronous([&](boost::asio::yield_context yield) {
-        numRowsInObjectsBuffer_ = 0;
-        abortWrite_ = false;
-        auto res = writeConnection_("BEGIN", yield);
-        if (!res || res.status() != PGRES_COMMAND_OK)
-        {
-            std::stringstream msg;
-            msg << "Postgres error creating transaction: " << res.msg();
-            throw std::runtime_error(msg.str());
-        }
-    });
-}
-
-bool
-PostgresBackend::doFinishWrites()
-{
-    synchronous([&](boost::asio::yield_context yield) {
-        if (!abortWrite_)
-        {
-            std::string txStr = transactionsBuffer_.str();
-            writeConnection_.bulkInsert("transactions", txStr, yield);
-            writeConnection_.bulkInsert(
-                "account_transactions", accountTxBuffer_.str(), yield);
-            std::string objectsStr = objectsBuffer_.str();
-            if (objectsStr.size())
-                writeConnection_.bulkInsert("objects", objectsStr, yield);
-            BOOST_LOG_TRIVIAL(debug)
-                << __func__ << " objects size = " << objectsStr.size()
-                << " txns size = " << txStr.size();
-            std::string successorStr = successorBuffer_.str();
-            if (successorStr.size())
-                writeConnection_.bulkInsert("successor", successorStr, yield);
-            if (!range)
-            {
-                std::stringstream indexCreate;
-                indexCreate
-                    << "CREATE INDEX diff ON objects USING hash(ledger_seq) "
-                       "WHERE NOT "
-                       "ledger_seq = "
-                    << std::to_string(inProcessLedger);
-                writeConnection_(indexCreate.str().data(), yield);
-            }
-        }
-        auto res = writeConnection_("COMMIT", yield);
-        if (!res || res.status() != PGRES_COMMAND_OK)
-        {
-            std::stringstream msg;
-            msg << "Postgres error committing transaction: " << res.msg();
-            throw std::runtime_error(msg.str());
-        }
-        transactionsBuffer_.str("");
-        transactionsBuffer_.clear();
-        objectsBuffer_.str("");
-        objectsBuffer_.clear();
-        successorBuffer_.str("");
-        successorBuffer_.clear();
-        successors_.clear();
-        accountTxBuffer_.str("");
-        accountTxBuffer_.clear();
-        numRowsInObjectsBuffer_ = 0;
-    });
-
-    return !abortWrite_;
-}
-
-bool
-PostgresBackend::doOnlineDelete(
-    std::uint32_t const numLedgersToKeep,
-    boost::asio::yield_context& yield) const
-{
-    auto rng = fetchLedgerRange();
-    if (!rng)
-        return false;
-    std::uint32_t minLedger = rng->maxSequence - numLedgersToKeep;
-    if (minLedger <= rng->minSequence)
-        return false;
-    PgQuery pgQuery(pgPool_);
-    pgQuery("SET statement_timeout TO 0", yield);
-    std::optional<ripple::uint256> cursor;
-    while (true)
-    {
-        auto [objects, curCursor] = retryOnTimeout([&]() {
-            return fetchLedgerPage(cursor, minLedger, 256, false, yield);
-        });
-        BOOST_LOG_TRIVIAL(debug) << __func__ << " fetched a page";
-        std::stringstream objectsBuffer;
-
-        for (auto& obj : objects)
-        {
-            objectsBuffer << "\\\\x" << ripple::strHex(obj.key) << '\t'
-                          << std::to_string(minLedger) << '\t' << "\\\\x"
-                          << ripple::strHex(obj.blob) << '\n';
-        }
-        pgQuery.bulkInsert("objects", objectsBuffer.str(), yield);
-        cursor = curCursor;
-        if (!cursor)
-            break;
-    }
-    BOOST_LOG_TRIVIAL(info) << __func__ << " finished inserting into objects";
-    {
-        std::stringstream sql;
-        sql << "DELETE FROM ledgers WHERE ledger_seq < "
-            << std::to_string(minLedger);
-        auto res = pgQuery(sql.str().data(), yield);
-        if (res.msg() != "ok")
-            throw std::runtime_error("Error deleting from ledgers table");
-    }
-    {
-        std::stringstream sql;
-        sql << "DELETE FROM keys WHERE ledger_seq < "
-            << std::to_string(minLedger);
-        auto res = pgQuery(sql.str().data(), yield);
-        if (res.msg() != "ok")
-            throw std::runtime_error("Error deleting from keys table");
-    }
-    {
-        std::stringstream sql;
-        sql << "DELETE FROM books WHERE ledger_seq < "
-            << std::to_string(minLedger);
-        auto res = pgQuery(sql.str().data(), yield);
-        if (res.msg() != "ok")
-            throw std::runtime_error("Error deleting from books table");
-    }
-    return true;
-}
-
-}  // namespace Backend

src/backend/PostgresBackend.h

@@ -1,165 +0,0 @@
-#ifndef RIPPLE_APP_REPORTING_POSTGRESBACKEND_H_INCLUDED
-#define RIPPLE_APP_REPORTING_POSTGRESBACKEND_H_INCLUDED
-#include <boost/json.hpp>
-#include <backend/BackendInterface.h>
-
-namespace Backend {
-class PostgresBackend : public BackendInterface
-{
-private:
-    mutable size_t numRowsInObjectsBuffer_ = 0;
-    mutable std::stringstream objectsBuffer_;
-    mutable size_t numRowsInSuccessorBuffer_ = 0;
-    mutable std::stringstream successorBuffer_;
-    mutable std::stringstream transactionsBuffer_;
-    mutable std::stringstream accountTxBuffer_;
-    std::shared_ptr<PgPool> pgPool_;
-    mutable PgQuery writeConnection_;
-    mutable bool abortWrite_ = false;
-    std::uint32_t writeInterval_ = 1000000;
-    std::uint32_t inProcessLedger = 0;
-    mutable std::unordered_set<std::string> successors_;
-
-    const char* const set_timeout = "SET statement_timeout TO 10000";
-
-public:
-    PostgresBackend(
-        boost::asio::io_context& ioc,
-        boost::json::object const& config);
-
-    std::optional<std::uint32_t>
-    fetchLatestLedgerSequence(boost::asio::yield_context& yield) const override;
-
-    std::optional<ripple::LedgerInfo>
-    fetchLedgerBySequence(
-        std::uint32_t const sequence,
-        boost::asio::yield_context& yield) const override;
-
-    std::optional<ripple::LedgerInfo>
-    fetchLedgerByHash(
-        ripple::uint256 const& hash,
-        boost::asio::yield_context& yield) const override;
-
-    std::optional<Blob>
-    doFetchLedgerObject(
-        ripple::uint256 const& key,
-        std::uint32_t const sequence,
-        boost::asio::yield_context& yield) const override;
-
-    // returns a transaction, metadata pair
-    std::optional<TransactionAndMetadata>
-    fetchTransaction(
-        ripple::uint256 const& hash,
-        boost::asio::yield_context& yield) const override;
-
-    std::vector<TransactionAndMetadata>
-    fetchAllTransactionsInLedger(
-        std::uint32_t const ledgerSequence,
-        boost::asio::yield_context& yield) const override;
-
-    std::vector<ripple::uint256>
-    fetchAllTransactionHashesInLedger(
-        std::uint32_t const ledgerSequence,
-        boost::asio::yield_context& yield) const override;
-
-    std::optional<NFT>
-    fetchNFT(
-        ripple::uint256 const& tokenID,
-        std::uint32_t const ledgerSequence,
-        boost::asio::yield_context& yield) const override;
-
-    TransactionsAndCursor
-    fetchNFTTransactions(
-        ripple::uint256 const& tokenID,
-        std::uint32_t const limit,
-        bool const forward,
-        std::optional<TransactionsCursor> const& cursorIn,
-        boost::asio::yield_context& yield) const override;
-
-    std::vector<LedgerObject>
-    fetchLedgerDiff(
-        std::uint32_t const ledgerSequence,
-        boost::asio::yield_context& yield) const override;
-
-    std::optional<LedgerRange>
-    hardFetchLedgerRange(boost::asio::yield_context& yield) const override;
-
-    std::optional<ripple::uint256>
-    doFetchSuccessorKey(
-        ripple::uint256 key,
-        std::uint32_t const ledgerSequence,
-        boost::asio::yield_context& yield) const override;
-
-    std::vector<TransactionAndMetadata>
-    fetchTransactions(
-        std::vector<ripple::uint256> const& hashes,
-        boost::asio::yield_context& yield) const override;
-
-    std::vector<Blob>
-    doFetchLedgerObjects(
-        std::vector<ripple::uint256> const& keys,
-        std::uint32_t const sequence,
-        boost::asio::yield_context& yield) const override;
-
-    TransactionsAndCursor
-    fetchAccountTransactions(
-        ripple::AccountID const& account,
-        std::uint32_t const limit,
-        bool forward,
-        std::optional<TransactionsCursor> const& cursor,
-        boost::asio::yield_context& yield) const override;
-
-    void
-    writeLedger(
-        ripple::LedgerInfo const& ledgerInfo,
-        std::string&& ledgerHeader) override;
-
-    void
-    doWriteLedgerObject(
-        std::string&& key,
-        std::uint32_t const seq,
-        std::string&& blob) override;
-
-    void
-    writeSuccessor(
-        std::string&& key,
-        std::uint32_t const seq,
-        std::string&& successor) override;
-
-    void
-    writeTransaction(
-        std::string&& hash,
-        std::uint32_t const seq,
-        std::uint32_t const date,
-        std::string&& transaction,
-        std::string&& metadata) override;
-
-    void
-    writeNFTs(std::vector<NFTsData>&& data) override;
-
-    void
-    writeAccountTransactions(
-        std::vector<AccountTransactionsData>&& data) override;
-
-    void
-    writeNFTTransactions(std::vector<NFTTransactionsData>&& data) override;
-
-    void
-    open(bool readOnly) override;
-
-    void
-    close() override;
-
-    void
-    startWrites() const override;
-
-    bool
-    doFinishWrites() override;
-
-    bool
-    doOnlineDelete(
-        std::uint32_t const numLedgersToKeep,
-        boost::asio::yield_context& yield) const override;
-};
-}  // namespace Backend
-#endif

src/backend/README.md

@@ -1,6 +1,6 @@
 # Clio Backend
 ## Background
-The backend of Clio is responsible for handling the proper reading and writing of past ledger data from and to a given database. As of right now, Cassandra is the only supported database that is production-ready. However, support for more databases like PostgreSQL and DynamoDB may be added in future versions. Support for database types can be easily extended by creating new implementations which implements the virtual methods of `BackendInterface.h`. Then, use the Factory Object Design Pattern to simply add logic statements to `BackendFactory.h` that return the new database interface for a specific `type` in Clio's configuration file. 
+The backend of Clio is responsible for handling the proper reading and writing of past ledger data from and to a given database. As of right now, Cassandra and ScyllaDB are the only supported databases that are production-ready. Support for database types can be easily extended by creating new implementations which implements the virtual methods of `BackendInterface.h`. Then, use the Factory Object Design Pattern to simply add logic statements to `BackendFactory.h` that return the new database interface for a specific `type` in Clio's configuration file. 
 
 ## Data Model
 The data model used by Clio to read and write ledger data is different from what Rippled uses. Rippled uses a novel data structure named [*SHAMap*](https://github.com/ripple/rippled/blob/master/src/ripple/shamap/README.md), which is a combination of a Merkle Tree and a Radix Trie. In a SHAMap, ledger objects are stored in the root vertices of the tree. Thus, looking up a record located at the leaf node of the SHAMap executes a tree search, where the path from the root node to the leaf node is the key of the record. Rippled nodes can also generate a proof-tree by forming a subtree with all the path nodes and their neighbors, which can then be used to prove the existnce of the leaf node data to other Rippled nodes. In short, the main purpose of the SHAMap data structure is to facilitate the fast validation of data integrity between different decentralized Rippled nodes.
@@ -129,4 +129,4 @@ In each new ledger version with sequence `n`, a ledger object `v` can either be
  2. If `v` is...
 	 1. Being **created**, add two new records of `seq=n` with one being `e` pointing to `v`, and `v` pointing to `w` (Linked List insertion operation).
 	 2. Being **modified**, do nothing.
-	 3. Being **deleted**, add a record of `seq=n` with `e` pointing to `v`'s `next` value (Linked List deletion operation).
+	 3. Being **deleted**, add a record of `seq=n` with `e` pointing to `v`'s `next` value (Linked List deletion operation).

src/backend/SimpleCache.cpp

@@ -1,3 +1,22 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
 #include <backend/SimpleCache.h>
 namespace Backend {
 
@@ -18,7 +37,7 @@ SimpleCache::update(
         return;
 
     {
-        std::unique_lock lck{mtx_};
+        std::scoped_lock lck{mtx_};
         if (seq > latestSeq_)
         {
             assert(seq == latestSeq_ + 1 || latestSeq_ == 0);
@@ -106,7 +125,7 @@ SimpleCache::setFull()
         return;
 
     full_ = true;
-    std::unique_lock lck{mtx_};
+    std::scoped_lock lck{mtx_};
     deletes_.clear();
 }
 

src/backend/SimpleCache.h

@@ -1,5 +1,23 @@
-#ifndef CLIO_SIMPLECACHE_H_INCLUDED
-#define CLIO_SIMPLECACHE_H_INCLUDED
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
 
 #include <ripple/basics/base_uint.h>
 #include <ripple/basics/hardened_hash.h>
@@ -78,4 +96,3 @@ public:
 };
 
 }  // namespace Backend
-#endif

src/backend/Types.h

@@ -1,5 +1,24 @@
-#ifndef CLIO_TYPES_H_INCLUDED
-#define CLIO_TYPES_H_INCLUDED
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
 #include <ripple/basics/base_uint.h>
 #include <ripple/protocol/AccountID.h>
 #include <optional>
@@ -89,4 +108,3 @@ constexpr ripple::uint256 lastKey{
 constexpr ripple::uint256 hi192{
     "0000000000000000000000000000000000000000000000001111111111111111"};
 }  // namespace Backend
-#endif

src/config/Config.cpp

@@ -0,0 +1,190 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include <config/Config.h>
+#include <log/Logger.h>
+
+#include <fstream>
+
+namespace clio {
+
+// Note: `store_(store)` MUST use `()` instead of `{}` otherwise gcc
+// picks `initializer_list` constructor and anything passed becomes an
+// array :-D
+Config::Config(boost::json::value store) : store_(std::move(store))
+{
+}
+
+Config::operator bool() const noexcept
+{
+    return not store_.is_null();
+}
+
+bool
+Config::contains(key_type key) const
+{
+    return lookup(key).has_value();
+}
+
+std::optional<boost::json::value>
+Config::lookup(key_type key) const
+{
+    if (store_.is_null())
+        return std::nullopt;
+
+    std::reference_wrapper<boost::json::value const> cur = std::cref(store_);
+    auto hasBrokenPath = false;
+    auto tokenized = detail::Tokenizer<key_type, Separator>{key};
+    std::string subkey{};
+
+    auto maybeSection = tokenized.next();
+    while (maybeSection.has_value())
+    {
+        auto section = maybeSection.value();
+        subkey += section;
+
+        if (not hasBrokenPath)
+        {
+            if (not cur.get().is_object())
+                throw detail::StoreException(
+                    "Not an object at '" + subkey + "'");
+            if (not cur.get().as_object().contains(section))
+                hasBrokenPath = true;
+            else
+                cur = std::cref(cur.get().as_object().at(section));
+        }
+
+        subkey += Separator;
+        maybeSection = tokenized.next();
+    }
+
+    if (hasBrokenPath)
+        return std::nullopt;
+    return std::make_optional(cur);
+}
+
+std::optional<Config::array_type>
+Config::maybeArray(key_type key) const
+{
+    try
+    {
+        auto maybe_arr = lookup(key);
+        if (maybe_arr && maybe_arr->is_array())
+        {
+            auto& arr = maybe_arr->as_array();
+            array_type out;
+            out.reserve(arr.size());
+
+            std::transform(
+                std::begin(arr),
+                std::end(arr),
+                std::back_inserter(out),
+                [](auto&& element) { return Config{std::move(element)}; });
+            return std::make_optional<array_type>(std::move(out));
+        }
+    }
+    catch (detail::StoreException const&)
+    {
+        // ignore store error, but rethrow key errors
+    }
+
+    return std::nullopt;
+}
+
+Config::array_type
+Config::array(key_type key) const
+{
+    if (auto maybe_arr = maybeArray(key); maybe_arr)
+        return maybe_arr.value();
+    throw std::logic_error("No array found at '" + key + "'");
+}
+
+Config::array_type
+Config::arrayOr(key_type key, array_type fallback) const
+{
+    if (auto maybe_arr = maybeArray(key); maybe_arr)
+        return maybe_arr.value();
+    return fallback;
+}
+
+Config::array_type
+Config::arrayOrThrow(key_type key, std::string_view err) const
+{
+    try
+    {
+        return maybeArray(key).value();
+    }
+    catch (std::exception const&)
+    {
+        throw std::runtime_error(err.data());
+    }
+}
+
+Config
+Config::section(key_type key) const
+{
+    auto maybe_element = lookup(key);
+    if (maybe_element && maybe_element->is_object())
+        return Config{std::move(*maybe_element)};
+    throw std::logic_error("No section found at '" + key + "'");
+}
+
+Config::array_type
+Config::array() const
+{
+    if (not store_.is_array())
+        throw std::logic_error("_self_ is not an array");
+
+    array_type out;
+    auto const& arr = store_.as_array();
+    out.reserve(arr.size());
+
+    std::transform(
+        std::cbegin(arr),
+        std::cend(arr),
+        std::back_inserter(out),
+        [](auto const& element) { return Config{element}; });
+    return out;
+}
+
+Config
+ConfigReader::open(std::filesystem::path path)
+{
+    try
+    {
+        std::ifstream in(path, std::ios::in | std::ios::binary);
+        if (in)
+        {
+            std::stringstream contents;
+            contents << in.rdbuf();
+            auto opts = boost::json::parse_options{};
+            opts.allow_comments = true;
+            return Config{boost::json::parse(contents.str(), {}, opts)};
+        }
+    }
+    catch (std::exception const& e)
+    {
+        LogService::error() << "Could not read configuration file from '"
+                            << path.string() << "': " << e.what();
+    }
+
+    return Config{};
+}
+
+}  // namespace clio

src/config/Config.h

@@ -0,0 +1,405 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include <config/detail/Helpers.h>
+
+#include <boost/json.hpp>
+#include <filesystem>
+#include <optional>
+#include <string>
+#include <string_view>
+
+namespace clio {
+
+/**
+ * @brief Convenience wrapper to query a JSON configuration file.
+ *
+ * Any custom data type can be supported by implementing the right `tag_invoke`
+ * for `boost::json::value_to`.
+ */
+class Config final
+{
+    boost::json::value store_;
+    static constexpr char Separator = '.';
+
+public:
+    using key_type = std::string;           /*! The type of key used */
+    using array_type = std::vector<Config>; /*! The type of array used */
+    using write_cursor_type = std::pair<
+        std::optional<std::reference_wrapper<boost::json::value>>,
+        key_type>;
+
+    /**
+     * @brief Construct a new Config object.
+     * @param store boost::json::value that backs this instance
+     */
+    explicit Config(boost::json::value store = {});
+
+    //
+    // Querying the store
+    //
+
+    /**
+     * @brief Checks whether underlying store is not null.
+     *
+     * @return true If the store is null
+     * @return false If the store is not null
+     */
+    operator bool() const noexcept;
+
+    /**
+     * @brief Checks whether something exists under given key.
+     *
+     * @param key The key to check
+     * @return true If something exists under key
+     * @return false If nothing exists under key
+     * @throws std::logic_error If the key is of invalid format
+     */
+    [[nodiscard]] bool
+    contains(key_type key) const;
+
+    //
+    // Key value access
+    //
+
+    /**
+     * @brief Interface for fetching values by key that returns std::optional.
+     *
+     * Will attempt to fetch the value under the desired key. If the value
+     * exists and can be represented by the desired type Result then it will be
+     * returned wrapped in an optional. If the value exists but the conversion
+     * to Result is not possible - a runtime_error will be thrown. If the value
+     * does not exist under the specified key - std::nullopt is returned.
+     *
+     * @tparam Result The desired return type
+     * @param key The key to check
+     * @return std::optional<Result> Optional value of desired type
+     * @throws std::logic_error Thrown if conversion to Result is not possible
+     * or key is of invalid format
+     */
+    template <typename Result>
+    [[nodiscard]] std::optional<Result>
+    maybeValue(key_type key) const
+    {
+        auto maybe_element = lookup(key);
+        if (maybe_element)
+            return std::make_optional<Result>(
+                checkedAs<Result>(key, *maybe_element));
+        return std::nullopt;
+    }
+
+    /**
+     * @brief Interface for fetching values by key.
+     *
+     * Will attempt to fetch the value under the desired key. If the value
+     * exists and can be represented by the desired type Result then it will be
+     * returned. If the value exists but the conversion
+     * to Result is not possible OR the value does not exist - a logic_error
+     * will be thrown.
+     *
+     * @tparam Result The desired return type
+     * @param key The key to check
+     * @return Result Value of desired type
+     * @throws std::logic_error Thrown if conversion to Result is not
+     * possible, value does not exist under specified key path or the key is of
+     * invalid format
+     */
+    template <typename Result>
+    [[nodiscard]] Result
+    value(key_type key) const
+    {
+        return maybeValue<Result>(key).value();
+    }
+
+    /**
+     * @brief Interface for fetching values by key with fallback.
+     *
+     * Will attempt to fetch the value under the desired key. If the value
+     * exists and can be represented by the desired type Result then it will be
+     * returned. If the value exists but the conversion
+     * to Result is not possible - a logic_error will be thrown. If the value
+     * does not exist under the specified key - user specified fallback is
+     * returned.
+     *
+     * @tparam Result The desired return type
+     * @param key The key to check
+     * @param fallback The fallback value
+     * @return Result Value of desired type
+     * @throws std::logic_error Thrown if conversion to Result is not possible
+     * or the key is of invalid format
+     */
+    template <typename Result>
+    [[nodiscard]] Result
+    valueOr(key_type key, Result fallback) const
+    {
+        try
+        {
+            return maybeValue<Result>(key).value_or(fallback);
+        }
+        catch (detail::StoreException const&)
+        {
+            return fallback;
+        }
+    }
+
+    /**
+     * @brief Interface for fetching values by key with custom error handling.
+     *
+     * Will attempt to fetch the value under the desired key. If the value
+     * exists and can be represented by the desired type Result then it will be
+     * returned. If the value exists but the conversion
+     * to Result is not possible OR the value does not exist - a runtime_error
+     * will be thrown with the user specified message.
+     *
+     * @tparam Result The desired return type
+     * @param key The key to check
+     * @param err The custom error message
+     * @return Result Value of desired type
+     * @throws std::runtime_error Thrown if conversion to Result is not possible
+     * or value does not exist under key
+     */
+    template <typename Result>
+    [[nodiscard]] Result
+    valueOrThrow(key_type key, std::string_view err) const
+    {
+        try
+        {
+            return maybeValue<Result>(key).value();
+        }
+        catch (std::exception const&)
+        {
+            throw std::runtime_error(err.data());
+        }
+    }
+
+    /**
+     * @brief Interface for fetching an array by key that returns std::optional.
+     *
+     * Will attempt to fetch an array under the desired key. If the array
+     * exists then it will be
+     * returned wrapped in an optional. If the array does not exist under the
+     * specified key - std::nullopt is returned.
+     *
+     * @param key The key to check
+     * @return std::optional<array_type> Optional array
+     * @throws std::logic_error Thrown if the key is of invalid format
+     */
+    [[nodiscard]] std::optional<array_type>
+    maybeArray(key_type key) const;
+
+    /**
+     * @brief Interface for fetching an array by key.
+     *
+     * Will attempt to fetch an array under the desired key. If the array
+     * exists then it will be
+     * returned. If the array does not exist under the
+     * specified key an std::logic_error is thrown.
+     *
+     * @param key The key to check
+     * @return array_type The array
+     * @throws std::logic_error Thrown if there is no array under the desired
+     * key or the key is of invalid format
+     */
+    [[nodiscard]] array_type
+    array(key_type key) const;
+
+    /**
+     * @brief Interface for fetching an array by key with fallback.
+     *
+     * Will attempt to fetch an array under the desired key. If the array
+     * exists then it will be returned.
+     * If the array does not exist or another type is stored under the desired
+     * key - user specified fallback is returned.
+     *
+     * @param key The key to check
+     * @param fallback The fallback array
+     * @return array_type The array
+     * @throws std::logic_error Thrown if the key is of invalid format
+     */
+    [[nodiscard]] array_type
+    arrayOr(key_type key, array_type fallback) const;
+
+    /**
+     * @brief Interface for fetching an array by key with custom error handling.
+     *
+     * Will attempt to fetch an array under the desired key. If the array
+     * exists then it will be returned.
+     * If the array does not exist or another type is stored under the desired
+     * key - std::runtime_error is thrown with the user specified error message.
+     *
+     * @param key The key to check
+     * @param err The custom error message
+     * @return array_type The array
+     * @throws std::runtime_error Thrown if there is no array under the desired
+     * key
+     */
+    [[nodiscard]] array_type
+    arrayOrThrow(key_type key, std::string_view err) const;
+
+    /**
+     * @brief Interface for fetching a sub section by key.
+     *
+     * Will attempt to fetch an entire section under the desired key and return
+     * it as a Config instance. If the section does not exist or another type is
+     * stored under the desired key - std::logic_error is thrown.
+     *
+     * @param key The key to check
+     * @return Config Section represented as a separate instance of Config
+     * @throws std::logic_error Thrown if there is no section under the
+     * desired key or the key is of invalid format
+     */
+    [[nodiscard]] Config
+    section(key_type key) const;
+
+    //
+    // Direct self-value access
+    //
+
+    /**
+     * @brief Interface for reading the value directly referred to by the
+     * instance. Wraps as std::optional.
+     *
+     * See @ref maybeValue(key_type) const for how this works.
+     */
+    template <typename Result>
+    [[nodiscard]] std::optional<Result>
+    maybeValue() const
+    {
+        if (store_.is_null())
+            return std::nullopt;
+        return std::make_optional<Result>(checkedAs<Result>("_self_", store_));
+    }
+
+    /**
+     * @brief Interface for reading the value directly referred to by the
+     * instance.
+     *
+     * See @ref value(key_type) const for how this works.
+     */
+    template <typename Result>
+    [[nodiscard]] Result
+    value() const
+    {
+        return maybeValue<Result>().value();
+    }
+
+    /**
+     * @brief Interface for reading the value directly referred to by the
+     * instance with user-specified fallback.
+     *
+     * See @ref valueOr(key_type, Result) const for how this works.
+     */
+    template <typename Result>
+    [[nodiscard]] Result
+    valueOr(Result fallback) const
+    {
+        return maybeValue<Result>().valueOr(fallback);
+    }
+
+    /**
+     * @brief Interface for reading the value directly referred to by the
+     * instance with user-specified error message.
+     *
+     * See @ref valueOrThrow(key_type, std::string_view) const for how this
+     * works.
+     */
+    template <typename Result>
+    [[nodiscard]] Result
+    valueOrThrow(std::string_view err) const
+    {
+        try
+        {
+            return maybeValue<Result>().value();
+        }
+        catch (std::exception const&)
+        {
+            throw std::runtime_error(err.data());
+        }
+    }
+
+    /**
+     * @brief Interface for reading the array directly referred to by the
+     * instance.
+     *
+     * See @ref array(key_type) const for how this works.
+     */
+    [[nodiscard]] array_type
+    array() const;
+
+private:
+    template <typename Return>
+    [[nodiscard]] Return
+    checkedAs(key_type key, boost::json::value const& value) const
+    {
+        auto has_error = false;
+        if constexpr (std::is_same_v<Return, bool>)
+        {
+            if (not value.is_bool())
+                has_error = true;
+        }
+        else if constexpr (std::is_same_v<Return, std::string>)
+        {
+            if (not value.is_string())
+                has_error = true;
+        }
+        else if constexpr (std::is_same_v<Return, double>)
+        {
+            if (not value.is_number())
+                has_error = true;
+        }
+        else if constexpr (
+            std::is_convertible_v<Return, uint64_t> ||
+            std::is_convertible_v<Return, int64_t>)
+        {
+            if (not value.is_int64() && not value.is_uint64())
+                has_error = true;
+        }
+
+        if (has_error)
+            throw std::runtime_error(
+                "Type for key '" + key + "' is '" +
+                std::string{to_string(value.kind())} +
+                "' in JSON but requested '" + detail::typeName<Return>() + "'");
+
+        return boost::json::value_to<Return>(value);
+    }
+
+    std::optional<boost::json::value>
+    lookup(key_type key) const;
+
+    write_cursor_type
+    lookupForWrite(key_type key);
+};
+
+/**
+ * @brief Simple configuration file reader.
+ *
+ * Reads the JSON file under specified path and creates a @ref Config object
+ * from its contents.
+ */
+class ConfigReader final
+{
+public:
+    static Config
+    open(std::filesystem::path path);
+};
+
+}  // namespace clio

src/config/detail/Helpers.h

@@ -0,0 +1,164 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include <optional>
+#include <queue>
+#include <stdexcept>
+#include <string>
+
+namespace clio::detail {
+
+/**
+ * @brief Thrown when a KeyPath related error occurs
+ */
+struct KeyException : public ::std::logic_error
+{
+    KeyException(::std::string msg) : ::std::logic_error{msg}
+    {
+    }
+};
+
+/**
+ * @brief Thrown when a Store (config's storage) related error occurs.
+ */
+struct StoreException : public ::std::logic_error
+{
+    StoreException(::std::string msg) : ::std::logic_error{msg}
+    {
+    }
+};
+
+/**
+ * @brief Simple string tokenizer. Used by @ref Config.
+ *
+ * @tparam KeyType The type of key to use
+ * @tparam Separator The separator character
+ */
+template <typename KeyType, char Separator>
+class Tokenizer final
+{
+    using opt_key_t = std::optional<KeyType>;
+    KeyType key_;
+    KeyType token_{};
+    std::queue<KeyType> tokens_{};
+
+public:
+    explicit Tokenizer(KeyType key) : key_{key}
+    {
+        if (key.empty())
+            throw KeyException("Empty key");
+
+        for (auto const& c : key)
+        {
+            if (c == Separator)
+                saveToken();
+            else
+                token_ += c;
+        }
+
+        saveToken();
+    }
+
+    [[nodiscard]] opt_key_t
+    next()
+    {
+        if (tokens_.empty())
+            return std::nullopt;
+        auto token = tokens_.front();
+        tokens_.pop();
+        return std::make_optional(std::move(token));
+    }
+
+private:
+    void
+    saveToken()
+    {
+        if (token_.empty())
+            throw KeyException("Empty token in key '" + key_ + "'.");
+        tokens_.push(std::move(token_));
+        token_ = {};
+    }
+};
+
+template <typename T>
+static constexpr const char*
+typeName()
+{
+    return typeid(T).name();
+}
+
+template <>
+constexpr const char*
+typeName<uint64_t>()
+{
+    return "uint64_t";
+}
+
+template <>
+constexpr const char*
+typeName<int64_t>()
+{
+    return "int64_t";
+}
+
+template <>
+constexpr const char*
+typeName<uint32_t>()
+{
+    return "uint32_t";
+}
+
+template <>
+constexpr const char*
+typeName<int32_t>()
+{
+    return "int32_t";
+}
+
+template <>
+constexpr const char*
+typeName<bool>()
+{
+    return "bool";
+}
+
+template <>
+constexpr const char*
+typeName<std::string>()
+{
+    return "std::string";
+}
+
+template <>
+constexpr const char*
+typeName<const char*>()
+{
+    return "const char*";
+}
+
+template <>
+constexpr const char*
+typeName<double>()
+{
+    return "double";
+}
+
+};  // namespace clio::detail

src/etl/ETLHelpers.h

@@ -1,5 +1,24 @@
-#ifndef RIPPLE_APP_REPORTING_ETLHELPERS_H_INCLUDED
-#define RIPPLE_APP_REPORTING_ETLHELPERS_H_INCLUDED
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
 #include <ripple/basics/base_uint.h>
 #include <condition_variable>
 #include <mutex>
@@ -140,7 +159,7 @@ public:
     std::optional<T>
     tryPop()
     {
-        std::unique_lock lck(m_);
+        std::scoped_lock lck(m_);
         if (queue_.empty())
             return {};
         T ret = std::move(queue_.front());
@@ -171,5 +190,3 @@ getMarkers(size_t numMarkers)
     }
     return markers;
 }
-
-#endif  // RIPPLE_APP_REPORTING_ETLHELPERS_H_INCLUDED

src/etl/ETLSource.cpp

@@ -1,3 +1,22 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
 #include <ripple/beast/net/IPEndpoint.h>
 #include <ripple/protocol/STLedgerEntry.h>
 #include <boost/asio/strand.hpp>
@@ -5,18 +24,23 @@
 #include <boost/beast/ssl.hpp>
 #include <boost/json.hpp>
 #include <boost/json/src.hpp>
-#include <boost/log/trivial.hpp>
+
 #include <backend/DBHelpers.h>
 #include <etl/ETLSource.h>
 #include <etl/ProbingETLSource.h>
 #include <etl/ReportingETL.h>
+#include <log/Logger.h>
 #include <rpc/RPCHelpers.h>
+#include <util/Profiler.h>
+
 #include <thread>
 
+using namespace clio;
+
 void
 ForwardCache::freshen()
 {
-    BOOST_LOG_TRIVIAL(trace) << "Freshening ForwardCache";
+    log_.trace() << "Freshening ForwardCache";
 
     auto numOutstanding =
         std::make_shared<std::atomic_uint>(latestForwarded_.size());
@@ -34,7 +58,7 @@ ForwardCache::freshen()
                     resp = {};
 
                 {
-                    std::unique_lock lk(mtx_);
+                    std::scoped_lock lk(mtx_);
                     latestForwarded_[command] = resp;
                 }
             });
@@ -44,7 +68,7 @@ ForwardCache::freshen()
 void
 ForwardCache::clear()
 {
-    std::unique_lock lk(mtx_);
+    std::scoped_lock lk(mtx_);
     for (auto& cacheEntry : latestForwarded_)
         latestForwarded_[cacheEntry.first] = {};
 }
@@ -127,15 +151,11 @@ ETLSourceImpl<Derived>::reconnect(boost::beast::error_code ec)
     if (ec != boost::asio::error::operation_aborted &&
         ec != boost::asio::error::connection_refused)
     {
-        BOOST_LOG_TRIVIAL(error)
-            << __func__ << " : "
-            << "error code = " << ec << " - " << toString();
+        log_.error() << "error code = " << ec << " - " << toString();
     }
     else
     {
-        BOOST_LOG_TRIVIAL(warning)
-            << __func__ << " : "
-            << "error code = " << ec << " - " << toString();
+        log_.warn() << "error code = " << ec << " - " << toString();
     }
 
     // exponentially increasing timeouts, with a max of 30 seconds
@@ -144,7 +164,7 @@ ETLSourceImpl<Derived>::reconnect(boost::beast::error_code ec)
     timer_.expires_after(boost::asio::chrono::seconds(waitTime));
     timer_.async_wait([this](auto ec) {
         bool startAgain = (ec != boost::asio::error::operation_aborted);
-        BOOST_LOG_TRIVIAL(trace) << __func__ << " async_wait : ec = " << ec;
+        log_.trace() << "async_wait : ec = " << ec;
         derived().close(startAgain);
     });
 }
@@ -168,8 +188,8 @@ PlainETLSource::close(bool startAgain)
                 [this, startAgain](auto ec) {
                     if (ec)
                     {
-                        BOOST_LOG_TRIVIAL(error)
-                            << __func__ << " async_close : "
+                        log_.error()
+                            << " async_close : "
                             << "error code = " << ec << " - " << toString();
                     }
                     closing_ = false;
@@ -213,8 +233,8 @@ SslETLSource::close(bool startAgain)
                 [this, startAgain](auto ec) {
                     if (ec)
                     {
-                        BOOST_LOG_TRIVIAL(error)
-                            << __func__ << " async_close : "
+                        log_.error()
+                            << " async_close : "
                             << "error code = " << ec << " - " << toString();
                     }
                     closing_ = false;
@@ -246,8 +266,7 @@ ETLSourceImpl<Derived>::onResolve(
     boost::beast::error_code ec,
     boost::asio::ip::tcp::resolver::results_type results)
 {
-    BOOST_LOG_TRIVIAL(trace)
-        << __func__ << " : ec = " << ec << " - " << toString();
+    log_.trace() << "ec = " << ec << " - " << toString();
     if (ec)
     {
         // try again
@@ -269,8 +288,7 @@ PlainETLSource::onConnect(
     boost::beast::error_code ec,
     boost::asio::ip::tcp::resolver::results_type::endpoint_type endpoint)
 {
-    BOOST_LOG_TRIVIAL(trace)
-        << __func__ << " : ec = " << ec << " - " << toString();
+    log_.trace() << "ec = " << ec << " - " << toString();
     if (ec)
     {
         // start over
@@ -311,8 +329,7 @@ SslETLSource::onConnect(
     boost::beast::error_code ec,
     boost::asio::ip::tcp::resolver::results_type::endpoint_type endpoint)
 {
-    BOOST_LOG_TRIVIAL(trace)
-        << __func__ << " : ec = " << ec << " - " << toString();
+    log_.trace() << "ec = " << ec << " - " << toString();
     if (ec)
     {
         // start over
@@ -371,8 +388,7 @@ template <class Derived>
 void
 ETLSourceImpl<Derived>::onHandshake(boost::beast::error_code ec)
 {
-    BOOST_LOG_TRIVIAL(trace)
-        << __func__ << " : ec = " << ec << " - " << toString();
+    log_.trace() << "ec = " << ec << " - " << toString();
     if (auto action = hooks_.onConnected(ec);
         action == ETLSourceHooks::Action::STOP)
         return;
@@ -389,7 +405,7 @@ ETLSourceImpl<Derived>::onHandshake(boost::beast::error_code ec)
             {"streams",
              {"ledger", "manifests", "validations", "transactions_proposed"}}};
         std::string s = boost::json::serialize(jv);
-        BOOST_LOG_TRIVIAL(trace) << "Sending subscribe stream message";
+        log_.trace() << "Sending subscribe stream message";
 
         derived().ws().set_option(
             boost::beast::websocket::stream_base::decorator(
@@ -415,8 +431,7 @@ ETLSourceImpl<Derived>::onWrite(
     boost::beast::error_code ec,
     size_t bytesWritten)
 {
-    BOOST_LOG_TRIVIAL(trace)
-        << __func__ << " : ec = " << ec << " - " << toString();
+    log_.trace() << "ec = " << ec << " - " << toString();
     if (ec)
     {
         // start over
@@ -433,8 +448,7 @@ template <class Derived>
 void
 ETLSourceImpl<Derived>::onRead(boost::beast::error_code ec, size_t size)
 {
-    BOOST_LOG_TRIVIAL(trace)
-        << __func__ << " : ec = " << ec << " - " << toString();
+    log_.trace() << "ec = " << ec << " - " << toString();
     // if error or error reading message, start over
     if (ec)
     {
@@ -446,8 +460,7 @@ ETLSourceImpl<Derived>::onRead(boost::beast::error_code ec, size_t size)
         boost::beast::flat_buffer buffer;
         swap(readBuffer_, buffer);
 
-        BOOST_LOG_TRIVIAL(trace)
-            << __func__ << " : calling async_read - " << toString();
+        log_.trace() << "calling async_read - " << toString();
         derived().ws().async_read(
             readBuffer_, [this](auto ec, size_t size) { onRead(ec, size); });
     }
@@ -457,7 +470,7 @@ template <class Derived>
 bool
 ETLSourceImpl<Derived>::handleMessage()
 {
-    BOOST_LOG_TRIVIAL(trace) << __func__ << " : " << toString();
+    log_.trace() << toString();
 
     setLastMsgTime();
     connected_ = true;
@@ -466,9 +479,9 @@ ETLSourceImpl<Derived>::handleMessage()
         std::string msg{
             static_cast<char const*>(readBuffer_.data().data()),
             readBuffer_.size()};
-        BOOST_LOG_TRIVIAL(trace) << __func__ << msg;
+        log_.trace() << msg;
         boost::json::value raw = boost::json::parse(msg);
-        BOOST_LOG_TRIVIAL(trace) << __func__ << " parsed";
+        log_.trace() << "parsed";
         boost::json::object response = raw.as_object();
 
         uint32_t ledgerIndex = 0;
@@ -487,20 +500,16 @@ ETLSourceImpl<Derived>::handleMessage()
                 setValidatedRange(
                     {validatedLedgers.c_str(), validatedLedgers.size()});
             }
-            BOOST_LOG_TRIVIAL(debug)
-                << __func__ << " : "
-                << "Received a message on ledger "
-                << " subscription stream. Message : " << response << " - "
-                << toString();
+            log_.info() << "Received a message on ledger "
+                        << " subscription stream. Message : " << response
+                        << " - " << toString();
         }
         else if (
             response.contains("type") && response["type"] == "ledgerClosed")
         {
-            BOOST_LOG_TRIVIAL(debug)
-                << __func__ << " : "
-                << "Received a message on ledger "
-                << " subscription stream. Message : " << response << " - "
-                << toString();
+            log_.info() << "Received a message on ledger "
+                        << " subscription stream. Message : " << response
+                        << " - " << toString();
             if (response.contains("ledger_index"))
             {
                 ledgerIndex = response["ledger_index"].as_int64();
@@ -539,23 +548,23 @@ ETLSourceImpl<Derived>::handleMessage()
 
         if (ledgerIndex != 0)
         {
-            BOOST_LOG_TRIVIAL(trace)
-                << __func__ << " : "
-                << "Pushing ledger sequence = " << ledgerIndex << " - "
-                << toString();
+            log_.trace() << "Pushing ledger sequence = " << ledgerIndex << " - "
+                         << toString();
             networkValidatedLedgers_->push(ledgerIndex);
         }
         return true;
     }
     catch (std::exception const& e)
     {
-        BOOST_LOG_TRIVIAL(error) << "Exception in handleMessage : " << e.what();
+        log_.error() << "Exception in handleMessage : " << e.what();
         return false;
     }
 }
 
 class AsyncCallData
 {
+    clio::Logger log_{"ETL"};
+
     std::unique_ptr<org::xrpl::rpc::v1::GetLedgerDataResponse> cur_;
     std::unique_ptr<org::xrpl::rpc::v1::GetLedgerDataResponse> next_;
 
@@ -585,11 +594,11 @@ public:
 
         unsigned char prefix = marker.data()[0];
 
-        BOOST_LOG_TRIVIAL(debug)
-            << "Setting up AsyncCallData. marker = " << ripple::strHex(marker)
-            << " . prefix = " << ripple::strHex(std::string(1, prefix))
-            << " . nextPrefix_ = "
-            << ripple::strHex(std::string(1, nextPrefix_));
+        log_.debug() << "Setting up AsyncCallData. marker = "
+                     << ripple::strHex(marker)
+                     << " . prefix = " << ripple::strHex(std::string(1, prefix))
+                     << " . nextPrefix_ = "
+                     << ripple::strHex(std::string(1, nextPrefix_));
 
         assert(nextPrefix_ > prefix || nextPrefix_ == 0x00);
 
@@ -609,26 +618,24 @@ public:
         bool abort,
         bool cacheOnly = false)
     {
-        BOOST_LOG_TRIVIAL(trace) << "Processing response. "
-                                 << "Marker prefix = " << getMarkerPrefix();
+        log_.trace() << "Processing response. "
+                     << "Marker prefix = " << getMarkerPrefix();
         if (abort)
         {
-            BOOST_LOG_TRIVIAL(error) << "AsyncCallData aborted";
+            log_.error() << "AsyncCallData aborted";
             return CallStatus::ERRORED;
         }
         if (!status_.ok())
         {
-            BOOST_LOG_TRIVIAL(error)
-                << "AsyncCallData status_ not ok: "
-                << " code = " << status_.error_code()
-                << " message = " << status_.error_message();
+            log_.error() << "AsyncCallData status_ not ok: "
+                         << " code = " << status_.error_code()
+                         << " message = " << status_.error_message();
             return CallStatus::ERRORED;
         }
         if (!next_->is_unlimited())
         {
-            BOOST_LOG_TRIVIAL(warning)
-                << "AsyncCallData is_unlimited is false. Make sure "
-                   "secure_gateway is set correctly at the ETL source";
+            log_.warn() << "AsyncCallData is_unlimited is false. Make sure "
+                           "secure_gateway is set correctly at the ETL source";
         }
 
         std::swap(cur_, next_);
@@ -651,7 +658,7 @@ public:
             call(stub, cq);
         }
 
-        BOOST_LOG_TRIVIAL(trace) << "Writing objects";
+        log_.trace() << "Writing objects";
         std::vector<Backend::LedgerObject> cacheUpdates;
         cacheUpdates.reserve(cur_->ledger_objects().objects_size());
         for (int i = 0; i < cur_->ledger_objects().objects_size(); ++i)
@@ -681,7 +688,7 @@ public:
         }
         backend.cache().update(
             cacheUpdates, request_.ledger().sequence(), cacheOnly);
-        BOOST_LOG_TRIVIAL(trace) << "Wrote objects";
+        log_.trace() << "Wrote objects";
 
         return more ? CallStatus::MORE : CallStatus::DONE;
     }
@@ -745,8 +752,8 @@ ETLSourceImpl<Derived>::loadInitialLedger(
         calls.emplace_back(sequence, markers[i], nextMarker);
     }
 
-    BOOST_LOG_TRIVIAL(debug) << "Starting data download for ledger " << sequence
-                             << ". Using source = " << toString();
+    log_.debug() << "Starting data download for ledger " << sequence
+                 << ". Using source = " << toString();
 
     for (auto& c : calls)
         c.call(stub_, cq);
@@ -764,21 +771,19 @@ ETLSourceImpl<Derived>::loadInitialLedger(
 
         if (!ok)
         {
-            BOOST_LOG_TRIVIAL(error) << "loadInitialLedger - ok is false";
+            log_.error() << "loadInitialLedger - ok is false";
             return false;
             // handle cancelled
         }
         else
         {
-            BOOST_LOG_TRIVIAL(trace)
-                << "Marker prefix = " << ptr->getMarkerPrefix();
+            log_.trace() << "Marker prefix = " << ptr->getMarkerPrefix();
             auto result = ptr->process(stub_, cq, *backend_, abort, cacheOnly);
             if (result != AsyncCallData::CallStatus::MORE)
             {
                 numFinished++;
-                BOOST_LOG_TRIVIAL(debug)
-                    << "Finished a marker. "
-                    << "Current number of finished = " << numFinished;
+                log_.debug() << "Finished a marker. "
+                             << "Current number of finished = " << numFinished;
                 std::string lastKey = ptr->getLastKey();
                 if (lastKey.size())
                     edgeKeys.push_back(ptr->getLastKey());
@@ -789,89 +794,84 @@ ETLSourceImpl<Derived>::loadInitialLedger(
             }
             if (backend_->cache().size() > progress)
             {
-                BOOST_LOG_TRIVIAL(info)
-                    << "Downloaded " << backend_->cache().size()
-                    << " records from rippled";
+                log_.info() << "Downloaded " << backend_->cache().size()
+                            << " records from rippled";
                 progress += incr;
             }
         }
     }
-    BOOST_LOG_TRIVIAL(info)
-        << __func__ << " - finished loadInitialLedger. cache size = "
-        << backend_->cache().size();
+    log_.info() << "Finished loadInitialLedger. cache size = "
+                << backend_->cache().size();
     size_t numWrites = 0;
     if (!abort)
     {
         backend_->cache().setFull();
         if (!cacheOnly)
         {
-            auto start = std::chrono::system_clock::now();
-            for (auto& key : edgeKeys)
-            {
-                BOOST_LOG_TRIVIAL(debug)
-                    << __func__
-                    << " writing edge key = " << ripple::strHex(key);
-                auto succ = backend_->cache().getSuccessor(
-                    *ripple::uint256::fromVoidChecked(key), sequence);
-                if (succ)
-                    backend_->writeSuccessor(
-                        std::move(key), sequence, uint256ToString(succ->key));
-            }
-            ripple::uint256 prev = Backend::firstKey;
-            while (auto cur = backend_->cache().getSuccessor(prev, sequence))
-            {
-                assert(cur);
-                if (prev == Backend::firstKey)
+            auto seconds = util::timed<std::chrono::seconds>([&]() {
+                for (auto& key : edgeKeys)
                 {
-                    backend_->writeSuccessor(
-                        uint256ToString(prev),
-                        sequence,
-                        uint256ToString(cur->key));
+                    log_.debug()
+                        << "Writing edge key = " << ripple::strHex(key);
+                    auto succ = backend_->cache().getSuccessor(
+                        *ripple::uint256::fromVoidChecked(key), sequence);
+                    if (succ)
+                        backend_->writeSuccessor(
+                            std::move(key),
+                            sequence,
+                            uint256ToString(succ->key));
                 }
-
-                if (isBookDir(cur->key, cur->blob))
+                ripple::uint256 prev = Backend::firstKey;
+                while (auto cur =
+                           backend_->cache().getSuccessor(prev, sequence))
                 {
-                    auto base = getBookBase(cur->key);
-                    // make sure the base is not an actual object
-                    if (!backend_->cache().get(cur->key, sequence))
+                    assert(cur);
+                    if (prev == Backend::firstKey)
                     {
-                        auto succ =
-                            backend_->cache().getSuccessor(base, sequence);
-                        assert(succ);
-                        if (succ->key == cur->key)
-                        {
-                            BOOST_LOG_TRIVIAL(debug)
-                                << __func__ << " Writing book successor = "
-                                << ripple::strHex(base) << " - "
-                                << ripple::strHex(cur->key);
-
-                            backend_->writeSuccessor(
-                                uint256ToString(base),
-                                sequence,
-                                uint256ToString(cur->key));
-                        }
+                        backend_->writeSuccessor(
+                            uint256ToString(prev),
+                            sequence,
+                            uint256ToString(cur->key));
                     }
-                    ++numWrites;
+
+                    if (isBookDir(cur->key, cur->blob))
+                    {
+                        auto base = getBookBase(cur->key);
+                        // make sure the base is not an actual object
+                        if (!backend_->cache().get(cur->key, sequence))
+                        {
+                            auto succ =
+                                backend_->cache().getSuccessor(base, sequence);
+                            assert(succ);
+                            if (succ->key == cur->key)
+                            {
+                                log_.debug() << "Writing book successor = "
+                                             << ripple::strHex(base) << " - "
+                                             << ripple::strHex(cur->key);
+
+                                backend_->writeSuccessor(
+                                    uint256ToString(base),
+                                    sequence,
+                                    uint256ToString(cur->key));
+                            }
+                        }
+                        ++numWrites;
+                    }
+                    prev = std::move(cur->key);
+                    if (numWrites % 100000 == 0 && numWrites != 0)
+                        log_.info()
+                            << "Wrote " << numWrites << " book successors";
                 }
-                prev = std::move(cur->key);
-                if (numWrites % 100000 == 0 && numWrites != 0)
-                    BOOST_LOG_TRIVIAL(info) << __func__ << " Wrote "
-                                            << numWrites << " book successors";
-            }
 
-            backend_->writeSuccessor(
-                uint256ToString(prev),
-                sequence,
-                uint256ToString(Backend::lastKey));
+                backend_->writeSuccessor(
+                    uint256ToString(prev),
+                    sequence,
+                    uint256ToString(Backend::lastKey));
 
-            ++numWrites;
-            auto end = std::chrono::system_clock::now();
-            auto seconds =
-                std::chrono::duration_cast<std::chrono::seconds>(end - start)
-                    .count();
-            BOOST_LOG_TRIVIAL(info)
-                << __func__
-                << " - Looping through cache and submitting all writes took "
+                ++numWrites;
+            });
+            log_.info()
+                << "Looping through cache and submitting all writes took "
                 << seconds
                 << " seconds. numWrites = " << std::to_string(numWrites);
         }
@@ -902,18 +902,17 @@ ETLSourceImpl<Derived>::fetchLedger(
     grpc::Status status = stub_->GetLedger(&context, request, &response);
     if (status.ok() && !response.is_unlimited())
     {
-        BOOST_LOG_TRIVIAL(warning)
-            << "ETLSourceImpl::fetchLedger - is_unlimited is "
-               "false. Make sure secure_gateway is set "
-               "correctly on the ETL source. source = "
-            << toString() << " status = " << status.error_message();
+        log_.warn() << "ETLSourceImpl::fetchLedger - is_unlimited is "
+                       "false. Make sure secure_gateway is set "
+                       "correctly on the ETL source. source = "
+                    << toString() << " status = " << status.error_message();
     }
     return {status, std::move(response)};
 }
 
 static std::unique_ptr<ETLSource>
 make_ETLSource(
-    boost::json::object const& config,
+    clio::Config const& config,
     boost::asio::io_context& ioContext,
     std::shared_ptr<BackendInterface> backend,
     std::shared_ptr<SubscriptionManager> subscriptions,
@@ -934,31 +933,24 @@ make_ETLSource(
 }
 
 ETLLoadBalancer::ETLLoadBalancer(
-    boost::json::object const& config,
+    clio::Config const& config,
     boost::asio::io_context& ioContext,
     std::shared_ptr<BackendInterface> backend,
     std::shared_ptr<SubscriptionManager> subscriptions,
     std::shared_ptr<NetworkValidatedLedgers> nwvl)
 {
-    if (config.contains("num_markers") && config.at("num_markers").is_int64())
-    {
-        downloadRanges_ = config.at("num_markers").as_int64();
-
-        downloadRanges_ = std::clamp(downloadRanges_, {1}, {256});
-    }
+    if (auto value = config.maybeValue<uint32_t>("num_markers"); value)
+        downloadRanges_ = std::clamp(*value, 1u, 256u);
     else if (backend->fetchLedgerRange())
-    {
         downloadRanges_ = 4;
-    }
 
-    for (auto& entry : config.at("etl_sources").as_array())
+    for (auto const& entry : config.array("etl_sources"))
     {
         std::unique_ptr<ETLSource> source = make_ETLSource(
-            entry.as_object(), ioContext, backend, subscriptions, nwvl, *this);
+            entry, ioContext, backend, subscriptions, nwvl, *this);
 
         sources_.push_back(std::move(source));
-        BOOST_LOG_TRIVIAL(info) << __func__ << " : added etl source - "
-                                << sources_.back()->toString();
+        log_.info() << "Added etl source - " << sources_.back()->toString();
     }
 }
 
@@ -971,9 +963,9 @@ ETLLoadBalancer::loadInitialLedger(uint32_t sequence, bool cacheOnly)
                 source->loadInitialLedger(sequence, downloadRanges_, cacheOnly);
             if (!res)
             {
-                BOOST_LOG_TRIVIAL(error) << "Failed to download initial ledger."
-                                         << " Sequence = " << sequence
-                                         << " source = " << source->toString();
+                log_.error() << "Failed to download initial ledger."
+                             << " Sequence = " << sequence
+                             << " source = " << source->toString();
             }
             return res;
         },
@@ -988,26 +980,24 @@ ETLLoadBalancer::fetchLedger(
 {
     org::xrpl::rpc::v1::GetLedgerResponse response;
     bool success = execute(
-        [&response, ledgerSequence, getObjects, getObjectNeighbors](
+        [&response, ledgerSequence, getObjects, getObjectNeighbors, log = log_](
             auto& source) {
             auto [status, data] = source->fetchLedger(
                 ledgerSequence, getObjects, getObjectNeighbors);
             response = std::move(data);
             if (status.ok() && response.validated())
             {
-                BOOST_LOG_TRIVIAL(info)
-                    << "Successfully fetched ledger = " << ledgerSequence
-                    << " from source = " << source->toString();
+                log.info() << "Successfully fetched ledger = " << ledgerSequence
+                           << " from source = " << source->toString();
                 return true;
             }
             else
             {
-                BOOST_LOG_TRIVIAL(warning)
-                    << "Error getting ledger = " << ledgerSequence
-                    << " Reply : " << response.DebugString()
-                    << " error_code : " << status.error_code()
-                    << " error_msg : " << status.error_message()
-                    << " source = " << source->toString();
+                log.warn() << "Could not fetch ledger " << ledgerSequence
+                           << ", Reply: " << response.DebugString()
+                           << ", error_code: " << status.error_code()
+                           << ", error_msg: " << status.error_message()
+                           << ", source = " << source->toString();
                 return false;
             }
         },
@@ -1048,7 +1038,7 @@ ETLSourceImpl<Derived>::forwardToRippled(
 {
     if (auto resp = forwardCache_.get(request); resp)
     {
-        BOOST_LOG_TRIVIAL(debug) << "request hit forwardCache";
+        log_.debug() << "request hit forwardCache";
         return resp;
     }
 
@@ -1062,14 +1052,13 @@ ETLSourceImpl<Derived>::requestFromRippled(
     std::string const& clientIp,
     boost::asio::yield_context& yield) const
 {
-    BOOST_LOG_TRIVIAL(trace) << "Attempting to forward request to tx. "
-                             << "request = " << boost::json::serialize(request);
+    log_.trace() << "Attempting to forward request to tx. "
+                 << "request = " << boost::json::serialize(request);
 
     boost::json::object response;
     if (!connected_)
     {
-        BOOST_LOG_TRIVIAL(error)
-            << "Attempted to proxy but failed to connect to tx";
+        log_.error() << "Attempted to proxy but failed to connect to tx";
         return {};
     }
     namespace beast = boost::beast;          // from <boost/beast.hpp>
@@ -1083,7 +1072,7 @@ ETLSourceImpl<Derived>::requestFromRippled(
         // These objects perform our I/O
         tcp::resolver resolver{ioc_};
 
-        BOOST_LOG_TRIVIAL(trace) << "Creating websocket";
+        log_.trace() << "Creating websocket";
         auto ws = std::make_unique<websocket::stream<beast::tcp_stream>>(ioc_);
 
         // Look up the domain name
@@ -1093,7 +1082,7 @@ ETLSourceImpl<Derived>::requestFromRippled(
 
         ws->next_layer().expires_after(std::chrono::seconds(3));
 
-        BOOST_LOG_TRIVIAL(trace) << "Connecting websocket";
+        log_.trace() << "Connecting websocket";
         // Make the connection on the IP address we get from a lookup
         ws->next_layer().async_connect(results, yield[ec]);
         if (ec)
@@ -1112,15 +1101,15 @@ ETLSourceImpl<Derived>::requestFromRippled(
                         " websocket-client-coro");
                 req.set(http::field::forwarded, "for=" + clientIp);
             }));
-        BOOST_LOG_TRIVIAL(trace) << "client ip: " << clientIp;
+        log_.trace() << "client ip: " << clientIp;
 
-        BOOST_LOG_TRIVIAL(trace) << "Performing websocket handshake";
+        log_.trace() << "Performing websocket handshake";
         // Perform the websocket handshake
         ws->async_handshake(ip_, "/", yield[ec]);
         if (ec)
             return {};
 
-        BOOST_LOG_TRIVIAL(trace) << "Sending request";
+        log_.trace() << "Sending request";
         // Send the message
         ws->async_write(
             net::buffer(boost::json::serialize(request)), yield[ec]);
@@ -1138,11 +1127,11 @@ ETLSourceImpl<Derived>::requestFromRippled(
 
         if (!parsed.is_object())
         {
-            BOOST_LOG_TRIVIAL(error)
-                << "Error parsing response: " << std::string{begin, end};
+            log_.error() << "Error parsing response: "
+                         << std::string{begin, end};
             return {};
         }
-        BOOST_LOG_TRIVIAL(trace) << "Successfully forward request";
+        log_.trace() << "Successfully forward request";
 
         response = parsed.as_object();
 
@@ -1151,7 +1140,7 @@ ETLSourceImpl<Derived>::requestFromRippled(
     }
     catch (std::exception const& e)
     {
-        BOOST_LOG_TRIVIAL(error) << "Encountered exception : " << e.what();
+        log_.error() << "Encountered exception : " << e.what();
         return {};
     }
 }
@@ -1168,47 +1157,42 @@ ETLLoadBalancer::execute(Func f, uint32_t ledgerSequence)
     {
         auto& source = sources_[sourceIdx];
 
-        BOOST_LOG_TRIVIAL(debug)
-            << __func__ << " : "
-            << "Attempting to execute func. ledger sequence = "
-            << ledgerSequence << " - source = " << source->toString();
-        if (source->hasLedger(ledgerSequence) || true)
+        log_.debug() << "Attempting to execute func. ledger sequence = "
+                     << ledgerSequence << " - source = " << source->toString();
+        // Originally, it was (source->hasLedger(ledgerSequence) || true)
+        /* Sometimes rippled has ledger but doesn't actually know. However,
+        but this does NOT happen in the normal case and is safe to remove
+        This || true is only needed when loading full history standalone */
+        if (source->hasLedger(ledgerSequence))
         {
             bool res = f(source);
             if (res)
             {
-                BOOST_LOG_TRIVIAL(debug)
-                    << __func__ << " : "
-                    << "Successfully executed func at source = "
-                    << source->toString()
-                    << " - ledger sequence = " << ledgerSequence;
+                log_.debug() << "Successfully executed func at source = "
+                             << source->toString()
+                             << " - ledger sequence = " << ledgerSequence;
                 break;
             }
             else
             {
-                BOOST_LOG_TRIVIAL(warning)
-                    << __func__ << " : "
-                    << "Failed to execute func at source = "
-                    << source->toString()
-                    << " - ledger sequence = " << ledgerSequence;
+                log_.warn() << "Failed to execute func at source = "
+                            << source->toString()
+                            << " - ledger sequence = " << ledgerSequence;
             }
         }
         else
         {
-            BOOST_LOG_TRIVIAL(warning)
-                << __func__ << " : "
-                << "Ledger not present at source = " << source->toString()
-                << " - ledger sequence = " << ledgerSequence;
+            log_.warn() << "Ledger not present at source = "
+                        << source->toString()
+                        << " - ledger sequence = " << ledgerSequence;
         }
         sourceIdx = (sourceIdx + 1) % sources_.size();
         numAttempts++;
         if (numAttempts % sources_.size() == 0)
         {
-            BOOST_LOG_TRIVIAL(error)
-                << __func__ << " : "
-                << "Error executing function "
-                << " - ledger sequence = " << ledgerSequence
-                << " - Tried all sources. Sleeping and trying again";
+            log_.info() << "Ledger sequence " << ledgerSequence
+                        << " is not yet available from any configured sources. "
+                        << "Sleeping and trying again";
             std::this_thread::sleep_for(std::chrono::seconds(2));
         }
     }

src/etl/ETLSource.h

@@ -1,5 +1,32 @@
-#ifndef RIPPLE_APP_REPORTING_ETLSOURCE_H_INCLUDED
-#define RIPPLE_APP_REPORTING_ETLSOURCE_H_INCLUDED
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include <backend/BackendInterface.h>
+#include <config/Config.h>
+#include <etl/ETLHelpers.h>
+#include <log/Logger.h>
+#include <subscriptions/SubscriptionManager.h>
+
+#include "org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h"
+#include <grpcpp/grpcpp.h>
 
 #include <boost/algorithm/string.hpp>
 #include <boost/asio.hpp>
@@ -7,12 +34,6 @@
 #include <boost/beast/core/string.hpp>
 #include <boost/beast/ssl.hpp>
 #include <boost/beast/websocket.hpp>
-#include <backend/BackendInterface.h>
-#include <subscriptions/SubscriptionManager.h>
-
-#include "org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h"
-#include <etl/ETLHelpers.h>
-#include <grpcpp/grpcpp.h>
 
 class ETLLoadBalancer;
 class ETLSource;
@@ -30,6 +51,7 @@ class ForwardCache
 {
     using response_type = std::optional<boost::json::object>;
 
+    clio::Logger log_{"ETL"};
     mutable std::atomic_bool stopping_ = false;
     mutable std::shared_mutex mtx_;
     std::unordered_map<std::string, response_type> latestForwarded_;
@@ -44,33 +66,27 @@ class ForwardCache
 
 public:
     ForwardCache(
-        boost::json::object const& config,
+        clio::Config const& config,
         boost::asio::io_context& ioc,
         ETLSource const& source)
         : strand_(ioc), timer_(strand_), source_(source)
     {
-        if (config.contains("cache") && !config.at("cache").is_array())
-            throw std::runtime_error("ETLSource cache must be array");
-
-        if (config.contains("cache_duration") &&
-            !config.at("cache_duration").is_int64())
-            throw std::runtime_error(
-                "ETLSource cache_duration must be a number");
-
-        duration_ = config.contains("cache_duration")
-            ? config.at("cache_duration").as_int64()
-            : 10;
-
-        auto commands = config.contains("cache") ? config.at("cache").as_array()
-                                                 : boost::json::array{};
-
-        for (auto const& command : commands)
+        if (config.contains("cache"))
         {
-            if (!command.is_string())
-                throw std::runtime_error(
-                    "ETLSource forward command must be array of strings");
+            auto commands =
+                config.arrayOrThrow("cache", "ETLSource cache must be array");
 
-            latestForwarded_[command.as_string().c_str()] = {};
+            if (config.contains("cache_duration"))
+                duration_ = config.valueOrThrow<uint32_t>(
+                    "cache_duration",
+                    "ETLSource cache_duration must be a number");
+
+            for (auto const& command : commands)
+            {
+                auto key = command.valueOrThrow<std::string>(
+                    "ETLSource forward command must be array of strings");
+                latestForwarded_[key] = {};
+            }
         }
     }
 
@@ -131,6 +147,9 @@ public:
     {
     }
 
+protected:
+    clio::Logger log_{"ETL"};
+
 private:
     friend ForwardCache;
     friend ProbingETLSource;
@@ -165,7 +184,7 @@ class ETLSourceImpl : public ETLSource
 
     std::vector<std::pair<uint32_t, uint32_t>> validatedLedgers_;
 
-    std::string validatedLedgersRaw_;
+    std::string validatedLedgersRaw_{"N/A"};
 
     std::shared_ptr<NetworkValidatedLedgers> networkValidatedLedgers_;
 
@@ -222,7 +241,7 @@ protected:
     void
     run() override
     {
-        BOOST_LOG_TRIVIAL(trace) << __func__ << " : " << toString();
+        log_.trace() << toString();
 
         auto const host = ip_;
         auto const port = wsPort_;
@@ -262,7 +281,7 @@ public:
     /// Fetch ledger and load initial ledger will fail for this source
     /// Primarly used in read-only mode, to monitor when ledgers are validated
     ETLSourceImpl(
-        boost::json::object const& config,
+        clio::Config const& config,
         boost::asio::io_context& ioContext,
         std::shared_ptr<BackendInterface> backend,
         std::shared_ptr<SubscriptionManager> subscriptions,
@@ -279,20 +298,12 @@ public:
         , timer_(ioContext)
         , hooks_(hooks)
     {
-        if (config.contains("ip"))
+        ip_ = config.valueOr<std::string>("ip", {});
+        wsPort_ = config.valueOr<std::string>("ws_port", {});
+
+        if (auto value = config.maybeValue<std::string>("grpc_port"); value)
         {
-            auto ipJs = config.at("ip").as_string();
-            ip_ = {ipJs.c_str(), ipJs.size()};
-        }
-        if (config.contains("ws_port"))
-        {
-            auto portjs = config.at("ws_port").as_string();
-            wsPort_ = {portjs.c_str(), portjs.size()};
-        }
-        if (config.contains("grpc_port"))
-        {
-            auto portjs = config.at("grpc_port").as_string();
-            grpcPort_ = {portjs.c_str(), portjs.size()};
+            grpcPort_ = *value;
             try
             {
                 boost::asio::ip::tcp::endpoint endpoint{
@@ -304,14 +315,12 @@ public:
                 stub_ = org::xrpl::rpc::v1::XRPLedgerAPIService::NewStub(
                     grpc::CreateCustomChannel(
                         ss.str(), grpc::InsecureChannelCredentials(), chArgs));
-                BOOST_LOG_TRIVIAL(debug)
-                    << "Made stub for remote = " << toString();
+                log_.debug() << "Made stub for remote = " << toString();
             }
             catch (std::exception const& e)
             {
-                BOOST_LOG_TRIVIAL(debug)
-                    << "Exception while creating stub = " << e.what()
-                    << " . Remote = " << toString();
+                log_.debug() << "Exception while creating stub = " << e.what()
+                             << " . Remote = " << toString();
             }
         }
     }
@@ -383,7 +392,6 @@ public:
     getValidatedRange() const
     {
         std::lock_guard lck(mtx_);
-
         return validatedLedgersRaw_;
     }
 
@@ -401,9 +409,8 @@ public:
     std::string
     toString() const override
     {
-        return "{ validated_ledger : " + getValidatedRange() +
-            " , ip : " + ip_ + " , web socket port : " + wsPort_ +
-            ", grpc port : " + grpcPort_ + " }";
+        return "{validated_ledger: " + getValidatedRange() + ", ip: " + ip_ +
+            ", web socket port: " + wsPort_ + ", grpc port: " + grpcPort_ + "}";
     }
 
     boost::json::object
@@ -498,7 +505,7 @@ class PlainETLSource : public ETLSourceImpl<PlainETLSource>
 
 public:
     PlainETLSource(
-        boost::json::object const& config,
+        clio::Config const& config,
         boost::asio::io_context& ioc,
         std::shared_ptr<BackendInterface> backend,
         std::shared_ptr<SubscriptionManager> subscriptions,
@@ -547,7 +554,7 @@ class SslETLSource : public ETLSourceImpl<SslETLSource>
 
 public:
     SslETLSource(
-        boost::json::object const& config,
+        clio::Config const& config,
         boost::asio::io_context& ioc,
         std::optional<std::reference_wrapper<boost::asio::ssl::context>> sslCtx,
         std::shared_ptr<BackendInterface> backend,
@@ -604,13 +611,13 @@ public:
 class ETLLoadBalancer
 {
 private:
+    clio::Logger log_{"ETL"};
     std::vector<std::unique_ptr<ETLSource>> sources_;
-
     std::uint32_t downloadRanges_ = 16;
 
 public:
     ETLLoadBalancer(
-        boost::json::object const& config,
+        clio::Config const& config,
         boost::asio::io_context& ioContext,
         std::shared_ptr<BackendInterface> backend,
         std::shared_ptr<SubscriptionManager> subscriptions,
@@ -618,7 +625,7 @@ public:
 
     static std::shared_ptr<ETLLoadBalancer>
     make_ETLLoadBalancer(
-        boost::json::object const& config,
+        clio::Config const& config,
         boost::asio::io_context& ioc,
         std::shared_ptr<BackendInterface> backend,
         std::shared_ptr<SubscriptionManager> subscriptions,
@@ -715,5 +722,3 @@ private:
     bool
     execute(Func f, uint32_t ledgerSequence);
 };
-
-#endif

src/etl/NFTHelpers.cpp

@@ -1,3 +1,22 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
 #include <ripple/app/tx/impl/details/NFTokenUtils.h>
 #include <ripple/protocol/STBase.h>
 #include <ripple/protocol/STTx.h>
@@ -107,8 +126,7 @@ getNFTokenMintData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
             NFTsData(tokenIDResult.front(), *owner, txMeta, false)};
 
     std::stringstream msg;
-    msg << __func__ << " - unexpected NFTokenMint data in tx "
-        << sttx.getTransactionID();
+    msg << " - unexpected NFTokenMint data in tx " << sttx.getTransactionID();
     throw std::runtime_error(msg.str());
 }
 
@@ -173,7 +191,7 @@ getNFTokenBurnData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
     }
 
     std::stringstream msg;
-    msg << __func__ << " - could not determine owner at burntime for tx "
+    msg << " - could not determine owner at burntime for tx "
         << sttx.getTransactionID();
     throw std::runtime_error(msg.str());
 }
@@ -198,7 +216,7 @@ getNFTokenAcceptOfferData(
         if (affectedBuyOffer == txMeta.getNodes().end())
         {
             std::stringstream msg;
-            msg << __func__ << " - unexpected NFTokenAcceptOffer data in tx "
+            msg << " - unexpected NFTokenAcceptOffer data in tx "
                 << sttx.getTransactionID();
             throw std::runtime_error(msg.str());
         }
@@ -228,7 +246,7 @@ getNFTokenAcceptOfferData(
     if (affectedSellOffer == txMeta.getNodes().end())
     {
         std::stringstream msg;
-        msg << __func__ << " - unexpected NFTokenAcceptOffer data in tx "
+        msg << " - unexpected NFTokenAcceptOffer data in tx "
             << sttx.getTransactionID();
         throw std::runtime_error(msg.str());
     }
@@ -278,7 +296,7 @@ getNFTokenAcceptOfferData(
     }
 
     std::stringstream msg;
-    msg << __func__ << " - unexpected NFTokenAcceptOffer data in tx "
+    msg << " - unexpected NFTokenAcceptOffer data in tx "
         << sttx.getTransactionID();
     throw std::runtime_error(msg.str());
 }

src/etl/ProbingETLSource.cpp

@@ -1,15 +1,36 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
 #include <etl/ProbingETLSource.h>
+#include <log/Logger.h>
+
+using namespace clio;
 
 ProbingETLSource::ProbingETLSource(
-    boost::json::object const& config,
+    clio::Config const& config,
     boost::asio::io_context& ioc,
     std::shared_ptr<BackendInterface> backend,
     std::shared_ptr<SubscriptionManager> subscriptions,
     std::shared_ptr<NetworkValidatedLedgers> nwvl,
     ETLLoadBalancer& balancer,
     boost::asio::ssl::context sslCtx)
-    : ioc_{ioc}
-    , sslCtx_{std::move(sslCtx)}
+    : sslCtx_{std::move(sslCtx)}
     , sslSrc_{make_shared<SslETLSource>(
           config,
           ioc,
@@ -69,7 +90,16 @@ boost::json::object
 ProbingETLSource::toJson() const
 {
     if (!currentSrc_)
-        return {};
+    {
+        boost::json::object sourcesJson = {
+            {"ws", plainSrc_->toJson()},
+            {"wss", sslSrc_->toJson()},
+        };
+
+        return {
+            {"probing", sourcesJson},
+        };
+    }
     return currentSrc_->toJson();
 }
 
@@ -77,7 +107,8 @@ std::string
 ProbingETLSource::toString() const
 {
     if (!currentSrc_)
-        return "{ probing }";
+        return "{probing... ws: " + plainSrc_->toString() +
+            ", wss: " + sslSrc_->toString() + "}";
     return currentSrc_->toString();
 }
 
@@ -140,9 +171,8 @@ ProbingETLSource::make_SSLHooks() noexcept
                 {
                     plainSrc_->pause();
                     currentSrc_ = sslSrc_;
-                    BOOST_LOG_TRIVIAL(info)
-                        << "Selected WSS as the main source: "
-                        << currentSrc_->toString();
+                    log_.info() << "Selected WSS as the main source: "
+                                << currentSrc_->toString();
                 }
                 return ETLSourceHooks::Action::PROCEED;
             },
@@ -171,9 +201,8 @@ ProbingETLSource::make_PlainHooks() noexcept
                 {
                     sslSrc_->pause();
                     currentSrc_ = plainSrc_;
-                    BOOST_LOG_TRIVIAL(info)
-                        << "Selected Plain WS as the main source: "
-                        << currentSrc_->toString();
+                    log_.info() << "Selected Plain WS as the main source: "
+                                << currentSrc_->toString();
                 }
                 return ETLSourceHooks::Action::PROCEED;
             },

src/etl/ProbingETLSource.h

@@ -1,22 +1,45 @@
-#ifndef RIPPLE_APP_REPORTING_PROBINGETLSOURCE_H_INCLUDED
-#define RIPPLE_APP_REPORTING_PROBINGETLSOURCE_H_INCLUDED
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
 
 #include <boost/asio.hpp>
 #include <boost/beast/core.hpp>
 #include <boost/beast/core/string.hpp>
 #include <boost/beast/ssl.hpp>
 #include <boost/beast/websocket.hpp>
-#include <etl/ETLSource.h>
+
 #include <mutex>
 
+#include <config/Config.h>
+#include <etl/ETLSource.h>
+#include <log/Logger.h>
+
 /// This ETLSource implementation attempts to connect over both secure websocket
 /// and plain websocket. First to connect pauses the other and the probing is
 /// considered done at this point. If however the connected source loses
 /// connection the probing is kickstarted again.
 class ProbingETLSource : public ETLSource
 {
+    clio::Logger log_{"ETL"};
+
     std::mutex mtx_;
-    boost::asio::io_context& ioc_;
     boost::asio::ssl::context sslCtx_;
     std::shared_ptr<ETLSource> sslSrc_;
     std::shared_ptr<ETLSource> plainSrc_;
@@ -24,7 +47,7 @@ class ProbingETLSource : public ETLSource
 
 public:
     ProbingETLSource(
-        boost::json::object const& config,
+        clio::Config const& config,
         boost::asio::io_context& ioc,
         std::shared_ptr<BackendInterface> backend,
         std::shared_ptr<SubscriptionManager> subscriptions,
@@ -87,5 +110,3 @@ private:
     ETLSourceHooks
     make_PlainHooks() noexcept;
 };
-
-#endif

src/etl/README.md

@@ -22,7 +22,7 @@ read-only mode. In read-only mode, the server does not perform ETL and simply
 publishes new ledgers as they are written to the database. 
 If the database is not updated within a certain time period
 (currently hard coded at 20 seconds), clio will begin the ETL
-process and start writing to the database. Postgres will report an error when
+process and start writing to the database. The database will report an error when
 trying to write a record with a key that already exists. ETL uses this error to
 determine that another process is writing to the database, and subsequently
 falls back to a soft read-only mode. clio can also operate in strict

src/etl/ReportingETL.h

@@ -1,5 +1,23 @@
-#ifndef RIPPLE_APP_REPORTING_REPORTINGETL_H_INCLUDED
-#define RIPPLE_APP_REPORTING_REPORTINGETL_H_INCLUDED
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
 
 #include <ripple/ledger/ReadView.h>
 #include <boost/algorithm/string.hpp>
@@ -8,6 +26,7 @@
 #include <boost/beast/websocket.hpp>
 #include <backend/BackendInterface.h>
 #include <etl/ETLSource.h>
+#include <log/Logger.h>
 #include <subscriptions/SubscriptionManager.h>
 
 #include "org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h"
@@ -55,6 +74,8 @@ class SubscriptionManager;
 class ReportingETL
 {
 private:
+    clio::Logger log_{"ETL"};
+
     std::shared_ptr<BackendInterface> backend_;
     std::shared_ptr<SubscriptionManager> subscriptions_;
     std::shared_ptr<ETLLoadBalancer> loadBalancer_;
@@ -77,6 +98,14 @@ private:
     // thread responsible for syncing the cache on startup
     std::thread cacheDownloader_;
 
+    struct ClioPeer
+    {
+        std::string ip;
+        int port;
+    };
+
+    std::vector<ClioPeer> clioPeers;
+
     std::thread worker_;
     boost::asio::io_context& ioContext_;
 
@@ -145,7 +174,7 @@ private:
     void
     setLastPublish()
     {
-        std::unique_lock lck(publishTimeMtx_);
+        std::scoped_lock lck(publishTimeMtx_);
         lastPublish_ = std::chrono::system_clock::now();
     }
 
@@ -157,7 +186,7 @@ private:
     void
     setLastClose(std::chrono::time_point<ripple::NetClock> lastCloseTime)
     {
-        std::unique_lock lck(closeTimeMtx_);
+        std::scoped_lock lck(closeTimeMtx_);
         lastCloseTime_ = lastCloseTime;
     }
 
@@ -177,6 +206,16 @@ private:
     void
     loadCache(uint32_t seq);
 
+    void
+    loadCacheFromDb(uint32_t seq);
+
+    bool
+    loadCacheFromClioPeer(
+        uint32_t ledgerSequence,
+        std::string const& ip,
+        std::string const& port,
+        boost::asio::yield_context& yield);
+
     /// Run ETL. Extracts ledgers and writes them to the database, until a
     /// write conflict occurs (or the server shuts down).
     /// @note database must already be populated when this function is
@@ -244,7 +283,7 @@ private:
     /// following parent
     /// @param parent the previous ledger
     /// @param rawData data extracted from an ETL source
-    /// @return the newly built ledger and data to write to Postgres
+    /// @return the newly built ledger and data to write to the database
     std::pair<ripple::LedgerInfo, bool>
     buildNextLedger(org::xrpl::rpc::v1::GetLedgerResponse& rawData);
 
@@ -282,7 +321,7 @@ private:
     void
     run()
     {
-        BOOST_LOG_TRIVIAL(info) << "Starting reporting etl";
+        log_.info() << "Starting reporting etl";
         stopping_ = false;
 
         doWork();
@@ -293,7 +332,7 @@ private:
 
 public:
     ReportingETL(
-        boost::json::object const& config,
+        clio::Config const& config,
         boost::asio::io_context& ioc,
         std::shared_ptr<BackendInterface> backend,
         std::shared_ptr<SubscriptionManager> subscriptions,
@@ -302,7 +341,7 @@ public:
 
     static std::shared_ptr<ReportingETL>
     make_ReportingETL(
-        boost::json::object const& config,
+        clio::Config const& config,
         boost::asio::io_context& ioc,
         std::shared_ptr<BackendInterface> backend,
         std::shared_ptr<SubscriptionManager> subscriptions,
@@ -319,8 +358,8 @@ public:
 
     ~ReportingETL()
     {
-        BOOST_LOG_TRIVIAL(info) << "onStop called";
-        BOOST_LOG_TRIVIAL(debug) << "Stopping Reporting ETL";
+        log_.info() << "onStop called";
+        log_.debug() << "Stopping Reporting ETL";
         stopping_ = true;
 
         if (worker_.joinable())
@@ -328,7 +367,7 @@ public:
         if (cacheDownloader_.joinable())
             cacheDownloader_.join();
 
-        BOOST_LOG_TRIVIAL(debug) << "Joined ReportingETL worker thread";
+        log_.debug() << "Joined ReportingETL worker thread";
     }
 
     boost::json::object
@@ -374,5 +413,3 @@ public:
         return now - (rippleEpochStart + closeTime);
     }
 };
-
-#endif

src/log/Logger.cpp

@@ -0,0 +1,209 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include <config/Config.h>
+#include <log/Logger.h>
+
+#include <algorithm>
+#include <array>
+#include <filesystem>
+
+namespace clio {
+
+Logger LogService::general_log_ = Logger{"General"};
+Logger LogService::alert_log_ = Logger{"Alert"};
+
+std::ostream&
+operator<<(std::ostream& stream, Severity sev)
+{
+    static constexpr std::array<const char*, 6> labels = {
+        "TRC",
+        "DBG",
+        "NFO",
+        "WRN",
+        "ERR",
+        "FTL",
+    };
+
+    return stream << labels.at(static_cast<int>(sev));
+}
+
+Severity
+tag_invoke(boost::json::value_to_tag<Severity>, boost::json::value const& value)
+{
+    if (not value.is_string())
+        throw std::runtime_error("`log_level` must be a string");
+    auto const& logLevel = value.as_string();
+
+    if (boost::iequals(logLevel, "trace"))
+        return Severity::TRC;
+    else if (boost::iequals(logLevel, "debug"))
+        return Severity::DBG;
+    else if (boost::iequals(logLevel, "info"))
+        return Severity::NFO;
+    else if (
+        boost::iequals(logLevel, "warning") || boost::iequals(logLevel, "warn"))
+        return Severity::WRN;
+    else if (boost::iequals(logLevel, "error"))
+        return Severity::ERR;
+    else if (boost::iequals(logLevel, "fatal"))
+        return Severity::FTL;
+    else
+        throw std::runtime_error(
+            "Could not parse `log_level`: expected `trace`, `debug`, `info`, "
+            "`warning`, `error` or `fatal`");
+}
+
+void
+LogService::init(Config const& config)
+{
+    namespace src = boost::log::sources;
+    namespace keywords = boost::log::keywords;
+    namespace sinks = boost::log::sinks;
+
+    boost::log::add_common_attributes();
+    boost::log::register_simple_formatter_factory<Severity, char>("Severity");
+    auto const defaultFormat =
+        "%TimeStamp% (%SourceLocation%) [%ThreadID%] %Channel%:%Severity% "
+        "%Message%";
+    std::string format =
+        config.valueOr<std::string>("log_format", defaultFormat);
+
+    if (config.valueOr("log_to_console", false))
+    {
+        boost::log::add_console_log(std::cout, keywords::format = format);
+    }
+
+    auto logDir = config.maybeValue<std::string>("log_directory");
+    if (logDir)
+    {
+        boost::filesystem::path dirPath{logDir.value()};
+        if (!boost::filesystem::exists(dirPath))
+            boost::filesystem::create_directories(dirPath);
+        auto const rotationSize =
+            config.valueOr<uint64_t>("log_rotation_size", 2048u) * 1024u *
+            1024u;
+        auto const rotationPeriod =
+            config.valueOr<uint32_t>("log_rotation_hour_interval", 12u);
+        auto const dirSize =
+            config.valueOr<uint64_t>("log_directory_max_size", 50u * 1024u) *
+            1024u * 1024u;
+        auto fileSink = boost::log::add_file_log(
+            keywords::file_name = dirPath / "clio.log",
+            keywords::target_file_name = dirPath / "clio_%Y-%m-%d_%H-%M-%S.log",
+            keywords::auto_flush = true,
+            keywords::format = format,
+            keywords::open_mode = std::ios_base::app,
+            keywords::rotation_size = rotationSize,
+            keywords::time_based_rotation =
+                sinks::file::rotation_at_time_interval(
+                    boost::posix_time::hours(rotationPeriod)));
+        fileSink->locked_backend()->set_file_collector(
+            sinks::file::make_collector(
+                keywords::target = dirPath, keywords::max_size = dirSize));
+        fileSink->locked_backend()->scan_for_files();
+    }
+
+    // get default severity, can be overridden per channel using
+    // the `log_channels` array
+    auto defaultSeverity = config.valueOr<Severity>("log_level", Severity::NFO);
+    static constexpr std::array<const char*, 7> channels = {
+        "General",
+        "WebServer",
+        "Backend",
+        "RPC",
+        "ETL",
+        "Subscriptions",
+        "Performance",
+    };
+
+    auto core = boost::log::core::get();
+    auto min_severity = boost::log::expressions::channel_severity_filter(
+        log_channel, log_severity);
+
+    for (auto const& channel : channels)
+        min_severity[channel] = defaultSeverity;
+    min_severity["Alert"] =
+        Severity::WRN;  // Channel for alerts, always warning severity
+
+    for (auto const overrides = config.arrayOr("log_channels", {});
+         auto const& cfg : overrides)
+    {
+        auto name = cfg.valueOrThrow<std::string>(
+            "channel", "Channel name is required");
+        if (not std::count(std::begin(channels), std::end(channels), name))
+            throw std::runtime_error(
+                "Can't override settings for log channel " + name +
+                ": invalid channel");
+
+        min_severity[name] =
+            cfg.valueOr<Severity>("log_level", defaultSeverity);
+    }
+
+    core->set_filter(min_severity);
+    LogService::info() << "Default log level = " << defaultSeverity;
+}
+
+Logger::Pump
+Logger::trace(source_location_t const& loc) const
+{
+    return {logger_, Severity::TRC, loc};
+};
+Logger::Pump
+Logger::debug(source_location_t const& loc) const
+{
+    return {logger_, Severity::DBG, loc};
+};
+Logger::Pump
+Logger::info(source_location_t const& loc) const
+{
+    return {logger_, Severity::NFO, loc};
+};
+Logger::Pump
+Logger::warn(source_location_t const& loc) const
+{
+    return {logger_, Severity::WRN, loc};
+};
+Logger::Pump
+Logger::error(source_location_t const& loc) const
+{
+    return {logger_, Severity::ERR, loc};
+};
+Logger::Pump
+Logger::fatal(source_location_t const& loc) const
+{
+    return {logger_, Severity::FTL, loc};
+};
+
+std::string
+Logger::Pump::pretty_path(source_location_t const& loc, size_t max_depth) const
+{
+    auto const file_path = std::string{loc.file_name()};
+    auto idx = file_path.size();
+    while (max_depth-- > 0)
+    {
+        idx = file_path.rfind('/', idx - 1);
+        if (idx == std::string::npos || idx == 0)
+            break;
+    }
+    return file_path.substr(idx == std::string::npos ? 0 : idx + 1) + ':' +
+        std::to_string(loc.line());
+}
+
+}  // namespace clio

src/log/Logger.h

@@ -0,0 +1,314 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include <boost/algorithm/string/predicate.hpp>
+#include <boost/filesystem.hpp>
+#include <boost/json.hpp>
+#include <boost/log/core/core.hpp>
+#include <boost/log/expressions/predicates/channel_severity_filter.hpp>
+#include <boost/log/sinks/unlocked_frontend.hpp>
+#include <boost/log/sources/record_ostream.hpp>
+#include <boost/log/sources/severity_channel_logger.hpp>
+#include <boost/log/sources/severity_feature.hpp>
+#include <boost/log/sources/severity_logger.hpp>
+#include <boost/log/utility/manipulators/add_value.hpp>
+#include <boost/log/utility/setup/common_attributes.hpp>
+#include <boost/log/utility/setup/console.hpp>
+#include <boost/log/utility/setup/file.hpp>
+#include <boost/log/utility/setup/formatter_parser.hpp>
+
+#if defined(HAS_SOURCE_LOCATION) && __has_builtin(__builtin_source_location)
+// this is used by fully compatible compilers like gcc
+#include <source_location>
+
+#elif defined(HAS_EXPERIMENTAL_SOURCE_LOCATION)
+// this is used by clang on linux where source_location is still not out of
+// experimental headers
+#include <experimental/source_location>
+#endif
+
+#include <optional>
+#include <string>
+
+namespace clio {
+
+class Config;
+#if defined(HAS_SOURCE_LOCATION) && __has_builtin(__builtin_source_location)
+using source_location_t = std::source_location;
+#define CURRENT_SRC_LOCATION source_location_t::current()
+
+#elif defined(HAS_EXPERIMENTAL_SOURCE_LOCATION)
+using source_location_t = std::experimental::source_location;
+#define CURRENT_SRC_LOCATION source_location_t::current()
+
+#else
+// A workaround for AppleClang that is lacking source_location atm.
+// TODO: remove this workaround when all compilers catch up to c++20
+class SourceLocation
+{
+    std::string_view file_;
+    std::size_t line_;
+
+public:
+    SourceLocation(std::string_view file, std::size_t line)
+        : file_{file}, line_{line}
+    {
+    }
+    std::string_view
+    file_name() const
+    {
+        return file_;
+    }
+    std::size_t
+    line() const
+    {
+        return line_;
+    }
+};
+using source_location_t = SourceLocation;
+#define CURRENT_SRC_LOCATION \
+    source_location_t(__builtin_FILE(), __builtin_LINE())
+#endif
+
+/**
+ * @brief Custom severity levels for @ref Logger.
+ */
+enum class Severity {
+    TRC,
+    DBG,
+    NFO,
+    WRN,
+    ERR,
+    FTL,
+};
+
+BOOST_LOG_ATTRIBUTE_KEYWORD(log_severity, "Severity", Severity);
+BOOST_LOG_ATTRIBUTE_KEYWORD(log_channel, "Channel", std::string);
+
+/**
+ * @brief Custom labels for @ref Severity in log output.
+ *
+ * @param stream std::ostream The output stream
+ * @param sev Severity The severity to output to the ostream
+ * @return std::ostream& The same ostream we were given
+ */
+std::ostream&
+operator<<(std::ostream& stream, Severity sev);
+
+/**
+ * @brief Custom JSON parser for @ref Severity.
+ *
+ * @param value The JSON string to parse
+ * @return Severity The parsed severity
+ * @throws std::runtime_error Thrown if severity is not in the right format
+ */
+Severity
+tag_invoke(
+    boost::json::value_to_tag<Severity>,
+    boost::json::value const& value);
+
+/**
+ * @brief A simple thread-safe logger for the channel specified
+ * in the constructor.
+ *
+ * This is cheap to copy and move. Designed to be used as a member variable or
+ * otherwise. See @ref LogService::init() for setup of the logging core and
+ * severity levels for each channel.
+ */
+class Logger final
+{
+    using logger_t =
+        boost::log::sources::severity_channel_logger_mt<Severity, std::string>;
+    mutable logger_t logger_;
+
+    friend class LogService;  // to expose the Pump interface
+
+    /**
+     * @brief Helper that pumps data into a log record via `operator<<`.
+     */
+    class Pump final
+    {
+        using pump_opt_t =
+            std::optional<boost::log::aux::record_pump<logger_t>>;
+
+        boost::log::record rec_;
+        pump_opt_t pump_ = std::nullopt;
+
+    public:
+        ~Pump() = default;
+        Pump(logger_t& logger, Severity sev, source_location_t const& loc)
+            : rec_{logger.open_record(boost::log::keywords::severity = sev)}
+        {
+            if (rec_)
+            {
+                pump_.emplace(boost::log::aux::make_record_pump(logger, rec_));
+                pump_->stream() << boost::log::add_value(
+                    "SourceLocation", pretty_path(loc));
+            }
+        }
+
+        Pump(Pump&&) = delete;
+        Pump(Pump const&) = delete;
+        Pump&
+        operator=(Pump const&) = delete;
+        Pump&
+        operator=(Pump&&) = delete;
+
+        /**
+         * @brief Perfectly forwards any incoming data into the underlying
+         * boost::log pump if the pump is available. nop otherwise.
+         *
+         * @tparam T Type of data to pump
+         * @param data The data to pump
+         * @return Pump& Reference to itself for chaining
+         */
+        template <typename T>
+        [[maybe_unused]] Pump&
+        operator<<(T&& data)
+        {
+            if (pump_)
+                pump_->stream() << std::forward<T>(data);
+            return *this;
+        }
+
+    private:
+        [[nodiscard]] std::string
+        pretty_path(source_location_t const& loc, size_t max_depth = 3) const;
+    };
+
+public:
+    ~Logger() = default;
+    /**
+     * @brief Construct a new Logger object that produces loglines for the
+     * specified channel.
+     *
+     * See @ref LogService::init() for general setup and configuration of
+     * severity levels per channel.
+     *
+     * @param channel The channel this logger will report into.
+     */
+    Logger(std::string channel)
+        : logger_{boost::log::keywords::channel = channel}
+    {
+    }
+    Logger(Logger const&) = default;
+    Logger(Logger&&) = default;
+    Logger&
+    operator=(Logger const&) = default;
+    Logger&
+    operator=(Logger&&) = default;
+
+    /*! Interface for logging at @ref Severity::TRC severity */
+    [[nodiscard]] Pump
+    trace(source_location_t const& loc = CURRENT_SRC_LOCATION) const;
+
+    /*! Interface for logging at @ref Severity::DBG severity */
+    [[nodiscard]] Pump
+    debug(source_location_t const& loc = CURRENT_SRC_LOCATION) const;
+
+    /*! Interface for logging at @ref Severity::INFO severity */
+    [[nodiscard]] Pump
+    info(source_location_t const& loc = CURRENT_SRC_LOCATION) const;
+
+    /*! Interface for logging at @ref Severity::WRN severity */
+    [[nodiscard]] Pump
+    warn(source_location_t const& loc = CURRENT_SRC_LOCATION) const;
+
+    /*! Interface for logging at @ref Severity::ERR severity */
+    [[nodiscard]] Pump
+    error(source_location_t const& loc = CURRENT_SRC_LOCATION) const;
+
+    /*! Interface for logging at @ref Severity::FTL severity */
+    [[nodiscard]] Pump
+    fatal(source_location_t const& loc = CURRENT_SRC_LOCATION) const;
+};
+
+/**
+ * @brief A global logging service.
+ *
+ * Used to initialize and setup the logging core as well as a globally available
+ * entrypoint for logging into the `General` channel as well as raising alerts.
+ */
+class LogService
+{
+    static Logger general_log_; /*! Global logger for General channel */
+    static Logger alert_log_;   /*! Global logger for Alerts channel */
+
+public:
+    LogService() = delete;
+
+    /**
+     * @brief Global log core initialization from a @ref Config
+     */
+    static void
+    init(Config const& config);
+
+    /*! Globally accesible General logger at @ref Severity::TRC severity */
+    [[nodiscard]] static Logger::Pump
+    trace(source_location_t const& loc = CURRENT_SRC_LOCATION)
+    {
+        return general_log_.trace(loc);
+    }
+
+    /*! Globally accesible General logger at @ref Severity::DBG severity */
+    [[nodiscard]] static Logger::Pump
+    debug(source_location_t const& loc = CURRENT_SRC_LOCATION)
+    {
+        return general_log_.debug(loc);
+    }
+
+    /*! Globally accesible General logger at @ref Severity::NFO severity */
+    [[nodiscard]] static Logger::Pump
+    info(source_location_t const& loc = CURRENT_SRC_LOCATION)
+    {
+        return general_log_.info(loc);
+    }
+
+    /*! Globally accesible General logger at @ref Severity::WRN severity */
+    [[nodiscard]] static Logger::Pump
+    warn(source_location_t const& loc = CURRENT_SRC_LOCATION)
+    {
+        return general_log_.warn(loc);
+    }
+
+    /*! Globally accesible General logger at @ref Severity::ERR severity */
+    [[nodiscard]] static Logger::Pump
+    error(source_location_t const& loc = CURRENT_SRC_LOCATION)
+    {
+        return general_log_.error(loc);
+    }
+
+    /*! Globally accesible General logger at @ref Severity::FTL severity */
+    [[nodiscard]] static Logger::Pump
+    fatal(source_location_t const& loc = CURRENT_SRC_LOCATION)
+    {
+        return general_log_.fatal(loc);
+    }
+
+    /*! Globally accesible Alert logger */
+    [[nodiscard]] static Logger::Pump
+    alert(source_location_t const& loc = CURRENT_SRC_LOCATION)
+    {
+        return alert_log_.warn(loc);
+    }
+};
+
+};  // namespace clio

src/main/Build.h

@@ -1,5 +1,23 @@
-#ifndef CLIO_BUILD_INFO_H
-#define CLIO_BUILD_INFO_H
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
 
 #include <string>
 
@@ -12,5 +30,3 @@ std::string const&
 getClioFullVersionString();
 
 }  // namespace Build
-
-#endif  // CLIO_BUILD_INFO_H

src/main/impl/Build.cpp

@@ -1,59 +0,0 @@
-#include <ripple/beast/core/SemanticVersion.h>
-#include <boost/preprocessor/stringize.hpp>
-#include <algorithm>
-#include <main/Build.h>
-#include <optional>
-#include <stdexcept>
-
-namespace Build {
-
-//--------------------------------------------------------------------------
-//  The build version number. You must edit this for each release
-//  and follow the format described at http://semver.org/
-//------------------------------------------------------------------------------
-// clang-format off
-char const* const versionString = "1.0.3"
-// clang-format on
-
-#if defined(DEBUG) || defined(SANITIZER)
-    "+"
-#ifdef CLIO_GIT_COMMIT_HASH
-    CLIO_GIT_COMMIT_HASH
-    "."
-#endif
-#ifdef DEBUG
-    "DEBUG"
-#ifdef SANITIZER
-    "."
-#endif
-#endif
-
-#ifdef SANITIZER
-    BOOST_PP_STRINGIZE(SANITIZER)
-#endif
-#endif
-
-    //--------------------------------------------------------------------------
-    ;
-
-std::string const&
-getClioVersionString()
-{
-    static std::string const value = [] {
-        std::string const s = versionString;
-        beast::SemanticVersion v;
-        if (!v.parse(s) || v.print() != s)
-            throw std::runtime_error(s + ": Bad server version string");
-        return s;
-    }();
-    return value;
-}
-
-std::string const&
-getClioFullVersionString()
-{
-    static std::string const value = "clio-" + getClioVersionString();
-    return value;
-}
-
-}  // namespace Build

src/main/main.cpp

@@ -1,3 +1,22 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
 #include <grpc/impl/codegen/port_platform.h>
 #ifdef GRPC_TSAN_ENABLED
 #undef GRPC_TSAN_ENABLED
@@ -6,25 +25,22 @@
 #undef GRPC_ASAN_ENABLED
 #endif
 
+#include <backend/BackendFactory.h>
+#include <config/Config.h>
+#include <etl/ReportingETL.h>
+#include <log/Logger.h>
+#include <webserver/Listener.h>
+
 #include <boost/asio/dispatch.hpp>
 #include <boost/asio/strand.hpp>
 #include <boost/beast/websocket.hpp>
 #include <boost/date_time/posix_time/posix_time_types.hpp>
+#include <boost/filesystem/path.hpp>
 #include <boost/json.hpp>
-#include <boost/log/core.hpp>
-#include <boost/log/expressions.hpp>
-#include <boost/log/sinks/text_file_backend.hpp>
-#include <boost/log/sources/record_ostream.hpp>
-#include <boost/log/sources/severity_logger.hpp>
-#include <boost/log/support/date_time.hpp>
-#include <boost/log/trivial.hpp>
-#include <boost/log/utility/setup/common_attributes.hpp>
-#include <boost/log/utility/setup/console.hpp>
-#include <boost/log/utility/setup/file.hpp>
+#include <boost/program_options.hpp>
+
 #include <algorithm>
-#include <backend/BackendFactory.h>
 #include <cstdlib>
-#include <etl/ReportingETL.h>
 #include <fstream>
 #include <functional>
 #include <iostream>
@@ -34,39 +50,73 @@
 #include <string>
 #include <thread>
 #include <vector>
-#include <webserver/Listener.h>
 
-std::optional<boost::json::object>
-parse_config(const char* filename)
+using namespace clio;
+namespace po = boost::program_options;
+
+/**
+ * @brief Parse command line and return path to configuration file
+ *
+ * @param argc
+ * @param argv
+ * @return std::string Path to configuration file
+ */
+std::string
+parseCli(int argc, char* argv[])
 {
-    try
+    static constexpr char defaultConfigPath[] = "/etc/opt/clio/config.json";
+
+    // clang-format off
+    po::options_description description("Options");
+    description.add_options()
+        ("help,h", "print help message and exit")
+        ("version,v", "print version and exit")
+        ("conf,c", po::value<std::string>()->default_value(defaultConfigPath), "configuration file")
+    ;
+    // clang-format on
+    po::positional_options_description positional;
+    positional.add("conf", 1);
+
+    po::variables_map parsed;
+    po::store(
+        po::command_line_parser(argc, argv)
+            .options(description)
+            .positional(positional)
+            .run(),
+        parsed);
+    po::notify(parsed);
+
+    if (parsed.count("version"))
     {
-        std::ifstream in(filename, std::ios::in | std::ios::binary);
-        if (in)
-        {
-            std::stringstream contents;
-            contents << in.rdbuf();
-            in.close();
-            std::cout << contents.str() << std::endl;
-            boost::json::value value = boost::json::parse(contents.str());
-            return value.as_object();
-        }
+        std::cout << Build::getClioFullVersionString() << '\n';
+        std::exit(EXIT_SUCCESS);
     }
-    catch (std::exception const& e)
+
+    if (parsed.count("help"))
     {
-        std::cout << e.what() << std::endl;
+        std::cout << "Clio server " << Build::getClioFullVersionString()
+                  << "\n\n"
+                  << description;
+        std::exit(EXIT_SUCCESS);
     }
-    return {};
+
+    return parsed["conf"].as<std::string>();
 }
 
+/**
+ * @brief Parse certificates from configuration file
+ *
+ * @param config The configuration
+ * @return std::optional<ssl::context> SSL context if certificates were parsed
+ */
 std::optional<ssl::context>
-parse_certs(boost::json::object const& config)
+parseCerts(Config const& config)
 {
     if (!config.contains("ssl_cert_file") || !config.contains("ssl_key_file"))
         return {};
 
-    auto certFilename = config.at("ssl_cert_file").as_string().c_str();
-    auto keyFilename = config.at("ssl_key_file").as_string().c_str();
+    auto certFilename = config.value<std::string>("ssl_cert_file");
+    auto keyFilename = config.value<std::string>("ssl_key_file");
 
     std::ifstream readCert(certFilename, std::ios::in | std::ios::binary);
     if (!readCert)
@@ -74,7 +124,6 @@ parse_certs(boost::json::object const& config)
 
     std::stringstream contents;
     contents << readCert.rdbuf();
-    readCert.close();
     std::string cert = contents.str();
 
     std::ifstream readKey(keyFilename, std::ios::in | std::ios::binary);
@@ -101,92 +150,12 @@ parse_certs(boost::json::object const& config)
     return ctx;
 }
 
-void
-initLogging(boost::json::object const& config)
-{
-    namespace src = boost::log::sources;
-    namespace keywords = boost::log::keywords;
-    namespace sinks = boost::log::sinks;
-    namespace trivial = boost::log::trivial;
-    boost::log::add_common_attributes();
-    std::string format = "[%TimeStamp%] [%ThreadID%] [%Severity%] %Message%";
-    if (!config.contains("log_to_console") ||
-        config.at("log_to_console").as_bool())
-    {
-        boost::log::add_console_log(std::cout, keywords::format = format);
-    }
-    if (config.contains("log_directory"))
-    {
-        if (!config.at("log_directory").is_string())
-            throw std::runtime_error("log directory must be a string");
-        boost::filesystem::path dirPath{
-            config.at("log_directory").as_string().c_str()};
-        if (!boost::filesystem::exists(dirPath))
-            boost::filesystem::create_directories(dirPath);
-        const int64_t rotationSize = config.contains("log_rotation_size")
-            ? config.at("log_rotation_size").as_int64() * 1024 * 1024u
-            : 2 * 1024 * 1024 * 1024u;
-        if (rotationSize <= 0)
-            throw std::runtime_error(
-                "log rotation size must be greater than 0");
-        const int64_t rotationPeriod =
-            config.contains("log_rotation_hour_interval")
-            ? config.at("log_rotation_hour_interval").as_int64()
-            : 12u;
-        if (rotationPeriod <= 0)
-            throw std::runtime_error(
-                "log rotation time interval must be greater than 0");
-        const int64_t dirSize = config.contains("log_directory_max_size")
-            ? config.at("log_directory_max_size").as_int64() * 1024 * 1024u
-            : 50 * 1024 * 1024 * 1024u;
-        if (dirSize <= 0)
-            throw std::runtime_error(
-                "log rotation directory max size must be greater than 0");
-        auto fileSink = boost::log::add_file_log(
-            keywords::file_name = dirPath / "clio.log",
-            keywords::target_file_name = dirPath / "clio_%Y-%m-%d_%H-%M-%S.log",
-            keywords::auto_flush = true,
-            keywords::format = format,
-            keywords::open_mode = std::ios_base::app,
-            keywords::rotation_size = rotationSize,
-            keywords::time_based_rotation =
-                sinks::file::rotation_at_time_interval(
-                    boost::posix_time::hours(rotationPeriod)));
-        fileSink->locked_backend()->set_file_collector(
-            sinks::file::make_collector(
-                keywords::target = dirPath, keywords::max_size = dirSize));
-        fileSink->locked_backend()->scan_for_files();
-    }
-    auto const logLevel = config.contains("log_level")
-        ? config.at("log_level").as_string()
-        : "info";
-    if (boost::iequals(logLevel, "trace"))
-        boost::log::core::get()->set_filter(
-            trivial::severity >= trivial::trace);
-    else if (boost::iequals(logLevel, "debug"))
-        boost::log::core::get()->set_filter(
-            trivial::severity >= trivial::debug);
-    else if (boost::iequals(logLevel, "info"))
-        boost::log::core::get()->set_filter(trivial::severity >= trivial::info);
-    else if (
-        boost::iequals(logLevel, "warning") || boost::iequals(logLevel, "warn"))
-        boost::log::core::get()->set_filter(
-            trivial::severity >= trivial::warning);
-    else if (boost::iequals(logLevel, "error"))
-        boost::log::core::get()->set_filter(
-            trivial::severity >= trivial::error);
-    else if (boost::iequals(logLevel, "fatal"))
-        boost::log::core::get()->set_filter(
-            trivial::severity >= trivial::fatal);
-    else
-    {
-        BOOST_LOG_TRIVIAL(warning) << "Unrecognized log level: " << logLevel
-                                   << ". Setting log level to info";
-        boost::log::core::get()->set_filter(trivial::severity >= trivial::info);
-    }
-    BOOST_LOG_TRIVIAL(info) << "Log level = " << logLevel;
-}
-
+/**
+ * @brief Start context threads
+ *
+ * @param ioc Context
+ * @param numThreads Number of worker threads to start
+ */
 void
 start(boost::asio::io_context& ioc, std::uint32_t numThreads)
 {
@@ -200,71 +169,51 @@ start(boost::asio::io_context& ioc, std::uint32_t numThreads)
 
 int
 main(int argc, char* argv[])
+try
 {
-    // Check command line arguments.
-    if (argc != 2)
-    {
-        std::cerr << "Usage: clio_server "
-                     "<config_file> \n"
-                  << "Example:\n"
-                  << "    clio_server config.json \n";
-        return EXIT_FAILURE;
-    }
-
-    if (std::string{argv[1]} == "-v" || std::string{argv[1]} == "--version")
-    {
-        std::cout << Build::getClioFullVersionString() << std::endl;
-        return EXIT_SUCCESS;
-    }
-
-    auto const config = parse_config(argv[1]);
+    auto const configPath = parseCli(argc, argv);
+    auto const config = ConfigReader::open(configPath);
     if (!config)
     {
-        std::cerr << "Couldnt parse config. Exiting..." << std::endl;
+        std::cerr << "Couldnt parse config '" << configPath << "'."
+                  << std::endl;
         return EXIT_FAILURE;
     }
 
-    initLogging(*config);
+    LogService::init(config);
+    LogService::info() << "Clio version: " << Build::getClioFullVersionString();
 
-    // Announce Clio version
-    BOOST_LOG_TRIVIAL(info)
-        << "Clio version: " << Build::getClioFullVersionString();
-
-    auto ctx = parse_certs(*config);
+    auto ctx = parseCerts(config);
     auto ctxRef = ctx
         ? std::optional<std::reference_wrapper<ssl::context>>{ctx.value()}
         : std::nullopt;
 
-    auto const threads = config->contains("io_threads")
-        ? config->at("io_threads").as_int64()
-        : 2;
-
+    auto const threads = config.valueOr("io_threads", 2);
     if (threads <= 0)
     {
-        BOOST_LOG_TRIVIAL(fatal) << "io_threads is less than 0";
+        LogService::fatal() << "io_threads is less than 0";
         return EXIT_FAILURE;
     }
-    BOOST_LOG_TRIVIAL(info) << "Number of io threads = " << threads;
+    LogService::info() << "Number of io threads = " << threads;
 
-    // io context to handle all incoming requests, as well as other things
+    // IO context to handle all incoming requests, as well as other things
     // This is not the only io context in the application
     boost::asio::io_context ioc{threads};
 
     // Rate limiter, to prevent abuse
-    DOSGuard dosGuard{config.value(), ioc};
+    auto sweepHandler = IntervalSweepHandler{config, ioc};
+    auto dosGuard = DOSGuard{config, sweepHandler};
 
     // Interface to the database
-    std::shared_ptr<BackendInterface> backend{
-        Backend::make_Backend(ioc, *config)};
+    auto backend = Backend::make_Backend(ioc, config);
 
     // Manages clients subscribed to streams
-    std::shared_ptr<SubscriptionManager> subscriptions{
-        SubscriptionManager::make_SubscriptionManager(*config, backend)};
+    auto subscriptions =
+        SubscriptionManager::make_SubscriptionManager(config, backend);
 
     // Tracks which ledgers have been validated by the
     // network
-    std::shared_ptr<NetworkValidatedLedgers> ledgers{
-        NetworkValidatedLedgers::make_ValidatedLedgers()};
+    auto ledgers = NetworkValidatedLedgers::make_ValidatedLedgers();
 
     // Handles the connection to one or more rippled nodes.
     // ETL uses the balancer to extract data.
@@ -272,16 +221,16 @@ main(int argc, char* argv[])
     // The balancer itself publishes to streams (transactions_proposed and
     // accounts_proposed)
     auto balancer = ETLLoadBalancer::make_ETLLoadBalancer(
-        *config, ioc, backend, subscriptions, ledgers);
+        config, ioc, backend, subscriptions, ledgers);
 
     // ETL is responsible for writing and publishing to streams. In read-only
     // mode, ETL only publishes
     auto etl = ReportingETL::make_ReportingETL(
-        *config, ioc, backend, subscriptions, balancer, ledgers);
+        config, ioc, backend, subscriptions, balancer, ledgers);
 
     // The server handles incoming RPCs
     auto httpServer = Server::make_HttpServer(
-        *config, ioc, ctxRef, backend, subscriptions, balancer, etl, dosGuard);
+        config, ioc, ctxRef, backend, subscriptions, balancer, etl, dosGuard);
 
     // Blocks until stopped.
     // When stopped, shared_ptrs fall out of scope
@@ -290,3 +239,7 @@ main(int argc, char* argv[])
 
     return EXIT_SUCCESS;
 }
+catch (std::exception const& e)
+{
+    LogService::fatal() << "Exit on exception: " << e.what();
+}

src/rpc/Counters.cpp

@@ -1,5 +1,25 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
 #include <rpc/Counters.h>
 #include <rpc/RPC.h>
+#include <rpc/RPCHelpers.h>
 
 namespace RPC {
 
@@ -10,7 +30,7 @@ Counters::initializeCounter(std::string const& method)
     if (methodInfo_.count(method) == 0)
     {
         lk.unlock();
-        std::unique_lock ulk(mutex_);
+        std::scoped_lock ulk(mutex_);
 
         // This calls the default constructor for methodInfo of the method.
         methodInfo_[method];
@@ -66,20 +86,23 @@ Counters::report()
 {
     std::shared_lock lk(mutex_);
     boost::json::object obj = {};
+    obj[JS(rpc)] = boost::json::object{};
+    auto& rpc = obj[JS(rpc)].as_object();
 
     for (auto const& [method, info] : methodInfo_)
     {
         boost::json::object counters = {};
-        counters["started"] = std::to_string(info.started);
-        counters["finished"] = std::to_string(info.finished);
-        counters["errored"] = std::to_string(info.errored);
+        counters[JS(started)] = std::to_string(info.started);
+        counters[JS(finished)] = std::to_string(info.finished);
+        counters[JS(errored)] = std::to_string(info.errored);
         counters["forwarded"] = std::to_string(info.forwarded);
-        counters["duration_us"] = std::to_string(info.duration);
+        counters[JS(duration_us)] = std::to_string(info.duration);
 
-        obj[method] = std::move(counters);
+        rpc[method] = std::move(counters);
     }
+    obj["work_queue"] = workQueue_.get().report();
 
     return obj;
 }
 
-}  // namespace RPC
+}  // namespace RPC

src/rpc/Counters.h

@@ -1,9 +1,29 @@
-#ifndef RPC_COUNTERS_H
-#define RPC_COUNTERS_H
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
 
 #include <boost/json.hpp>
 #include <chrono>
 #include <cstdint>
+#include <functional>
+#include <rpc/WorkQueue.h>
 #include <shared_mutex>
 #include <string>
 #include <unordered_map>
@@ -30,8 +50,10 @@ private:
     std::shared_mutex mutex_;
     std::unordered_map<std::string, MethodInfo> methodInfo_;
 
+    std::reference_wrapper<const WorkQueue> workQueue_;
+
 public:
-    Counters() = default;
+    Counters(WorkQueue const& wq) : workQueue_(std::cref(wq)){};
 
     void
     rpcErrored(std::string const& method);
@@ -49,5 +71,3 @@ public:
 };
 
 }  // namespace RPC
-
-#endif  // RPC_COUNTERS_H

src/rpc/Errors.cpp

@@ -0,0 +1,169 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include <rpc/Errors.h>
+
+#include <algorithm>
+
+using namespace std;
+
+namespace {
+template <class... Ts>
+struct overloadSet : Ts...
+{
+    using Ts::operator()...;
+};
+
+// explicit deduction guide (not needed as of C++20, but clang be clang)
+template <class... Ts>
+overloadSet(Ts...) -> overloadSet<Ts...>;
+}  // namespace
+
+namespace RPC {
+
+WarningInfo const&
+getWarningInfo(WarningCode code)
+{
+    constexpr static WarningInfo infos[]{
+        {warnUNKNOWN, "Unknown warning"},
+        {warnRPC_CLIO,
+         "This is a clio server. clio only serves validated data. If you "
+         "want to talk to rippled, include 'ledger_index':'current' in your "
+         "request"},
+        {warnRPC_OUTDATED, "This server may be out of date"},
+        {warnRPC_RATE_LIMIT, "You are about to be rate limited"}};
+
+    auto matchByCode = [code](auto const& info) { return info.code == code; };
+    if (auto it = find_if(begin(infos), end(infos), matchByCode);
+        it != end(infos))
+        return *it;
+
+    throw(out_of_range("Invalid WarningCode"));
+}
+
+boost::json::object
+makeWarning(WarningCode code)
+{
+    boost::json::object json;
+    auto const& info = getWarningInfo(code);
+    json["id"] = code;
+    json["message"] = static_cast<string>(info.message);
+    return json;
+}
+
+ClioErrorInfo const&
+getErrorInfo(ClioError code)
+{
+    constexpr static ClioErrorInfo infos[]{
+        {ClioError::rpcMALFORMED_CURRENCY,
+         "malformedCurrency",
+         "Malformed currency."},
+        {ClioError::rpcMALFORMED_REQUEST,
+         "malformedRequest",
+         "Malformed request."},
+        {ClioError::rpcMALFORMED_OWNER, "malformedOwner", "Malformed owner."},
+        {ClioError::rpcMALFORMED_ADDRESS,
+         "malformedAddress",
+         "Malformed address."},
+    };
+
+    auto matchByCode = [code](auto const& info) { return info.code == code; };
+    if (auto it = find_if(begin(infos), end(infos), matchByCode);
+        it != end(infos))
+        return *it;
+
+    throw(out_of_range("Invalid error code"));
+}
+
+boost::json::object
+makeError(
+    RippledError err,
+    optional<string_view> customError,
+    optional<string_view> customMessage)
+{
+    boost::json::object json;
+    auto const& info = ripple::RPC::get_error_info(err);
+
+    json["error"] = customError.value_or(info.token.c_str()).data();
+    json["error_code"] = static_cast<uint32_t>(err);
+    json["error_message"] = customMessage.value_or(info.message.c_str()).data();
+    json["status"] = "error";
+    json["type"] = "response";
+    return json;
+}
+
+boost::json::object
+makeError(
+    ClioError err,
+    optional<string_view> customError,
+    optional<string_view> customMessage)
+{
+    boost::json::object json;
+    auto const& info = getErrorInfo(err);
+
+    json["error"] = customError.value_or(info.error).data();
+    json["error_code"] = static_cast<uint32_t>(info.code);
+    json["error_message"] = customMessage.value_or(info.message).data();
+    json["status"] = "error";
+    json["type"] = "response";
+    return json;
+}
+
+boost::json::object
+makeError(Status const& status)
+{
+    auto wrapOptional = [](string_view const& str) {
+        return str.empty() ? nullopt : make_optional(str);
+    };
+
+    auto res = visit(
+        overloadSet{
+            [&status, &wrapOptional](RippledError err) {
+                if (err == ripple::rpcUNKNOWN)
+                {
+                    return boost::json::object{
+                        {"error", status.message},
+                        {"type", "response"},
+                        {"status", "error"}};
+                }
+
+                return makeError(
+                    err,
+                    wrapOptional(status.error),
+                    wrapOptional(status.message));
+            },
+            [&status, &wrapOptional](ClioError err) {
+                return makeError(
+                    err,
+                    wrapOptional(status.error),
+                    wrapOptional(status.message));
+            },
+        },
+        status.code);
+    if (status.extraInfo)
+    {
+        for (auto& [key, value] : status.extraInfo.value())
+        {
+            res[key] = value;
+        }
+    }
+    return res;
+}
+
+}  // namespace RPC

src/rpc/Errors.h

@@ -0,0 +1,256 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include <ripple/protocol/ErrorCodes.h>
+
+#include <boost/json/object.hpp>
+
+#include <optional>
+#include <string>
+#include <string_view>
+#include <variant>
+
+namespace RPC {
+
+/**
+ * @brief Custom clio RPC Errors.
+ */
+enum class ClioError {
+    rpcMALFORMED_CURRENCY = 5000,
+    rpcMALFORMED_REQUEST = 5001,
+    rpcMALFORMED_OWNER = 5002,
+    rpcMALFORMED_ADDRESS = 5003,
+};
+
+/**
+ * @brief Holds info about a particular @ref ClioError.
+ */
+struct ClioErrorInfo
+{
+    ClioError const code;
+    std::string_view const error;
+    std::string_view const message;
+};
+
+/**
+ * @brief Clio uses compatible Rippled error codes for most RPC errors.
+ */
+using RippledError = ripple::error_code_i;
+
+/**
+ * @brief Clio operates on a combination of Rippled and Custom Clio error codes.
+ *
+ * @see RippledError For rippled error codes
+ * @see ClioError For custom clio error codes
+ */
+using CombinedError = std::variant<RippledError, ClioError>;
+
+/**
+ * @brief A status returned from any RPC handler.
+ */
+struct Status
+{
+    CombinedError code = RippledError::rpcSUCCESS;
+    std::string error = "";
+    std::string message = "";
+    std::optional<boost::json::object> extraInfo;
+
+    Status() = default;
+    /* implicit */ Status(CombinedError code) : code(code){};
+    Status(CombinedError code, boost::json::object&& extraInfo)
+        : code(code), extraInfo(std::move(extraInfo)){};
+
+    // HACK. Some rippled handlers explicitly specify errors.
+    // This means that we have to be able to duplicate this
+    // functionality.
+    explicit Status(std::string const& message)
+        : code(ripple::rpcUNKNOWN), message(message)
+    {
+    }
+
+    Status(CombinedError code, std::string message)
+        : code(code), message(message)
+    {
+    }
+
+    Status(CombinedError code, std::string error, std::string message)
+        : code(code), error(error), message(message)
+    {
+    }
+
+    /**
+     * @brief Returns true if the Status is *not* OK.
+     */
+    operator bool() const
+    {
+        if (auto err = std::get_if<RippledError>(&code))
+            return *err != RippledError::rpcSUCCESS;
+        return true;
+    }
+
+    /**
+     * @brief Returns true if the Status contains the desired @ref RippledError
+     *
+     * @param other The RippledError to match
+     * @return bool true if status matches given error; false otherwise
+     */
+    bool
+    operator==(RippledError other) const
+    {
+        if (auto err = std::get_if<RippledError>(&code))
+            return *err == other;
+        return false;
+    }
+
+    /**
+     * @brief Returns true if the Status contains the desired @ref ClioError
+     *
+     * @param other The RippledError to match
+     * @return bool true if status matches given error; false otherwise
+     */
+    bool
+    operator==(ClioError other) const
+    {
+        if (auto err = std::get_if<ClioError>(&code))
+            return *err == other;
+        return false;
+    }
+};
+
+/**
+ * @brief Warning codes that can be returned by clio.
+ */
+enum WarningCode {
+    warnUNKNOWN = -1,
+    warnRPC_CLIO = 2001,
+    warnRPC_OUTDATED = 2002,
+    warnRPC_RATE_LIMIT = 2003
+};
+
+/**
+ * @brief Holds information about a clio warning.
+ */
+struct WarningInfo
+{
+    constexpr WarningInfo() = default;
+    constexpr WarningInfo(WarningCode code, char const* message)
+        : code(code), message(message)
+    {
+    }
+
+    WarningCode code = warnUNKNOWN;
+    std::string_view const message = "unknown warning";
+};
+
+/**
+ * @brief Invalid parameters error.
+ */
+class InvalidParamsError : public std::exception
+{
+    std::string msg;
+
+public:
+    explicit InvalidParamsError(std::string const& msg) : msg(msg)
+    {
+    }
+
+    const char*
+    what() const throw() override
+    {
+        return msg.c_str();
+    }
+};
+
+/**
+ * @brief Account not found error.
+ */
+class AccountNotFoundError : public std::exception
+{
+    std::string account;
+
+public:
+    explicit AccountNotFoundError(std::string const& acct) : account(acct)
+    {
+    }
+    const char*
+    what() const throw() override
+    {
+        return account.c_str();
+    }
+};
+
+/**
+ * @brief A globally available @ref Status that represents a successful state
+ */
+static Status OK;
+
+/**
+ * @brief Get the warning info object from a warning code.
+ *
+ * @param code The warning code
+ * @return WarningInfo const& A reference to the static warning info
+ */
+WarningInfo const&
+getWarningInfo(WarningCode code);
+
+/**
+ * @brief Generate JSON from a warning code.
+ *
+ * @param code The @ref WarningCode
+ * @return boost::json::object The JSON output
+ */
+boost::json::object
+makeWarning(WarningCode code);
+
+/**
+ * @brief Generate JSON from a @ref Status.
+ *
+ * @param status The @ref Status
+ * @return boost::json::object The JSON output
+ */
+boost::json::object
+makeError(Status const& status);
+
+/**
+ * @brief Generate JSON from a @ref RippledError.
+ *
+ * @param status The rippled @ref RippledError
+ * @return boost::json::object The JSON output
+ */
+boost::json::object
+makeError(
+    RippledError err,
+    std::optional<std::string_view> customError = std::nullopt,
+    std::optional<std::string_view> customMessage = std::nullopt);
+
+/**
+ * @brief Generate JSON from a @ref ClioError.
+ *
+ * @param status The clio's custom @ref ClioError
+ * @return boost::json::object The JSON output
+ */
+boost::json::object
+makeError(
+    ClioError err,
+    std::optional<std::string_view> customError = std::nullopt,
+    std::optional<std::string_view> customMessage = std::nullopt);
+
+}  // namespace RPC

src/rpc/Handlers.h

@@ -1,5 +1,23 @@
-#ifndef REPORTING_HANDLERS_H_INCLUDED
-#define REPORTING_HANDLERS_H_INCLUDED
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
 
 #include <rpc/RPC.h>
 
@@ -45,7 +63,7 @@ Result
 doChannelVerify(Context const& context);
 
 // book methods
-Result
+[[nodiscard]] Result
 doBookChanges(Context const& context);
 
 Result
@@ -61,6 +79,9 @@ doNFTSellOffers(Context const& context);
 Result
 doNFTInfo(Context const& context);
 
+Result
+doNFTHistory(Context const& context);
+
 // ledger methods
 Result
 doLedger(Context const& context);
@@ -99,4 +120,3 @@ doServerInfo(Context const& context);
 Result
 doRandom(Context const& context);
 }  // namespace RPC
-#endif

src/rpc/RPC.cpp

@@ -1,23 +1,87 @@
-#include <boost/asio/spawn.hpp>
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
 #include <etl/ETLSource.h>
+#include <log/Logger.h>
 #include <rpc/Handlers.h>
 #include <rpc/RPCHelpers.h>
+#include <webserver/HttpBase.h>
+#include <webserver/WsBase.h>
+
+#include <boost/asio/spawn.hpp>
+
 #include <unordered_map>
 
-namespace RPC {
+using namespace std;
+using namespace clio;
 
-std::optional<Context>
+// local to compilation unit loggers
+namespace {
+clio::Logger gPerfLog{"Performance"};
+clio::Logger gLog{"RPC"};
+}  // namespace
+
+namespace RPC {
+Context::Context(
+    boost::asio::yield_context& yield_,
+    string const& command_,
+    uint32_t version_,
+    boost::json::object const& params_,
+    shared_ptr<BackendInterface const> const& backend_,
+    shared_ptr<SubscriptionManager> const& subscriptions_,
+    shared_ptr<ETLLoadBalancer> const& balancer_,
+    shared_ptr<ReportingETL const> const& etl_,
+    shared_ptr<WsBase> const& session_,
+    util::TagDecoratorFactory const& tagFactory_,
+    Backend::LedgerRange const& range_,
+    Counters& counters_,
+    string const& clientIp_)
+    : Taggable(tagFactory_)
+    , yield(yield_)
+    , method(command_)
+    , version(version_)
+    , params(params_)
+    , backend(backend_)
+    , subscriptions(subscriptions_)
+    , balancer(balancer_)
+    , etl(etl_)
+    , session(session_)
+    , range(range_)
+    , counters(counters_)
+    , clientIp(clientIp_)
+{
+    gPerfLog.debug() << tag() << "new Context created";
+}
+
+optional<Context>
 make_WsContext(
     boost::asio::yield_context& yc,
     boost::json::object const& request,
-    std::shared_ptr<BackendInterface const> const& backend,
-    std::shared_ptr<SubscriptionManager> const& subscriptions,
-    std::shared_ptr<ETLLoadBalancer> const& balancer,
-    std::shared_ptr<ReportingETL const> const& etl,
-    std::shared_ptr<WsBase> const& session,
+    shared_ptr<BackendInterface const> const& backend,
+    shared_ptr<SubscriptionManager> const& subscriptions,
+    shared_ptr<ETLLoadBalancer> const& balancer,
+    shared_ptr<ReportingETL const> const& etl,
+    shared_ptr<WsBase> const& session,
+    util::TagDecoratorFactory const& tagFactory,
     Backend::LedgerRange const& range,
     Counters& counters,
-    std::string const& clientIp)
+    string const& clientIp)
 {
     boost::json::value commandValue = nullptr;
     if (!request.contains("command") && request.contains("method"))
@@ -28,9 +92,9 @@ make_WsContext(
     if (!commandValue.is_string())
         return {};
 
-    std::string command = commandValue.as_string().c_str();
+    string command = commandValue.as_string().c_str();
 
-    return Context{
+    return make_optional<Context>(
         yc,
         command,
         1,
@@ -40,27 +104,29 @@ make_WsContext(
         balancer,
         etl,
         session,
+        tagFactory,
         range,
         counters,
-        clientIp};
+        clientIp);
 }
 
-std::optional<Context>
+optional<Context>
 make_HttpContext(
     boost::asio::yield_context& yc,
     boost::json::object const& request,
-    std::shared_ptr<BackendInterface const> const& backend,
-    std::shared_ptr<SubscriptionManager> const& subscriptions,
-    std::shared_ptr<ETLLoadBalancer> const& balancer,
-    std::shared_ptr<ReportingETL const> const& etl,
+    shared_ptr<BackendInterface const> const& backend,
+    shared_ptr<SubscriptionManager> const& subscriptions,
+    shared_ptr<ETLLoadBalancer> const& balancer,
+    shared_ptr<ReportingETL const> const& etl,
+    util::TagDecoratorFactory const& tagFactory,
     Backend::LedgerRange const& range,
     RPC::Counters& counters,
-    std::string const& clientIp)
+    string const& clientIp)
 {
     if (!request.contains("method") || !request.at("method").is_string())
         return {};
 
-    std::string const& command = request.at("method").as_string().c_str();
+    string const& command = request.at("method").as_string().c_str();
 
     if (command == "subscribe" || command == "unsubscribe")
         return {};
@@ -76,7 +142,7 @@ make_HttpContext(
     if (!array.at(0).is_object())
         return {};
 
-    return Context{
+    return make_optional<Context>(
         yc,
         command,
         1,
@@ -86,111 +152,44 @@ make_HttpContext(
         balancer,
         etl,
         nullptr,
+        tagFactory,
         range,
         counters,
-        clientIp};
+        clientIp);
 }
 
-constexpr static WarningInfo warningInfos[]{
-    {warnUNKNOWN, "Unknown warning"},
-    {warnRPC_CLIO,
-     "This is a clio server. clio only serves validated data. If you "
-     "want to talk to rippled, include 'ledger_index':'current' in your "
-     "request"},
-    {warnRPC_OUTDATED, "This server may be out of date"},
-    {warnRPC_RATE_LIMIT, "You are about to be rate limited"}};
-
-WarningInfo const&
-get_warning_info(warning_code code)
-{
-    for (WarningInfo const& info : warningInfos)
-    {
-        if (info.code == code)
-        {
-            return info;
-        }
-    }
-    throw(std::out_of_range("Invalid warning_code"));
-}
-
-boost::json::object
-make_warning(warning_code code)
-{
-    boost::json::object json;
-    WarningInfo const& info(get_warning_info(code));
-    json["id"] = code;
-    json["message"] = static_cast<std::string>(info.message);
-    return json;
-}
-
-boost::json::object
-make_error(Error err)
-{
-    boost::json::object json;
-    ripple::RPC::ErrorInfo const& info(ripple::RPC::get_error_info(err));
-    json["error"] = info.token;
-    json["error_code"] = static_cast<std::uint32_t>(err);
-    json["error_message"] = info.message;
-    json["status"] = "error";
-    json["type"] = "response";
-    return json;
-}
-
-boost::json::object
-make_error(Status const& status)
-{
-    if (status.error == ripple::rpcUNKNOWN)
-    {
-        return {
-            {"error", status.message},
-            {"type", "response"},
-            {"status", "error"}};
-    }
-
-    boost::json::object json;
-    ripple::RPC::ErrorInfo const& info(
-        ripple::RPC::get_error_info(status.error));
-    json["error"] =
-        status.strCode.size() ? status.strCode.c_str() : info.token.c_str();
-    json["error_code"] = static_cast<std::uint32_t>(status.error);
-    json["error_message"] =
-        status.message.size() ? status.message.c_str() : info.message.c_str();
-    json["status"] = "error";
-    json["type"] = "response";
-    return json;
-}
-
-using LimitRange = std::tuple<std::uint32_t, std::uint32_t, std::uint32_t>;
-using HandlerFunction = std::function<Result(Context const&)>;
+using LimitRange = tuple<uint32_t, uint32_t, uint32_t>;
+using HandlerFunction = function<Result(Context const&)>;
 
 struct Handler
 {
-    std::string method;
-    std::function<Result(Context const&)> handler;
-    std::optional<LimitRange> limit;
+    string method;
+    function<Result(Context const&)> handler;
+    optional<LimitRange> limit;
+    bool isClioOnly = false;
 };
 
 class HandlerTable
 {
-    std::unordered_map<std::string, Handler> handlerMap_;
+    unordered_map<string, Handler> handlerMap_;
 
 public:
-    HandlerTable(std::initializer_list<Handler> handlers)
+    HandlerTable(initializer_list<Handler> handlers)
     {
         for (auto const& handler : handlers)
         {
-            handlerMap_[handler.method] = std::move(handler);
+            handlerMap_[handler.method] = move(handler);
         }
     }
 
     bool
-    contains(std::string const& method)
+    contains(string const& method)
     {
         return handlerMap_.contains(method);
     }
 
-    std::optional<LimitRange>
-    getLimitRange(std::string const& command)
+    optional<LimitRange>
+    getLimitRange(string const& command)
     {
         if (!handlerMap_.contains(command))
             return {};
@@ -198,14 +197,20 @@ public:
         return handlerMap_[command].limit;
     }
 
-    std::optional<HandlerFunction>
-    getHandler(std::string const& command)
+    optional<HandlerFunction>
+    getHandler(string const& command)
     {
         if (!handlerMap_.contains(command))
             return {};
 
         return handlerMap_[command].handler;
     }
+
+    bool
+    isClioOnly(string const& command)
+    {
+        return handlerMap_.contains(command) && handlerMap_[command].isClioOnly;
+    }
 };
 
 static HandlerTable handlerTable{
@@ -218,13 +223,14 @@ static HandlerTable handlerTable{
     {"account_offers", &doAccountOffers, LimitRange{10, 50, 256}},
     {"account_tx", &doAccountTx, LimitRange{1, 50, 100}},
     {"gateway_balances", &doGatewayBalances, {}},
-    {"noripple_check", &doNoRippleCheck, {}},
+    {"noripple_check", &doNoRippleCheck, LimitRange{1, 300, 500}},
     {"book_changes", &doBookChanges, {}},
     {"book_offers", &doBookOffers, LimitRange{1, 50, 100}},
     {"ledger", &doLedger, {}},
     {"ledger_data", &doLedgerData, LimitRange{1, 100, 2048}},
     {"nft_buy_offers", &doNFTBuyOffers, LimitRange{1, 50, 100}},
-    {"nft_info", &doNFTInfo},
+    {"nft_history", &doNFTHistory, LimitRange{1, 50, 100}, true},
+    {"nft_info", &doNFTInfo, {}, true},
     {"nft_sell_offers", &doNFTSellOffers, LimitRange{1, 50, 100}},
     {"ledger_entry", &doLedgerEntry, {}},
     {"ledger_range", &doLedgerRange, {}},
@@ -235,7 +241,7 @@ static HandlerTable handlerTable{
     {"transaction_entry", &doTransactionEntry, {}},
     {"random", &doRandom, {}}};
 
-static std::unordered_set<std::string> forwardCommands{
+static unordered_set<string> forwardCommands{
     "submit",
     "submit_multisigned",
     "fee",
@@ -247,32 +253,47 @@ static std::unordered_set<std::string> forwardCommands{
     "channel_verify"};
 
 bool
-validHandler(std::string const& method)
+validHandler(string const& method)
 {
     return handlerTable.contains(method) || forwardCommands.contains(method);
 }
 
+bool
+isClioOnly(string const& method)
+{
+    return handlerTable.isClioOnly(method);
+}
+
+bool
+shouldSuppressValidatedFlag(RPC::Context const& context)
+{
+    return boost::iequals(context.method, "subscribe") ||
+        boost::iequals(context.method, "unsubscribe");
+}
+
 Status
-getLimit(RPC::Context const& context, std::uint32_t& limit)
+getLimit(RPC::Context const& context, uint32_t& limit)
 {
     if (!handlerTable.getHandler(context.method))
-        return Status{Error::rpcUNKNOWN_COMMAND};
+        return Status{RippledError::rpcUNKNOWN_COMMAND};
 
     if (!handlerTable.getLimitRange(context.method))
-        return Status{Error::rpcINVALID_PARAMS, "rpcDoesNotRequireLimit"};
+        return Status{
+            RippledError::rpcINVALID_PARAMS, "rpcDoesNotRequireLimit"};
 
     auto [lo, def, hi] = *handlerTable.getLimitRange(context.method);
 
     if (context.params.contains(JS(limit)))
     {
+        string errMsg = "Invalid field 'limit', not unsigned integer.";
         if (!context.params.at(JS(limit)).is_int64())
-            return Status{Error::rpcINVALID_PARAMS, "limitNotInt"};
+            return Status{RippledError::rpcINVALID_PARAMS, errMsg};
 
-        limit = context.params.at(JS(limit)).as_int64();
-        if (limit <= 0)
-            return Status{Error::rpcINVALID_PARAMS, "limitNotPositive"};
+        int input = context.params.at(JS(limit)).as_int64();
+        if (input <= 0)
+            return Status{RippledError::rpcINVALID_PARAMS, errMsg};
 
-        limit = std::clamp(limit, lo, hi);
+        limit = clamp(static_cast<uint32_t>(input), lo, hi);
     }
     else
     {
@@ -287,6 +308,9 @@ shouldForwardToRippled(Context const& ctx)
 {
     auto request = ctx.params;
 
+    if (isClioOnly(ctx.method))
+        return false;
+
     if (forwardCommands.find(ctx.method) != forwardCommands.end())
         return true;
 
@@ -314,10 +338,7 @@ buildResponse(Context const& ctx)
         ctx.counters.rpcForwarded(ctx.method);
 
         if (!res)
-            return Status{Error::rpcFAILED_TO_FORWARD};
-
-        if (res->contains("result") && res->at("result").is_object())
-            return res->at("result").as_object();
+            return Status{RippledError::rpcFAILED_TO_FORWARD};
 
         return *res;
     }
@@ -325,33 +346,50 @@ buildResponse(Context const& ctx)
     if (ctx.method == "ping")
         return boost::json::object{};
 
+    if (ctx.backend->isTooBusy())
+    {
+        gLog.error() << "Database is too busy. Rejecting request";
+        return Status{RippledError::rpcTOO_BUSY};
+    }
+
     auto method = handlerTable.getHandler(ctx.method);
 
     if (!method)
-        return Status{Error::rpcUNKNOWN_COMMAND};
+        return Status{RippledError::rpcUNKNOWN_COMMAND};
 
     try
     {
+        gPerfLog.debug() << ctx.tag() << " start executing rpc `" << ctx.method
+                         << '`';
         auto v = (*method)(ctx);
+        gPerfLog.debug() << ctx.tag() << " finish executing rpc `" << ctx.method
+                         << '`';
 
-        if (auto object = std::get_if<boost::json::object>(&v))
-            (*object)["validated"] = true;
+        if (auto object = get_if<boost::json::object>(&v);
+            object && not shouldSuppressValidatedFlag(ctx))
+        {
+            (*object)[JS(validated)] = true;
+        }
 
         return v;
     }
     catch (InvalidParamsError const& err)
     {
-        return Status{Error::rpcINVALID_PARAMS, err.what()};
+        return Status{RippledError::rpcINVALID_PARAMS, err.what()};
     }
     catch (AccountNotFoundError const& err)
     {
-        return Status{Error::rpcACT_NOT_FOUND, err.what()};
+        return Status{RippledError::rpcACT_NOT_FOUND, err.what()};
     }
-    catch (std::exception const& err)
+    catch (Backend::DatabaseTimeout const& t)
     {
-        BOOST_LOG_TRIVIAL(error)
-            << __func__ << " caught exception : " << err.what();
-        return Status{Error::rpcINTERNAL};
+        gLog.error() << "Database timeout";
+        return Status{RippledError::rpcTOO_BUSY};
+    }
+    catch (exception const& err)
+    {
+        gLog.error() << ctx.tag() << " caught exception: " << err.what();
+        return Status{RippledError::rpcINTERNAL};
     }
 }
 

src/rpc/RPC.h

@@ -1,14 +1,37 @@
-#ifndef REPORTING_RPC_H_INCLUDED
-#define REPORTING_RPC_H_INCLUDED
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include <backend/BackendInterface.h>
+#include <log/Logger.h>
+#include <rpc/Counters.h>
+#include <rpc/Errors.h>
+#include <util/Taggable.h>
 
-#include <ripple/protocol/ErrorCodes.h>
 #include <boost/asio/spawn.hpp>
 #include <boost/json.hpp>
-#include <backend/BackendInterface.h>
+
 #include <optional>
-#include <rpc/Counters.h>
 #include <string>
 #include <variant>
+
 /*
  * This file contains various classes necessary for executing RPC handlers.
  * Context gives the handlers access to various other parts of the application
@@ -27,8 +50,9 @@ class ReportingETL;
 
 namespace RPC {
 
-struct Context
+struct Context : public util::Taggable
 {
+    clio::Logger perfLog_{"Performance"};
     boost::asio::yield_context& yield;
     std::string method;
     std::uint32_t version;
@@ -55,25 +79,11 @@ struct Context
         std::shared_ptr<ETLLoadBalancer> const& balancer_,
         std::shared_ptr<ReportingETL const> const& etl_,
         std::shared_ptr<WsBase> const& session_,
+        util::TagDecoratorFactory const& tagFactory_,
         Backend::LedgerRange const& range_,
         Counters& counters_,
-        std::string const& clientIp_)
-        : yield(yield_)
-        , method(command_)
-        , version(version_)
-        , params(params_)
-        , backend(backend_)
-        , subscriptions(subscriptions_)
-        , balancer(balancer_)
-        , etl(etl_)
-        , session(session_)
-        , range(range_)
-        , counters(counters_)
-        , clientIp(clientIp_)
-    {
-    }
+        std::string const& clientIp_);
 };
-using Error = ripple::error_code_i;
 
 struct AccountCursor
 {
@@ -81,120 +91,20 @@ struct AccountCursor
     std::uint32_t hint;
 
     std::string
-    toString()
+    toString() const
     {
         return ripple::strHex(index) + "," + std::to_string(hint);
     }
 
     bool
-    isNonZero()
+    isNonZero() const
     {
         return index.isNonZero() || hint != 0;
     }
 };
 
-struct Status
-{
-    Error error = Error::rpcSUCCESS;
-    std::string strCode = "";
-    std::string message = "";
-
-    Status(){};
-
-    Status(Error error_) : error(error_){};
-
-    // HACK. Some rippled handlers explicitly specify errors.
-    // This means that we have to be able to duplicate this
-    // functionality.
-    Status(std::string const& message_)
-        : error(ripple::rpcUNKNOWN), message(message_)
-    {
-    }
-
-    Status(Error error_, std::string message_)
-        : error(error_), message(message_)
-    {
-    }
-
-    Status(Error error_, std::string strCode_, std::string message_)
-        : error(error_), strCode(strCode_), message(message_)
-    {
-    }
-
-    /** Returns true if the Status is *not* OK. */
-    operator bool() const
-    {
-        return error != Error::rpcSUCCESS;
-    }
-};
-
-static Status OK;
-
 using Result = std::variant<Status, boost::json::object>;
 
-class InvalidParamsError : public std::exception
-{
-    std::string msg;
-
-public:
-    InvalidParamsError(std::string const& msg) : msg(msg)
-    {
-    }
-
-    const char*
-    what() const throw() override
-    {
-        return msg.c_str();
-    }
-};
-class AccountNotFoundError : public std::exception
-{
-    std::string account;
-
-public:
-    AccountNotFoundError(std::string const& acct) : account(acct)
-    {
-    }
-    const char*
-    what() const throw() override
-    {
-        return account.c_str();
-    }
-};
-
-enum warning_code {
-    warnUNKNOWN = -1,
-    warnRPC_CLIO = 2001,
-    warnRPC_OUTDATED = 2002,
-    warnRPC_RATE_LIMIT = 2003
-};
-
-struct WarningInfo
-{
-    constexpr WarningInfo() : code(warnUNKNOWN), message("unknown warning")
-    {
-    }
-
-    constexpr WarningInfo(warning_code code_, char const* message_)
-        : code(code_), message(message_)
-    {
-    }
-    warning_code code;
-    std::string_view const message;
-};
-
-WarningInfo const&
-get_warning_info(warning_code code);
-
-boost::json::object
-make_warning(warning_code code);
-
-boost::json::object
-make_error(Status const& status);
-
-boost::json::object
-make_error(Error err);
-
 std::optional<Context>
 make_WsContext(
     boost::asio::yield_context& yc,
@@ -204,6 +114,7 @@ make_WsContext(
     std::shared_ptr<ETLLoadBalancer> const& balancer,
     std::shared_ptr<ReportingETL const> const& etl,
     std::shared_ptr<WsBase> const& session,
+    util::TagDecoratorFactory const& tagFactory,
     Backend::LedgerRange const& range,
     Counters& counters,
     std::string const& clientIp);
@@ -216,6 +127,7 @@ make_HttpContext(
     std::shared_ptr<SubscriptionManager> const& subscriptions,
     std::shared_ptr<ETLLoadBalancer> const& balancer,
     std::shared_ptr<ReportingETL const> const& etl,
+    util::TagDecoratorFactory const& tagFactory,
     Backend::LedgerRange const& range,
     Counters& counters,
     std::string const& clientIp);
@@ -226,6 +138,9 @@ buildResponse(Context const& ctx);
 bool
 validHandler(std::string const& method);
 
+bool
+isClioOnly(std::string const& method);
+
 Status
 getLimit(RPC::Context const& context, std::uint32_t& limit);
 
@@ -233,20 +148,19 @@ template <class T>
 void
 logDuration(Context const& ctx, T const& dur)
 {
+    static clio::Logger log{"RPC"};
     std::stringstream ss;
-    ss << "Request processing duration = "
+    ss << ctx.tag() << "Request processing duration = "
        << std::chrono::duration_cast<std::chrono::milliseconds>(dur).count()
        << " milliseconds. request = " << ctx.params;
     auto seconds =
         std::chrono::duration_cast<std::chrono::seconds>(dur).count();
     if (seconds > 10)
-        BOOST_LOG_TRIVIAL(error) << ss.str();
+        log.error() << ss.str();
     else if (seconds > 1)
-        BOOST_LOG_TRIVIAL(warning) << ss.str();
+        log.warn() << ss.str();
     else
-        BOOST_LOG_TRIVIAL(info) << ss.str();
+        log.info() << ss.str();
 }
 
 }  // namespace RPC
-
-#endif  // REPORTING_RPC_H_INCLUDED

src/rpc/RPCHelpers.cpp

@@ -1,6 +1,38 @@
-#include <boost/algorithm/string.hpp>
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include <ripple/basics/StringUtilities.h>
 #include <backend/BackendInterface.h>
+#include <log/Logger.h>
 #include <rpc/RPCHelpers.h>
+#include <util/Profiler.h>
+
+#include <boost/algorithm/string.hpp>
+#include <boost/format.hpp>
+
+using namespace clio;
+
+// local to compilation unit loggers
+namespace {
+clio::Logger gLog{"RPC"};
+}  // namespace
+
 namespace RPC {
 
 std::optional<bool>
@@ -69,35 +101,11 @@ getRequiredUInt(boost::json::object const& request, std::string const& field)
         throw InvalidParamsError("Missing field " + field);
 }
 
-bool
-isOwnedByAccount(ripple::SLE const& sle, ripple::AccountID const& accountID)
-{
-    if (sle.getType() == ripple::ltRIPPLE_STATE)
-    {
-        return (sle.getFieldAmount(ripple::sfLowLimit).getIssuer() ==
-                accountID) ||
-            (sle.getFieldAmount(ripple::sfHighLimit).getIssuer() == accountID);
-    }
-    else if (sle.isFieldPresent(ripple::sfAccount))
-    {
-        return sle.getAccountID(ripple::sfAccount) == accountID;
-    }
-    else if (sle.getType() == ripple::ltSIGNER_LIST)
-    {
-        ripple::Keylet const accountSignerList =
-            ripple::keylet::signers(accountID);
-        return sle.key() == accountSignerList.key;
-    }
-
-    return false;
-}
-
 std::optional<AccountCursor>
 parseAccountCursor(
     BackendInterface const& backend,
     std::uint32_t seq,
     std::optional<std::string> jsonCursor,
-    ripple::AccountID const& accountID,
     boost::asio::yield_context& yield)
 {
     ripple::uint256 cursorIndex = beast::zero;
@@ -128,19 +136,6 @@ parseAccountCursor(
         return {};
     }
 
-    // We then must check if the object pointed to by the marker is actually
-    // owned by the account in the request.
-    auto const ownedNode = backend.fetchLedgerObject(cursorIndex, seq, yield);
-
-    if (!ownedNode)
-        return {};
-
-    ripple::SerialIter it{ownedNode->data(), ownedNode->size()};
-    ripple::SLE sle{it, cursorIndex};
-
-    if (!isOwnedByAccount(sle, accountID))
-        return {};
-
     return AccountCursor({cursorIndex, startHint});
 }
 
@@ -182,10 +177,10 @@ getHexMarker(boost::json::object const& request, ripple::uint256& marker)
     if (request.contains(JS(marker)))
     {
         if (!request.at(JS(marker)).is_string())
-            return Status{Error::rpcINVALID_PARAMS, "markerNotString"};
+            return Status{RippledError::rpcINVALID_PARAMS, "markerNotString"};
 
         if (!marker.parseHex(request.at(JS(marker)).as_string().c_str()))
-            return Status{Error::rpcINVALID_PARAMS, "malformedMarker"};
+            return Status{RippledError::rpcINVALID_PARAMS, "malformedMarker"};
     }
 
     return {};
@@ -202,14 +197,14 @@ getAccount(
     {
         if (required)
             return Status{
-                Error::rpcINVALID_PARAMS, field.to_string() + "Missing"};
+                RippledError::rpcINVALID_PARAMS, field.to_string() + "Missing"};
 
         return {};
     }
 
     if (!request.at(field).is_string())
         return Status{
-            Error::rpcINVALID_PARAMS, field.to_string() + "NotString"};
+            RippledError::rpcINVALID_PARAMS, field.to_string() + "NotString"};
 
     if (auto a = accountFromStringStrict(request.at(field).as_string().c_str());
         a)
@@ -218,7 +213,8 @@ getAccount(
         return {};
     }
 
-    return Status{Error::rpcINVALID_PARAMS, field.to_string() + "Malformed"};
+    return Status{
+        RippledError::rpcACT_MALFORMED, field.to_string() + "Malformed"};
 }
 
 Status
@@ -235,7 +231,7 @@ getOptionalAccount(
 
     if (!request.at(field).is_string())
         return Status{
-            Error::rpcINVALID_PARAMS, field.to_string() + "NotString"};
+            RippledError::rpcINVALID_PARAMS, field.to_string() + "NotString"};
 
     if (auto a = accountFromStringStrict(request.at(field).as_string().c_str());
         a)
@@ -244,7 +240,8 @@ getOptionalAccount(
         return {};
     }
 
-    return Status{Error::rpcINVALID_PARAMS, field.to_string() + "Malformed"};
+    return Status{
+        RippledError::rpcINVALID_PARAMS, field.to_string() + "Malformed"};
 }
 
 Status
@@ -268,7 +265,7 @@ getTaker(boost::json::object const& request, ripple::AccountID& takerID)
     if (request.contains(JS(taker)))
     {
         auto parsed = parseTaker(request.at(JS(taker)));
-        if (auto status = std::get_if<Status>(&parsed))
+        if (auto status = std::get_if<Status>(&parsed); status)
             return *status;
         else
             takerID = std::get<ripple::AccountID>(parsed);
@@ -281,13 +278,13 @@ Status
 getChannelId(boost::json::object const& request, ripple::uint256& channelId)
 {
     if (!request.contains(JS(channel_id)))
-        return Status{Error::rpcINVALID_PARAMS, "missingChannelID"};
+        return Status{RippledError::rpcINVALID_PARAMS, "missingChannelID"};
 
     if (!request.at(JS(channel_id)).is_string())
-        return Status{Error::rpcINVALID_PARAMS, "channelIDNotString"};
+        return Status{RippledError::rpcINVALID_PARAMS, "channelIDNotString"};
 
     if (!channelId.parseHex(request.at(JS(channel_id)).as_string().c_str()))
-        return Status{Error::rpcCHANNEL_MALFORMED, "malformedChannelID"};
+        return Status{RippledError::rpcCHANNEL_MALFORMED, "malformedChannelID"};
 
     return {};
 }
@@ -342,20 +339,6 @@ canHaveDeliveredAmount(
     return true;
 }
 
-std::optional<ripple::AccountID>
-accountFromSeed(std::string const& account)
-{
-    auto const seed = ripple::parseGenericSeed(account);
-
-    if (!seed)
-        return {};
-
-    auto const keypair =
-        ripple::generateKeyPair(ripple::KeyType::secp256k1, *seed);
-
-    return ripple::calcAccountID(keypair.first);
-}
-
 std::optional<ripple::AccountID>
 accountFromStringStrict(std::string const& account)
 {
@@ -419,12 +402,11 @@ deserializeTxPlusMeta(Backend::TransactionAndMetadata const& blobs)
             blobs.metadata.begin(),
             blobs.metadata.end(),
             std::ostream_iterator<unsigned char>(meta));
-        BOOST_LOG_TRIVIAL(error)
-            << __func__
-            << " Failed to deserialize transaction. txn = " << txn.str()
-            << " - meta = " << meta.str()
-            << " txn length = " << std::to_string(blobs.transaction.size())
-            << " meta length = " << std::to_string(blobs.metadata.size());
+        gLog.error() << "Failed to deserialize transaction. txn = " << txn.str()
+                     << " - meta = " << meta.str() << " txn length = "
+                     << std::to_string(blobs.transaction.size())
+                     << " meta length = "
+                     << std::to_string(blobs.metadata.size());
         throw e;
     }
 }
@@ -563,16 +545,18 @@ ledgerInfoFromRequest(Context const& ctx)
     if (!hashValue.is_null())
     {
         if (!hashValue.is_string())
-            return Status{Error::rpcINVALID_PARAMS, "ledgerHashNotString"};
+            return Status{
+                RippledError::rpcINVALID_PARAMS, "ledgerHashNotString"};
 
         ripple::uint256 ledgerHash;
         if (!ledgerHash.parseHex(hashValue.as_string().c_str()))
-            return Status{Error::rpcINVALID_PARAMS, "ledgerHashMalformed"};
+            return Status{
+                RippledError::rpcINVALID_PARAMS, "ledgerHashMalformed"};
 
         auto lgrInfo = ctx.backend->fetchLedgerByHash(ledgerHash, ctx.yield);
 
         if (!lgrInfo || lgrInfo->seq > ctx.range.maxSequence)
-            return Status{Error::rpcLGR_NOT_FOUND, "ledgerNotFound"};
+            return Status{RippledError::rpcLGR_NOT_FOUND, "ledgerNotFound"};
 
         return *lgrInfo;
     }
@@ -601,13 +585,13 @@ ledgerInfoFromRequest(Context const& ctx)
     }
 
     if (!ledgerSequence)
-        return Status{Error::rpcLGR_NOT_FOUND, "ledgerIndexMalformed"};
+        return Status{RippledError::rpcINVALID_PARAMS, "ledgerIndexMalformed"};
 
     auto lgrInfo =
         ctx.backend->fetchLedgerBySequence(*ledgerSequence, ctx.yield);
 
     if (!lgrInfo || lgrInfo->seq > ctx.range.maxSequence)
-        return Status{Error::rpcLGR_NOT_FOUND, "ledgerNotFound"};
+        return Status{RippledError::rpcLGR_NOT_FOUND, "ledgerNotFound"};
 
     return *lgrInfo;
 }
@@ -656,15 +640,17 @@ traverseOwnedNodes(
     std::uint32_t limit,
     std::optional<std::string> jsonCursor,
     boost::asio::yield_context& yield,
-    std::function<void(ripple::SLE)> atOwnedNode)
+    std::function<void(ripple::SLE&&)> atOwnedNode)
 {
-    auto parsedCursor =
-        parseAccountCursor(backend, sequence, jsonCursor, accountID, yield);
+    if (!backend.fetchLedgerObject(
+            ripple::keylet::account(accountID).key, sequence, yield))
+        return Status{RippledError::rpcACT_NOT_FOUND};
 
-    if (!parsedCursor)
+    auto maybeCursor = parseAccountCursor(backend, sequence, jsonCursor, yield);
+    if (!maybeCursor)
         return Status(ripple::rpcINVALID_PARAMS, "Malformed cursor");
 
-    auto [hexCursor, startHint] = *parsedCursor;
+    auto [hexCursor, startHint] = *maybeCursor;
 
     return traverseOwnedNodes(
         backend,
@@ -688,7 +674,7 @@ traverseOwnedNodes(
     std::uint32_t limit,
     std::optional<std::string> jsonCursor,
     boost::asio::yield_context& yield,
-    std::function<void(ripple::SLE)> atOwnedNode)
+    std::function<void(ripple::SLE&&)> atOwnedNode)
 {
     auto cursor = AccountCursor({beast::zero, 0});
 
@@ -710,22 +696,21 @@ traverseOwnedNodes(
         auto hintDir =
             backend.fetchLedgerObject(hintIndex.key, sequence, yield);
 
-        if (hintDir)
-        {
-            ripple::SerialIter it{hintDir->data(), hintDir->size()};
-            ripple::SLE sle{it, hintIndex.key};
+        if (!hintDir)
+            return Status(ripple::rpcINVALID_PARAMS, "Invalid marker");
 
-            for (auto const& key : sle.getFieldV256(ripple::sfIndexes))
-            {
-                if (key == hexMarker)
-                {
-                    // We found the hint, we can start here
-                    currentIndex = hintIndex;
-                    break;
-                }
-            }
+        ripple::SerialIter it{hintDir->data(), hintDir->size()};
+        ripple::SLE sle{it, hintIndex.key};
+
+        if (auto const& indexes = sle.getFieldV256(ripple::sfIndexes);
+            std::find(std::begin(indexes), std::end(indexes), hexMarker) ==
+            std::end(indexes))
+        {
+            // result in empty dataset
+            return AccountCursor({beast::zero, 0});
         }
 
+        currentIndex = hintIndex;
         bool found = false;
         for (;;)
         {
@@ -808,22 +793,22 @@ traverseOwnedNodes(
     }
     auto end = std::chrono::system_clock::now();
 
-    BOOST_LOG_TRIVIAL(debug) << "Time loading owned directories: "
-                             << ((end - start).count() / 1000000000.0);
+    gLog.debug() << "Time loading owned directories: "
+                 << std::chrono::duration_cast<std::chrono::milliseconds>(
+                        end - start)
+                        .count()
+                 << " milliseconds";
 
-    start = std::chrono::system_clock::now();
-    auto objects = backend.fetchLedgerObjects(keys, sequence, yield);
-    end = std::chrono::system_clock::now();
+    auto [objects, timeDiff] = util::timed(
+        [&]() { return backend.fetchLedgerObjects(keys, sequence, yield); });
 
-    BOOST_LOG_TRIVIAL(debug) << "Time loading owned entries: "
-                             << ((end - start).count() / 1000000000.0);
+    gLog.debug() << "Time loading owned entries: " << timeDiff
+                 << " milliseconds";
 
     for (auto i = 0; i < objects.size(); ++i)
     {
         ripple::SerialIter it{objects[i].data(), objects[i].size()};
-        ripple::SLE sle(it, keys[i]);
-
-        atOwnedNode(sle);
+        atOwnedNode(ripple::SLE{it, keys[i]});
     }
 
     if (limit == 0)
@@ -892,12 +877,12 @@ keypairFromRequst(boost::json::object const& request)
     }
 
     if (count == 0)
-        return Status{Error::rpcINVALID_PARAMS, "missing field secret"};
+        return Status{RippledError::rpcINVALID_PARAMS, "missing field secret"};
 
     if (count > 1)
     {
         return Status{
-            Error::rpcINVALID_PARAMS,
+            RippledError::rpcINVALID_PARAMS,
             "Exactly one of the following must be specified: "
             " passphrase, secret, seed, or seed_hex"};
     }
@@ -908,17 +893,18 @@ keypairFromRequst(boost::json::object const& request)
     if (has_key_type)
     {
         if (!request.at("key_type").is_string())
-            return Status{Error::rpcINVALID_PARAMS, "keyTypeNotString"};
+            return Status{RippledError::rpcINVALID_PARAMS, "keyTypeNotString"};
 
         std::string key_type = request.at("key_type").as_string().c_str();
         keyType = ripple::keyTypeFromString(key_type);
 
         if (!keyType)
-            return Status{Error::rpcINVALID_PARAMS, "invalidFieldKeyType"};
+            return Status{
+                RippledError::rpcINVALID_PARAMS, "invalidFieldKeyType"};
 
         if (secretType == "secret")
             return Status{
-                Error::rpcINVALID_PARAMS,
+                RippledError::rpcINVALID_PARAMS,
                 "The secret field is not allowed if key_type is used."};
     }
 
@@ -936,7 +922,7 @@ keypairFromRequst(boost::json::object const& request)
             if (keyType.value_or(ripple::KeyType::ed25519) !=
                 ripple::KeyType::ed25519)
                 return Status{
-                    Error::rpcINVALID_PARAMS,
+                    RippledError::rpcINVALID_PARAMS,
                     "Specified seed is for an Ed25519 wallet."};
 
             keyType = ripple::KeyType::ed25519;
@@ -952,7 +938,8 @@ keypairFromRequst(boost::json::object const& request)
         {
             if (!request.at(secretType).is_string())
                 return Status{
-                    Error::rpcINVALID_PARAMS, "secret value must be string"};
+                    RippledError::rpcINVALID_PARAMS,
+                    "secret value must be string"};
 
             std::string key = request.at(secretType).as_string().c_str();
 
@@ -971,7 +958,7 @@ keypairFromRequst(boost::json::object const& request)
         {
             if (!request.at("secret").is_string())
                 return Status{
-                    Error::rpcINVALID_PARAMS,
+                    RippledError::rpcINVALID_PARAMS,
                     "field secret should be a string"};
 
             std::string secret = request.at("secret").as_string().c_str();
@@ -981,12 +968,14 @@ keypairFromRequst(boost::json::object const& request)
 
     if (!seed)
         return Status{
-            Error::rpcBAD_SEED, "Bad Seed: invalid field message secretType"};
+            RippledError::rpcBAD_SEED,
+            "Bad Seed: invalid field message secretType"};
 
     if (keyType != ripple::KeyType::secp256k1 &&
         keyType != ripple::KeyType::ed25519)
         return Status{
-            Error::rpcINVALID_PARAMS, "keypairForSignature: invalid key type"};
+            RippledError::rpcINVALID_PARAMS,
+            "keypairForSignature: invalid key type"};
 
     return generateKeyPair(*keyType, *seed);
 }
@@ -1334,7 +1323,7 @@ postProcessOrderBook(
         }
         catch (std::exception const& e)
         {
-            BOOST_LOG_TRIVIAL(error) << "caught exception: " << e.what();
+            gLog.error() << "caught exception: " << e.what();
         }
     }
     return jsonOffers;
@@ -1344,54 +1333,62 @@ std::variant<Status, ripple::Book>
 parseBook(boost::json::object const& request)
 {
     if (!request.contains("taker_pays"))
-        return Status{Error::rpcINVALID_PARAMS, "missingTakerPays"};
+        return Status{
+            RippledError::rpcINVALID_PARAMS, "Missing field 'taker_pays'"};
 
     if (!request.contains("taker_gets"))
-        return Status{Error::rpcINVALID_PARAMS, "missingTakerGets"};
+        return Status{
+            RippledError::rpcINVALID_PARAMS, "Missing field 'taker_gets'"};
 
     if (!request.at("taker_pays").is_object())
-        return Status{Error::rpcINVALID_PARAMS, "takerPaysNotObject"};
+        return Status{
+            RippledError::rpcINVALID_PARAMS,
+            "Field 'taker_pays' is not an object"};
 
     if (!request.at("taker_gets").is_object())
-        return Status{Error::rpcINVALID_PARAMS, "takerGetsNotObject"};
+        return Status{
+            RippledError::rpcINVALID_PARAMS,
+            "Field 'taker_gets' is not an object"};
 
     auto taker_pays = request.at("taker_pays").as_object();
     if (!taker_pays.contains("currency"))
-        return Status{Error::rpcINVALID_PARAMS, "missingTakerPaysCurrency"};
+        return Status{RippledError::rpcSRC_CUR_MALFORMED};
 
     if (!taker_pays.at("currency").is_string())
-        return Status{Error::rpcINVALID_PARAMS, "takerPaysCurrencyNotString"};
+        return Status{RippledError::rpcSRC_CUR_MALFORMED};
 
     auto taker_gets = request.at("taker_gets").as_object();
     if (!taker_gets.contains("currency"))
-        return Status{Error::rpcINVALID_PARAMS, "missingTakerGetsCurrency"};
+        return Status{RippledError::rpcDST_AMT_MALFORMED};
 
     if (!taker_gets.at("currency").is_string())
-        return Status{Error::rpcINVALID_PARAMS, "takerGetsCurrencyNotString"};
+        return Status{
+            RippledError::rpcDST_AMT_MALFORMED,
+        };
 
     ripple::Currency pay_currency;
     if (!ripple::to_currency(
             pay_currency, taker_pays.at("currency").as_string().c_str()))
-        return Status{Error::rpcINVALID_PARAMS, "badTakerPaysCurrency"};
+        return Status{RippledError::rpcSRC_CUR_MALFORMED};
 
     ripple::Currency get_currency;
     if (!ripple::to_currency(
             get_currency, taker_gets["currency"].as_string().c_str()))
-        return Status{Error::rpcINVALID_PARAMS, "badTakerGetsCurrency"};
+        return Status{RippledError::rpcDST_AMT_MALFORMED};
 
     ripple::AccountID pay_issuer;
     if (taker_pays.contains("issuer"))
     {
         if (!taker_pays.at("issuer").is_string())
-            return Status{Error::rpcINVALID_PARAMS, "takerPaysIssuerNotString"};
+            return Status{
+                RippledError::rpcINVALID_PARAMS, "takerPaysIssuerNotString"};
 
         if (!ripple::to_issuer(
                 pay_issuer, taker_pays.at("issuer").as_string().c_str()))
-            return Status{Error::rpcINVALID_PARAMS, "badTakerPaysIssuer"};
+            return Status{RippledError::rpcSRC_ISR_MALFORMED};
 
         if (pay_issuer == ripple::noAccount())
-            return Status{
-                Error::rpcINVALID_PARAMS, "badTakerPaysIssuerAccountOne"};
+            return Status{RippledError::rpcSRC_ISR_MALFORMED};
     }
     else
     {
@@ -1400,33 +1397,38 @@ parseBook(boost::json::object const& request)
 
     if (isXRP(pay_currency) && !isXRP(pay_issuer))
         return Status{
-            Error::rpcINVALID_PARAMS,
+            RippledError::rpcSRC_ISR_MALFORMED,
             "Unneeded field 'taker_pays.issuer' for XRP currency "
             "specification."};
 
     if (!isXRP(pay_currency) && isXRP(pay_issuer))
         return Status{
-            Error::rpcINVALID_PARAMS,
+            RippledError::rpcSRC_ISR_MALFORMED,
             "Invalid field 'taker_pays.issuer', expected non-XRP "
             "issuer."};
 
+    if ((!isXRP(pay_currency)) && (!taker_pays.contains("issuer")))
+        return Status{
+            RippledError::rpcSRC_ISR_MALFORMED, "Missing non-XRP issuer."};
+
     ripple::AccountID get_issuer;
 
     if (taker_gets.contains("issuer"))
     {
         if (!taker_gets["issuer"].is_string())
             return Status{
-                Error::rpcINVALID_PARAMS, "taker_gets.issuer should be string"};
+                RippledError::rpcINVALID_PARAMS,
+                "taker_gets.issuer should be string"};
 
         if (!ripple::to_issuer(
                 get_issuer, taker_gets.at("issuer").as_string().c_str()))
             return Status{
-                Error::rpcINVALID_PARAMS,
+                RippledError::rpcDST_ISR_MALFORMED,
                 "Invalid field 'taker_gets.issuer', bad issuer."};
 
         if (get_issuer == ripple::noAccount())
             return Status{
-                Error::rpcINVALID_PARAMS,
+                RippledError::rpcDST_ISR_MALFORMED,
                 "Invalid field 'taker_gets.issuer', bad issuer account "
                 "one."};
     }
@@ -1437,17 +1439,17 @@ parseBook(boost::json::object const& request)
 
     if (ripple::isXRP(get_currency) && !ripple::isXRP(get_issuer))
         return Status{
-            Error::rpcINVALID_PARAMS,
+            RippledError::rpcDST_ISR_MALFORMED,
             "Unneeded field 'taker_gets.issuer' for XRP currency "
             "specification."};
 
     if (!ripple::isXRP(get_currency) && ripple::isXRP(get_issuer))
         return Status{
-            Error::rpcINVALID_PARAMS,
+            RippledError::rpcDST_ISR_MALFORMED,
             "Invalid field 'taker_gets.issuer', expected non-XRP issuer."};
 
     if (pay_currency == get_currency && pay_issuer == get_issuer)
-        return Status{Error::rpcINVALID_PARAMS, "badMarket"};
+        return Status{RippledError::rpcBAD_MARKET, "badMarket"};
 
     return ripple::Book{{pay_currency, pay_issuer}, {get_currency, get_issuer}};
 }
@@ -1457,12 +1459,12 @@ parseTaker(boost::json::value const& taker)
 {
     std::optional<ripple::AccountID> takerID = {};
     if (!taker.is_string())
-        return {Status{Error::rpcINVALID_PARAMS, "takerNotString"}};
+        return {Status{RippledError::rpcINVALID_PARAMS, "takerNotString"}};
 
     takerID = accountFromStringStrict(taker.as_string().c_str());
 
     if (!takerID)
-        return Status{Error::rpcINVALID_PARAMS, "invalidTakerAccount"};
+        return Status{RippledError::rpcBAD_ISSUER, "invalidTakerAccount"};
     return *takerID;
 }
 bool
@@ -1480,4 +1482,226 @@ specifiesCurrentOrClosedLedger(boost::json::object const& request)
     return false;
 }
 
+std::variant<ripple::uint256, Status>
+getNFTID(boost::json::object const& request)
+{
+    if (!request.contains(JS(nft_id)))
+        return Status{RippledError::rpcINVALID_PARAMS, "missingTokenID"};
+
+    if (!request.at(JS(nft_id)).is_string())
+        return Status{RippledError::rpcINVALID_PARAMS, "tokenIDNotString"};
+
+    ripple::uint256 tokenid;
+    if (!tokenid.parseHex(request.at(JS(nft_id)).as_string().c_str()))
+        return Status{RippledError::rpcINVALID_PARAMS, "malformedTokenID"};
+
+    return tokenid;
+}
+
+// TODO - this function is long and shouldn't be responsible for as much as it
+// is. Split it out into some helper functions.
+std::variant<Status, boost::json::object>
+traverseTransactions(
+    Context const& context,
+    std::function<Backend::TransactionsAndCursor(
+        std::shared_ptr<Backend::BackendInterface const> const& backend,
+        std::uint32_t const,
+        bool const,
+        std::optional<Backend::TransactionsCursor> const&,
+        boost::asio::yield_context& yield)> transactionFetcher)
+{
+    auto request = context.params;
+    boost::json::object response = {};
+
+    bool const binary = getBool(request, JS(binary), false);
+    bool const forward = getBool(request, JS(forward), false);
+
+    std::optional<Backend::TransactionsCursor> cursor;
+
+    if (request.contains(JS(marker)))
+    {
+        if (!request.at(JS(marker)).is_object())
+            return Status{RippledError::rpcINVALID_PARAMS, "invalidMarker"};
+        auto const& obj = request.at(JS(marker)).as_object();
+
+        std::optional<std::uint32_t> transactionIndex = {};
+        if (obj.contains(JS(seq)))
+        {
+            if (!obj.at(JS(seq)).is_int64())
+                return Status{
+                    RippledError::rpcINVALID_PARAMS, "transactionIndexNotInt"};
+
+            transactionIndex =
+                boost::json::value_to<std::uint32_t>(obj.at(JS(seq)));
+        }
+
+        std::optional<std::uint32_t> ledgerIndex = {};
+        if (obj.contains(JS(ledger)))
+        {
+            if (!obj.at(JS(ledger)).is_int64())
+                return Status{
+                    RippledError::rpcINVALID_PARAMS, "ledgerIndexNotInt"};
+
+            ledgerIndex =
+                boost::json::value_to<std::uint32_t>(obj.at(JS(ledger)));
+        }
+
+        if (!transactionIndex || !ledgerIndex)
+            return Status{
+                RippledError::rpcINVALID_PARAMS, "missingLedgerOrSeq"};
+
+        cursor = {*ledgerIndex, *transactionIndex};
+    }
+
+    auto minIndex = context.range.minSequence;
+    auto maxIndex = context.range.maxSequence;
+    std::optional<int64_t> min;
+    std::optional<int64_t> max;
+
+    if (request.contains(JS(ledger_index_min)))
+    {
+        if (!request.at(JS(ledger_index_min)).is_int64())
+        {
+            return Status{
+                RippledError::rpcINVALID_PARAMS, "ledgerSeqMinNotNumber"};
+        }
+
+        min = request.at(JS(ledger_index_min)).as_int64();
+
+        if (*min != -1)
+        {
+            if (context.range.maxSequence < *min ||
+                context.range.minSequence > *min)
+                return Status{
+                    RippledError::rpcLGR_IDX_MALFORMED,
+                    "ledgerSeqMinOutOfRange"};
+            else
+                minIndex = static_cast<uint32_t>(*min);
+        }
+
+        if (forward && !cursor)
+            cursor = {minIndex, 0};
+    }
+
+    if (request.contains(JS(ledger_index_max)))
+    {
+        if (!request.at(JS(ledger_index_max)).is_int64())
+        {
+            return Status{
+                RippledError::rpcINVALID_PARAMS, "ledgerSeqMaxNotNumber"};
+        }
+
+        max = request.at(JS(ledger_index_max)).as_int64();
+
+        if (*max != -1)
+        {
+            if (context.range.maxSequence < *max ||
+                context.range.minSequence > *max)
+                return Status{RippledError::rpcLGR_IDXS_INVALID};
+            else
+                maxIndex = static_cast<uint32_t>(*max);
+        }
+
+        if (minIndex > maxIndex)
+            return Status{RippledError::rpcINVALID_PARAMS, "invalidIndex"};
+
+        if (!forward && !cursor)
+            cursor = {maxIndex, INT32_MAX};
+    }
+
+    if (max && min && *max < *min)
+    {
+        return Status{RippledError::rpcLGR_IDXS_INVALID, "lgrIdxsInvalid"};
+    }
+
+    if (request.contains(JS(ledger_index)) || request.contains(JS(ledger_hash)))
+    {
+        if (request.contains(JS(ledger_index_max)) ||
+            request.contains(JS(ledger_index_min)))
+            return Status{
+                RippledError::rpcINVALID_PARAMS,
+                "containsLedgerSpecifierAndRange"};
+
+        auto v = ledgerInfoFromRequest(context);
+        if (auto status = std::get_if<Status>(&v); status)
+            return *status;
+
+        maxIndex = minIndex = std::get<ripple::LedgerInfo>(v).seq;
+    }
+
+    if (!cursor)
+    {
+        if (forward)
+            cursor = {minIndex, 0};
+        else
+            cursor = {maxIndex, INT32_MAX};
+    }
+
+    std::uint32_t limit;
+    if (auto const status = getLimit(context, limit); status)
+        return status;
+
+    if (request.contains(JS(limit)))
+        response[JS(limit)] = limit;
+
+    boost::json::array txns;
+    auto [blobs, retCursor] = transactionFetcher(
+        context.backend, limit, forward, cursor, context.yield);
+    auto timeDiff = util::timed([&, &retCursor = retCursor, &blobs = blobs]() {
+        if (retCursor)
+        {
+            boost::json::object cursorJson;
+            cursorJson[JS(ledger)] = retCursor->ledgerSequence;
+            cursorJson[JS(seq)] = retCursor->transactionIndex;
+            response[JS(marker)] = cursorJson;
+        }
+
+        for (auto const& txnPlusMeta : blobs)
+        {
+            if ((txnPlusMeta.ledgerSequence < minIndex && !forward) ||
+                (txnPlusMeta.ledgerSequence > maxIndex && forward))
+            {
+                response.erase(JS(marker));
+                break;
+            }
+            else if (txnPlusMeta.ledgerSequence > maxIndex && !forward)
+            {
+                gLog.debug()
+                    << "Skipping over transactions from incomplete ledger";
+                continue;
+            }
+
+            boost::json::object obj;
+
+            if (!binary)
+            {
+                auto [txn, meta] = toExpandedJson(txnPlusMeta);
+                obj[JS(meta)] = meta;
+                obj[JS(tx)] = txn;
+                obj[JS(tx)].as_object()[JS(ledger_index)] =
+                    txnPlusMeta.ledgerSequence;
+                obj[JS(tx)].as_object()[JS(date)] = txnPlusMeta.date;
+            }
+            else
+            {
+                obj[JS(meta)] = ripple::strHex(txnPlusMeta.metadata);
+                obj[JS(tx_blob)] = ripple::strHex(txnPlusMeta.transaction);
+                obj[JS(ledger_index)] = txnPlusMeta.ledgerSequence;
+                obj[JS(date)] = txnPlusMeta.date;
+            }
+            obj[JS(validated)] = true;
+            txns.push_back(obj);
+        }
+
+        response[JS(ledger_index_min)] = minIndex;
+        response[JS(ledger_index_max)] = maxIndex;
+        response[JS(transactions)] = txns;
+    });
+    gLog.info() << "serialization took " << timeDiff
+
+                << " milliseconds";
+
+    return response;
+}
+
 }  // namespace RPC

src/rpc/RPCHelpers.h

@@ -1,6 +1,24 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
 
-#ifndef XRPL_REPORTING_RPCHELPERS_H_INCLUDED
-#define XRPL_REPORTING_RPCHELPERS_H_INCLUDED
 /*
  * This file contains a variety of utility functions used when executing
  * the handlers
@@ -24,8 +42,6 @@
 namespace RPC {
 std::optional<ripple::AccountID>
 accountFromStringStrict(std::string const& account);
-std::optional<ripple::AccountID>
-accountFromSeed(std::string const& account);
 
 bool
 isOwnedByAccount(ripple::SLE const& sle, ripple::AccountID const& accountID);
@@ -38,7 +54,6 @@ parseAccountCursor(
     BackendInterface const& backend,
     std::uint32_t seq,
     std::optional<std::string> jsonCursor,
-    ripple::AccountID const& accountID,
     boost::asio::yield_context& yield);
 
 // TODO this function should probably be in a different file and namespace
@@ -99,7 +114,7 @@ traverseOwnedNodes(
     std::uint32_t limit,
     std::optional<std::string> jsonCursor,
     boost::asio::yield_context& yield,
-    std::function<void(ripple::SLE)> atOwnedNode);
+    std::function<void(ripple::SLE&&)> atOwnedNode);
 
 std::variant<Status, AccountCursor>
 traverseOwnedNodes(
@@ -111,7 +126,7 @@ traverseOwnedNodes(
     std::uint32_t limit,
     std::optional<std::string> jsonCursor,
     boost::asio::yield_context& yield,
-    std::function<void(ripple::SLE)> atOwnedNode);
+    std::function<void(ripple::SLE&&)> atOwnedNode);
 
 std::shared_ptr<ripple::SLE const>
 read(
@@ -254,5 +269,24 @@ getChannelId(boost::json::object const& request, ripple::uint256& channelId);
 bool
 specifiesCurrentOrClosedLedger(boost::json::object const& request);
 
+std::variant<ripple::uint256, Status>
+getNFTID(boost::json::object const& request);
+
+// This function is the driver for both `account_tx` and `nft_tx` and should
+// be used for any future transaction enumeration APIs.
+std::variant<Status, boost::json::object>
+traverseTransactions(
+    Context const& context,
+    std::function<Backend::TransactionsAndCursor(
+        std::shared_ptr<Backend::BackendInterface const> const& backend,
+        std::uint32_t const,
+        bool const,
+        std::optional<Backend::TransactionsCursor> const&,
+        boost::asio::yield_context& yield)> transactionFetcher);
+
+[[nodiscard]] boost::json::object const
+computeBookChanges(
+    ripple::LedgerInfo const& lgrInfo,
+    std::vector<Backend::TransactionAndMetadata> const& transactions);
+
 }  // namespace RPC
-#endif

src/rpc/WorkQueue.cpp

@@ -1,3 +1,22 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
 #include <rpc/WorkQueue.h>
 
 WorkQueue::WorkQueue(std::uint32_t numWorkers, uint32_t maxSize)

src/rpc/WorkQueue.h

@@ -1,10 +1,29 @@
-#ifndef CLIO_WORK_QUEUE_H
-#define CLIO_WORK_QUEUE_H
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include <log/Logger.h>
 
 #include <boost/asio.hpp>
 #include <boost/asio/spawn.hpp>
 #include <boost/json.hpp>
-#include <boost/log/trivial.hpp>
 
 #include <memory>
 #include <optional>
@@ -20,6 +39,7 @@ class WorkQueue
 
     std::atomic_uint64_t curSize_ = 0;
     uint32_t maxSize_ = std::numeric_limits<uint32_t>::max();
+    clio::Logger log_{"RPC"};
 
 public:
     WorkQueue(std::uint32_t numWorkers, uint32_t maxSize = 0);
@@ -30,10 +50,8 @@ public:
     {
         if (curSize_ >= maxSize_ && !isWhiteListed)
         {
-            BOOST_LOG_TRIVIAL(warning)
-                << __func__
-                << " queue is full. rejecting job. current size = " << curSize_
-                << " max size = " << maxSize_;
+            log_.warn() << "Queue is full. rejecting job. current size = "
+                        << curSize_ << " max size = " << maxSize_;
             return false;
         }
         ++curSize_;
@@ -52,17 +70,16 @@ public:
                 // durationUs_
                 ++queued_;
                 durationUs_ += wait;
-                BOOST_LOG_TRIVIAL(debug) << "WorkQueue wait time = " << wait
-                                         << " queue size = " << curSize_;
+                log_.info() << "WorkQueue wait time = " << wait
+                            << " queue size = " << curSize_;
                 f(yield);
                 --curSize_;
             });
         return true;
     }
 
-    // TODO: this is not actually being called. Wait for application refactor
     boost::json::object
-    report()
+    report() const
     {
         boost::json::object obj;
         obj["queued"] = queued_;
@@ -78,5 +95,3 @@ private:
     boost::asio::io_context ioc_ = {};
     std::optional<boost::asio::io_context::work> work_{ioc_};
 };
-
-#endif  // CLIO_WORK_QUEUE_H

src/rpc/common/AnyHandler.h

@@ -0,0 +1,103 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2023, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include <rpc/common/Concepts.h>
+#include <rpc/common/Types.h>
+#include <rpc/common/impl/Processors.h>
+
+namespace RPCng {
+
+/**
+ * @brief A type-erased Handler that can contain any (NextGen) RPC handler class
+ */
+class AnyHandler final
+{
+public:
+    template <
+        Handler HandlerType,
+        typename ProcessingStrategy = detail::DefaultProcessor<HandlerType>>
+    /* implicit */ AnyHandler(HandlerType&& handler)
+        : pimpl_{std::make_unique<Model<HandlerType, ProcessingStrategy>>(
+              std::forward<HandlerType>(handler))}
+    {
+    }
+
+    ~AnyHandler() = default;
+    AnyHandler(AnyHandler const& other) : pimpl_{other.pimpl_->clone()}
+    {
+    }
+    AnyHandler&
+    operator=(AnyHandler const& rhs)
+    {
+        AnyHandler copy{rhs};
+        pimpl_.swap(copy.pimpl_);
+        return *this;
+    }
+    AnyHandler(AnyHandler&&) = default;
+    AnyHandler&
+    operator=(AnyHandler&&) = default;
+
+    [[nodiscard]] ReturnType
+    process(boost::json::value const& value) const
+    {
+        return pimpl_->process(value);
+    }
+
+private:
+    struct Concept
+    {
+        virtual ~Concept() = default;
+
+        [[nodiscard]] virtual ReturnType
+        process(boost::json::value const& value) const = 0;
+
+        [[nodiscard]] virtual std::unique_ptr<Concept>
+        clone() const = 0;
+    };
+
+    template <typename HandlerType, typename ProcessorType>
+    struct Model : Concept
+    {
+        HandlerType handler;
+        ProcessorType processor;
+
+        Model(HandlerType&& handler) : handler{std::move(handler)}
+        {
+        }
+
+        [[nodiscard]] ReturnType
+        process(boost::json::value const& value) const override
+        {
+            return processor(handler, value);
+        }
+
+        [[nodiscard]] std::unique_ptr<Concept>
+        clone() const override
+        {
+            return std::make_unique<Model>(*this);
+        }
+    };
+
+private:
+    std::unique_ptr<Concept> pimpl_;
+};
+
+}  // namespace RPCng

src/rpc/common/Concepts.h

@@ -0,0 +1,60 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2023, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include <rpc/common/Types.h>
+
+#include <boost/json/value_from.hpp>
+#include <boost/json/value_to.hpp>
+
+#include <string>
+
+namespace RPCng {
+
+struct RpcSpec;
+
+/**
+ * @brief A concept that specifies what a requirement used with @ref FieldSpec
+ * must provide
+ */
+// clang-format off
+template <typename T>
+concept Requirement = requires(T a) {
+    { a.verify(boost::json::value{}, std::string{}) } -> std::same_as<MaybeError>;
+};
+// clang-format on
+
+/**
+ * @brief A concept that specifies what a Handler type must provide
+ *
+ * Note that value_from and value_to should be implemented using tag_invoke
+ * as per boost::json documentation for these functions.
+ */
+// clang-format off
+template <typename T>
+concept Handler = requires(T a, typename T::Input in, typename T::Output out) {
+    { a.spec() } -> std::same_as<RpcSpecConstRef>;
+    { a.process(in) } -> std::same_as<HandlerReturnType<decltype(out)>>;
+} 
+&& boost::json::has_value_from<typename T::Output>::value
+&& boost::json::has_value_to<typename T::Input>::value;
+// clang-format on
+
+}  // namespace RPCng

src/rpc/common/Specs.cpp

@@ -0,0 +1,42 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2023, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include <rpc/common/Specs.h>
+
+#include <boost/json/value.hpp>
+
+namespace RPCng {
+
+[[nodiscard]] MaybeError
+FieldSpec::validate(boost::json::value const& value) const
+{
+    return validator_(value);
+}
+
+[[nodiscard]] MaybeError
+RpcSpec::validate(boost::json::value const& value) const
+{
+    for (auto const& field : fields_)
+        if (auto ret = field.validate(value); not ret)
+            return Error{ret.error()};
+
+    return {};
+}
+
+}  // namespace RPCng

src/rpc/common/Specs.h

@@ -0,0 +1,70 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2023, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include <rpc/common/Concepts.h>
+#include <rpc/common/Types.h>
+#include <rpc/common/impl/Factories.h>
+
+#include <string>
+#include <vector>
+
+namespace RPCng {
+
+/**
+ * @brief Represents a Specification for one field of an RPC command
+ */
+struct FieldSpec final
+{
+    template <Requirement... Requirements>
+    FieldSpec(std::string const& key, Requirements&&... requirements)
+        : validator_{detail::makeFieldValidator<Requirements...>(
+              key,
+              std::forward<Requirements>(requirements)...)}
+    {
+    }
+
+    [[nodiscard]] MaybeError
+    validate(boost::json::value const& value) const;
+
+private:
+    std::function<MaybeError(boost::json::value const&)> validator_;
+};
+
+/**
+ * @brief Represents a Specification of an entire RPC command
+ *
+ * Note: this should really be all constexpr and handlers would expose
+ * static constexpr RpcSpec spec instead. Maybe some day in the future.
+ */
+struct RpcSpec final
+{
+    RpcSpec(std::initializer_list<FieldSpec> fields) : fields_{fields}
+    {
+    }
+
+    [[nodiscard]] MaybeError
+    validate(boost::json::value const& value) const;
+
+private:
+    std::vector<FieldSpec> fields_;
+};
+
+}  // namespace RPCng

src/rpc/common/Types.h

@@ -0,0 +1,56 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2023, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include <rpc/Errors.h>
+#include <util/Expected.h>
+
+#include <boost/json/value.hpp>
+
+namespace RPCng {
+
+/**
+ * @brief Return type used for Validators that can return error but don't have
+ * specific value to return
+ */
+using MaybeError = util::Expected<void, RPC::Status>;
+
+/**
+ * @brief The type that represents just the error part of @ref MaybeError
+ */
+using Error = util::Unexpected<RPC::Status>;
+
+/**
+ * @brief Return type for each individual handler
+ */
+template <typename OutputType>
+using HandlerReturnType = util::Expected<OutputType, RPC::Status>;
+
+/**
+ * @brief The final return type out of RPC engine
+ */
+using ReturnType = util::Expected<boost::json::value, RPC::Status>;
+
+struct RpcSpec;
+struct FieldSpec;
+
+using RpcSpecConstRef = RpcSpec const&;
+
+}  // namespace RPCng

src/rpc/common/Validators.cpp

@@ -0,0 +1,89 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2023, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include <rpc/common/Validators.h>
+
+#include <boost/json/value.hpp>
+
+#include <string_view>
+
+namespace RPCng::validation {
+
+[[nodiscard]] MaybeError
+Section::verify(boost::json::value const& value, std::string_view key) const
+{
+    if (not value.is_object() or not value.as_object().contains(key.data()))
+        return {};  // ignore. field does not exist, let 'required' fail
+                    // instead
+
+    auto const& res = value.at(key.data());
+    for (auto const& spec : specs)
+    {
+        if (auto const ret = spec.validate(res); not ret)
+            return Error{ret.error()};
+    }
+    return {};
+}
+
+[[nodiscard]] MaybeError
+Required::verify(boost::json::value const& value, std::string_view key) const
+{
+    if (not value.is_object() or not value.as_object().contains(key.data()))
+        return Error{RPC::Status{
+            RPC::RippledError::rpcINVALID_PARAMS,
+            "Required field '" + std::string{key} + "' missing"}};
+
+    return {};
+}
+
+[[nodiscard]] MaybeError
+ValidateArrayAt::verify(boost::json::value const& value, std::string_view key)
+    const
+{
+    if (not value.is_object() or not value.as_object().contains(key.data()))
+        return {};  // ignore. field does not exist, let 'required' fail
+                    // instead
+
+    if (not value.as_object().at(key.data()).is_array())
+        return Error{RPC::Status{RPC::RippledError::rpcINVALID_PARAMS}};
+
+    auto const& arr = value.as_object().at(key.data()).as_array();
+    if (idx_ >= arr.size())
+        return Error{RPC::Status{RPC::RippledError::rpcINVALID_PARAMS}};
+
+    auto const& res = arr.at(idx_);
+    for (auto const& spec : specs_)
+        if (auto const ret = spec.validate(res); not ret)
+            return Error{ret.error()};
+
+    return {};
+}
+
+[[nodiscard]] MaybeError
+CustomValidator::verify(boost::json::value const& value, std::string_view key)
+    const
+{
+    if (not value.is_object() or not value.as_object().contains(key.data()))
+        return {};  // ignore. field does not exist, let 'required' fail
+                    // instead
+
+    return validator_(value.as_object().at(key.data()), key);
+}
+
+}  // namespace RPCng::validation

src/rpc/common/Validators.h

@@ -0,0 +1,259 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2023, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include <rpc/common/Concepts.h>
+#include <rpc/common/Specs.h>
+#include <rpc/common/Types.h>
+
+namespace RPCng::validation {
+
+/**
+ * @brief Check that the type is the same as what was expected
+ *
+ * @tparam Expected The expected type that value should be convertible to
+ * @param value The json value to check the type of
+ * @return true if convertible; false otherwise
+ */
+template <typename Expected>
+[[nodiscard]] bool static checkType(boost::json::value const& value)
+{
+    auto hasError = false;
+    if constexpr (std::is_same_v<Expected, bool>)
+    {
+        if (not value.is_bool())
+            hasError = true;
+    }
+    else if constexpr (std::is_same_v<Expected, std::string>)
+    {
+        if (not value.is_string())
+            hasError = true;
+    }
+    else if constexpr (
+        std::is_same_v<Expected, double> or std::is_same_v<Expected, float>)
+    {
+        if (not value.is_double())
+            hasError = true;
+    }
+    else if constexpr (
+        std::is_convertible_v<Expected, uint64_t> or
+        std::is_convertible_v<Expected, int64_t>)
+    {
+        if (not value.is_int64() && not value.is_uint64())
+            hasError = true;
+    }
+    else if constexpr (std::is_same_v<Expected, boost::json::array>)
+    {
+        if (not value.is_array())
+            hasError = true;
+    }
+
+    return not hasError;
+}
+
+/**
+ * @brief A meta-validator that acts as a spec for a sub-object/section
+ */
+class Section final
+{
+    std::vector<FieldSpec> specs;
+
+public:
+    explicit Section(std::initializer_list<FieldSpec> specs) : specs{specs}
+    {
+    }
+
+    [[nodiscard]] MaybeError
+    verify(boost::json::value const& value, std::string_view key) const;
+};
+
+/**
+ * @brief A validator that simply requires a field to be present
+ */
+struct Required final
+{
+    [[nodiscard]] MaybeError
+    verify(boost::json::value const& value, std::string_view key) const;
+};
+
+/**
+ * @brief Validates that the type of the value is one of the given types
+ */
+template <typename... Types>
+struct Type final
+{
+    [[nodiscard]] MaybeError
+    verify(boost::json::value const& value, std::string_view key) const
+    {
+        if (not value.is_object() or not value.as_object().contains(key.data()))
+            return {};  // ignore. field does not exist, let 'required' fail
+                        // instead
+
+        auto const& res = value.as_object().at(key.data());
+        auto const convertible = (checkType<Types>(res) || ...);
+
+        if (not convertible)
+            return Error{RPC::Status{RPC::RippledError::rpcINVALID_PARAMS}};
+
+        return {};
+    }
+};
+
+/**
+ * @brief Validate that value is between specified min and max
+ */
+template <typename Type>
+class Between final
+{
+    Type min_;
+    Type max_;
+
+public:
+    explicit Between(Type min, Type max) : min_{min}, max_{max}
+    {
+    }
+
+    [[nodiscard]] MaybeError
+    verify(boost::json::value const& value, std::string_view key) const
+    {
+        if (not value.is_object() or not value.as_object().contains(key.data()))
+            return {};  // ignore. field does not exist, let 'required' fail
+                        // instead
+
+        using boost::json::value_to;
+        auto const res = value_to<Type>(value.as_object().at(key.data()));
+        // todo: may want a way to make this code more generic (e.g. use a free
+        // function that can be overridden for this comparison)
+        if (res < min_ || res > max_)
+            return Error{RPC::Status{RPC::RippledError::rpcINVALID_PARAMS}};
+
+        return {};
+    }
+};
+
+/**
+ * @brief Validates that the value is equal to the one passed in
+ */
+template <typename Type>
+class EqualTo final
+{
+    Type original_;
+
+public:
+    explicit EqualTo(Type original) : original_{original}
+    {
+    }
+
+    [[nodiscard]] MaybeError
+    verify(boost::json::value const& value, std::string_view key) const
+    {
+        if (not value.is_object() or not value.as_object().contains(key.data()))
+            return {};  // ignore. field does not exist, let 'required' fail
+                        // instead
+
+        using boost::json::value_to;
+        auto const res = value_to<Type>(value.as_object().at(key.data()));
+        if (res != original_)
+            return Error{RPC::Status{RPC::RippledError::rpcINVALID_PARAMS}};
+
+        return {};
+    }
+};
+
+/**
+ * @brief Deduction guide to help disambiguate what it means to EqualTo a
+ * "string" without specifying the type.
+ */
+EqualTo(char const*)->EqualTo<std::string>;
+
+/**
+ * @brief Validates that the value is one of the values passed in
+ */
+template <typename Type>
+class OneOf final
+{
+    std::vector<Type> options_;
+
+public:
+    explicit OneOf(std::initializer_list<Type> options) : options_{options}
+    {
+    }
+
+    [[nodiscard]] MaybeError
+    verify(boost::json::value const& value, std::string_view key) const
+    {
+        if (not value.is_object() or not value.as_object().contains(key.data()))
+            return {};  // ignore. field does not exist, let 'required' fail
+                        // instead
+
+        using boost::json::value_to;
+        auto const res = value_to<Type>(value.as_object().at(key.data()));
+        if (std::find(std::begin(options_), std::end(options_), res) ==
+            std::end(options_))
+            return Error{RPC::Status{RPC::RippledError::rpcINVALID_PARAMS}};
+
+        return {};
+    }
+};
+
+/**
+ * @brief Deduction guide to help disambiguate what it means to OneOf a
+ * few "strings" without specifying the type.
+ */
+OneOf(std::initializer_list<char const*>)->OneOf<std::string>;
+
+/**
+ * @brief A meta-validator that specifies a list of specs to run against the
+ * object at the given index in the array
+ */
+class ValidateArrayAt final
+{
+    std::size_t idx_;
+    std::vector<FieldSpec> specs_;
+
+public:
+    ValidateArrayAt(std::size_t idx, std::initializer_list<FieldSpec> specs)
+        : idx_{idx}, specs_{specs}
+    {
+    }
+
+    [[nodiscard]] MaybeError
+    verify(boost::json::value const& value, std::string_view key) const;
+};
+
+/**
+ * @brief A meta-validator that allows to specify a custom validation function
+ */
+class CustomValidator final
+{
+    std::function<MaybeError(boost::json::value const&, std::string_view)>
+        validator_;
+
+public:
+    template <typename Fn>
+    explicit CustomValidator(Fn&& fn) : validator_{std::forward<Fn>(fn)}
+    {
+    }
+
+    [[nodiscard]] MaybeError
+    verify(boost::json::value const& value, std::string_view key) const;
+};
+
+}  // namespace RPCng::validation

src/rpc/common/impl/Factories.h

@@ -0,0 +1,59 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2023, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include <rpc/common/Concepts.h>
+#include <rpc/common/Types.h>
+
+#include <boost/json/value.hpp>
+
+#include <optional>
+
+namespace RPCng::detail {
+
+template <Requirement... Requirements>
+[[nodiscard]] auto
+makeFieldValidator(std::string const& key, Requirements&&... requirements)
+{
+    return [key, ... r = std::forward<Requirements>(requirements)](
+               boost::json::value const& j) -> MaybeError {
+        // clang-format off
+        std::optional<RPC::Status> firstFailure = std::nullopt;
+
+        // This expands in order of Requirements and stops evaluating after 
+        // first failure which is stored in `firstFailure` and can be checked 
+        // later on to see whether the verification failed as a whole or not.
+        ([&j, &key, &firstFailure, req = &r]() {
+            if (firstFailure)
+                return; // already failed earlier - skip
+
+            if (auto const res = req->verify(j, key); not res)
+                firstFailure = res.error();
+        }(), ...);
+        // clang-format on
+
+        if (firstFailure)
+            return Error{firstFailure.value()};
+
+        return {};
+    };
+}
+
+}  // namespace RPCng::detail

src/rpc/common/impl/Processors.h

@@ -0,0 +1,52 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2023, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include <rpc/common/Concepts.h>
+#include <rpc/common/Types.h>
+
+namespace RPCng::detail {
+
+template <Handler HandlerType>
+struct DefaultProcessor final
+{
+    [[nodiscard]] ReturnType
+    operator()(HandlerType const& handler, boost::json::value const& value)
+        const
+    {
+        using boost::json::value_from;
+        using boost::json::value_to;
+
+        // first we run validation
+        auto const spec = handler.spec();
+        if (auto const ret = spec.validate(value); not ret)
+            return Error{ret.error()};  // forward Status
+
+        auto const inData = value_to<typename HandlerType::Input>(value);
+
+        // real handler is given expected Input, not json
+        if (auto const ret = handler.process(inData); not ret)
+            return Error{ret.error()};  // forward Status
+        else
+            return value_from(ret.value());
+    }
+};
+
+}  // namespace RPCng::detail

src/rpc/handlers/AccountChannels.cpp

@@ -1,3 +1,22 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
 #include <ripple/app/ledger/Ledger.h>
 #include <ripple/basics/StringUtilities.h>
 #include <ripple/protocol/ErrorCodes.h>
@@ -8,7 +27,7 @@
 #include <algorithm>
 #include <backend/BackendInterface.h>
 #include <backend/DBHelpers.h>
-#include <backend/Pg.h>
+
 #include <rpc/RPCHelpers.h>
 
 namespace RPC {
@@ -62,7 +81,7 @@ doAccountChannels(Context const& context)
         ripple::keylet::account(accountID).key, lgrInfo.seq, context.yield);
 
     if (!rawAcct)
-        return Status{Error::rpcACT_NOT_FOUND, "accountNotFound"};
+        return Status{RippledError::rpcACT_NOT_FOUND, "accountNotFound"};
 
     ripple::AccountID destAccount;
     if (auto const status =
@@ -78,16 +97,17 @@ doAccountChannels(Context const& context)
     if (request.contains(JS(marker)))
     {
         if (!request.at(JS(marker)).is_string())
-            return Status{Error::rpcINVALID_PARAMS, "markerNotString"};
+            return Status{RippledError::rpcINVALID_PARAMS, "markerNotString"};
 
         marker = request.at(JS(marker)).as_string().c_str();
     }
 
     response[JS(account)] = ripple::to_string(accountID);
     response[JS(channels)] = boost::json::value(boost::json::array_kind);
+    response[JS(limit)] = limit;
     boost::json::array& jsonChannels = response.at(JS(channels)).as_array();
 
-    auto const addToResponse = [&](ripple::SLE const& sle) {
+    auto const addToResponse = [&](ripple::SLE&& sle) {
         if (sle.getType() == ripple::ltPAYCHAN &&
             sle.getAccountID(ripple::sfAccount) == accountID &&
             (!destAccount ||

src/rpc/handlers/AccountCurrencies.cpp

@@ -1,3 +1,22 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
 #include <ripple/app/ledger/Ledger.h>
 #include <ripple/basics/StringUtilities.h>
 #include <ripple/protocol/ErrorCodes.h>
@@ -32,10 +51,10 @@ doAccountCurrencies(Context const& context)
         ripple::keylet::account(accountID).key, lgrInfo.seq, context.yield);
 
     if (!rawAcct)
-        return Status{Error::rpcACT_NOT_FOUND, "accountNotFound"};
+        return Status{RippledError::rpcACT_NOT_FOUND, "accountNotFound"};
 
     std::set<std::string> send, receive;
-    auto const addToResponse = [&](ripple::SLE const& sle) {
+    auto const addToResponse = [&](ripple::SLE&& sle) {
         if (sle.getType() == ripple::ltRIPPLE_STATE)
         {
             ripple::STAmount balance = sle.getFieldAmount(ripple::sfBalance);

src/rpc/handlers/AccountInfo.cpp

@@ -1,3 +1,22 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
 #include <ripple/protocol/Indexes.h>
 #include <ripple/protocol/STLedgerEntry.h>
 #include <boost/json.hpp>
@@ -34,7 +53,7 @@ doAccountInfo(Context const& context)
     else if (request.contains(JS(ident)))
         strIdent = request.at(JS(ident)).as_string().c_str();
     else
-        return Status{Error::rpcACT_MALFORMED};
+        return Status{RippledError::rpcACT_MALFORMED};
 
     // We only need to fetch the ledger header because the ledger hash is
     // supposed to be included in the response. The ledger sequence is specified
@@ -48,31 +67,20 @@ doAccountInfo(Context const& context)
     // Get info on account.
     auto accountID = accountFromStringStrict(strIdent);
     if (!accountID)
-        return Status{Error::rpcACT_MALFORMED};
-
-    assert(accountID.has_value());
+        return Status{RippledError::rpcACT_MALFORMED};
 
     auto key = ripple::keylet::account(accountID.value());
-
     std::optional<std::vector<unsigned char>> dbResponse =
         context.backend->fetchLedgerObject(key.key, lgrInfo.seq, context.yield);
 
     if (!dbResponse)
-    {
-        return Status{Error::rpcACT_NOT_FOUND};
-    }
+        return Status{RippledError::rpcACT_NOT_FOUND};
 
     ripple::STLedgerEntry sle{
         ripple::SerialIter{dbResponse->data(), dbResponse->size()}, key.key};
 
     if (!key.check(sle))
-        return Status{Error::rpcDB_DESERIALIZATION};
-
-    // if (!binary)
-    //     response[JS(account_data)] = getJson(sle);
-    // else
-    //     response[JS(account_data)] = ripple::strHex(*dbResponse);
-    // response[JS(db_time)] = time;
+        return Status{RippledError::rpcDB_DESERIALIZATION};
 
     response[JS(account_data)] = toJson(sle);
     response[JS(ledger_hash)] = ripple::strHex(lgrInfo.hash);
@@ -97,7 +105,7 @@ doAccountInfo(Context const& context)
                 ripple::SerialIter{signers->data(), signers->size()},
                 signersKey.key};
             if (!signersKey.check(sleSigners))
-                return Status{Error::rpcDB_DESERIALIZATION};
+                return Status{RippledError::rpcDB_DESERIALIZATION};
 
             signerList.push_back(toJson(sleSigners));
         }

src/rpc/handlers/AccountLines.cpp

@@ -1,3 +1,22 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
 #include <ripple/app/ledger/Ledger.h>
 #include <ripple/app/paths/TrustLine.h>
 #include <ripple/basics/StringUtilities.h>
@@ -107,7 +126,7 @@ doAccountLines(Context const& context)
         ripple::keylet::account(accountID).key, lgrInfo.seq, context.yield);
 
     if (!rawAcct)
-        return Status{Error::rpcACT_NOT_FOUND, "accountNotFound"};
+        return Status{RippledError::rpcACT_NOT_FOUND, "accountNotFound"};
 
     std::optional<ripple::AccountID> peerAccount;
     if (auto const status = getOptionalAccount(request, peerAccount, JS(peer));
@@ -121,22 +140,48 @@ doAccountLines(Context const& context)
     std::optional<std::string> marker = {};
     if (request.contains(JS(marker)))
     {
-        if (!request.at(JS(marker)).is_string())
-            return Status{Error::rpcINVALID_PARAMS, "markerNotString"};
+        if (not request.at(JS(marker)).is_string())
+            return Status{RippledError::rpcINVALID_PARAMS, "markerNotString"};
 
         marker = request.at(JS(marker)).as_string().c_str();
     }
 
+    auto ignoreDefault = false;
+    if (request.contains(JS(ignore_default)))
+    {
+        if (not request.at(JS(ignore_default)).is_bool())
+            return Status{
+                RippledError::rpcINVALID_PARAMS, "ignoreDefaultNotBool"};
+
+        ignoreDefault = request.at(JS(ignore_default)).as_bool();
+    }
+
     response[JS(account)] = ripple::to_string(accountID);
     response[JS(ledger_hash)] = ripple::strHex(lgrInfo.hash);
     response[JS(ledger_index)] = lgrInfo.seq;
+    response[JS(limit)] = limit;
     response[JS(lines)] = boost::json::value(boost::json::array_kind);
     boost::json::array& jsonLines = response.at(JS(lines)).as_array();
 
-    auto const addToResponse = [&](ripple::SLE const& sle) -> void {
+    auto const addToResponse = [&](ripple::SLE&& sle) -> void {
         if (sle.getType() == ripple::ltRIPPLE_STATE)
         {
-            addLine(jsonLines, sle, accountID, peerAccount);
+            auto ignore = false;
+            if (ignoreDefault)
+            {
+                if (sle.getFieldAmount(ripple::sfLowLimit).getIssuer() ==
+                    accountID)
+                    ignore =
+                        !(sle.getFieldU32(ripple::sfFlags) &
+                          ripple::lsfLowReserve);
+                else
+                    ignore =
+                        !(sle.getFieldU32(ripple::sfFlags) &
+                          ripple::lsfHighReserve);
+            }
+
+            if (!ignore)
+                addLine(jsonLines, sle, accountID, peerAccount);
         }
     };
 

src/rpc/handlers/AccountObjects.cpp

@@ -1,3 +1,22 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
 #include <ripple/app/ledger/Ledger.h>
 #include <ripple/app/paths/TrustLine.h>
 #include <ripple/app/tx/impl/details/NFTokenUtils.h>
@@ -45,13 +64,13 @@ doAccountNFTs(Context const& context)
         return status;
 
     if (!accountID)
-        return Status{Error::rpcINVALID_PARAMS, "malformedAccount"};
+        return Status{RippledError::rpcINVALID_PARAMS, "malformedAccount"};
 
     auto rawAcct = context.backend->fetchLedgerObject(
         ripple::keylet::account(accountID).key, lgrInfo.seq, context.yield);
 
     if (!rawAcct)
-        return Status{Error::rpcACT_NOT_FOUND, "accountNotFound"};
+        return Status{RippledError::rpcACT_NOT_FOUND, "accountNotFound"};
 
     std::uint32_t limit;
     if (auto const status = getLimit(context, limit); status)
@@ -63,6 +82,7 @@ doAccountNFTs(Context const& context)
 
     response[JS(account)] = ripple::toBase58(accountID);
     response[JS(validated)] = true;
+    response[JS(limit)] = limit;
 
     std::uint32_t numPages = 0;
     response[JS(account_nfts)] = boost::json::value(boost::json::array_kind);
@@ -156,7 +176,7 @@ doAccountObjects(Context const& context)
     if (request.contains("marker"))
     {
         if (!request.at("marker").is_string())
-            return Status{Error::rpcINVALID_PARAMS, "markerNotString"};
+            return Status{RippledError::rpcINVALID_PARAMS, "markerNotString"};
 
         marker = request.at("marker").as_string().c_str();
     }
@@ -165,11 +185,11 @@ doAccountObjects(Context const& context)
     if (request.contains(JS(type)))
     {
         if (!request.at(JS(type)).is_string())
-            return Status{Error::rpcINVALID_PARAMS, "typeNotString"};
+            return Status{RippledError::rpcINVALID_PARAMS, "typeNotString"};
 
         std::string typeAsString = request.at(JS(type)).as_string().c_str();
         if (types.find(typeAsString) == types.end())
-            return Status{Error::rpcINVALID_PARAMS, "typeInvalid"};
+            return Status{RippledError::rpcINVALID_PARAMS, "typeInvalid"};
 
         objectType = types[typeAsString];
     }
@@ -179,7 +199,7 @@ doAccountObjects(Context const& context)
     boost::json::array& jsonObjects =
         response.at(JS(account_objects)).as_array();
 
-    auto const addToResponse = [&](ripple::SLE const& sle) {
+    auto const addToResponse = [&](ripple::SLE&& sle) {
         if (!objectType || objectType == sle.getType())
         {
             jsonObjects.push_back(toJson(sle));
@@ -201,8 +221,7 @@ doAccountObjects(Context const& context)
     if (auto status = std::get_if<RPC::Status>(&next))
         return *status;
 
-    auto nextMarker = std::get<RPC::AccountCursor>(next);
-
+    auto const& nextMarker = std::get<RPC::AccountCursor>(next);
     if (nextMarker.isNonZero())
         response[JS(marker)] = nextMarker.toString();
 

src/rpc/handlers/AccountOffers.cpp

@@ -1,3 +1,22 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
 #include <ripple/app/ledger/Ledger.h>
 #include <ripple/app/paths/TrustLine.h>
 #include <ripple/basics/StringUtilities.h>
@@ -84,7 +103,7 @@ doAccountOffers(Context const& context)
         ripple::keylet::account(accountID).key, lgrInfo.seq, context.yield);
 
     if (!rawAcct)
-        return Status{Error::rpcACT_NOT_FOUND, "accountNotFound"};
+        return Status{RippledError::rpcACT_NOT_FOUND, "accountNotFound"};
 
     std::uint32_t limit;
     if (auto const status = getLimit(context, limit); status)
@@ -94,18 +113,19 @@ doAccountOffers(Context const& context)
     if (request.contains(JS(marker)))
     {
         if (!request.at(JS(marker)).is_string())
-            return Status{Error::rpcINVALID_PARAMS, "markerNotString"};
+            return Status{RippledError::rpcINVALID_PARAMS, "markerNotString"};
 
         marker = request.at(JS(marker)).as_string().c_str();
     }
 
     response[JS(account)] = ripple::to_string(accountID);
+    response[JS(limit)] = limit;
     response[JS(ledger_hash)] = ripple::strHex(lgrInfo.hash);
     response[JS(ledger_index)] = lgrInfo.seq;
     response[JS(offers)] = boost::json::value(boost::json::array_kind);
     boost::json::array& jsonLines = response.at(JS(offers)).as_array();
 
-    auto const addToResponse = [&](ripple::SLE const& sle) {
+    auto const addToResponse = [&](ripple::SLE&& sle) {
         if (sle.getType() == ripple::ltOFFER)
         {
             addOffer(jsonLines, sle);

src/rpc/handlers/AccountTx.cpp

@@ -1,212 +1,67 @@
-#include <backend/BackendInterface.h>
-#include <backend/Pg.h>
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include <log/Logger.h>
 #include <rpc/RPCHelpers.h>
+#include <util/Profiler.h>
+
+using namespace clio;
+
+// local to compilation unit loggers
+namespace {
+clio::Logger gLog{"RPC"};
+}  // namespace
 
 namespace RPC {
 
-using boost::json::value_to;
-
 Result
 doAccountTx(Context const& context)
 {
-    auto request = context.params;
-    boost::json::object response = {};
-
     ripple::AccountID accountID;
-    if (auto const status = getAccount(request, accountID); status)
+    if (auto const status = getAccount(context.params, accountID); status)
         return status;
 
-    bool const binary = getBool(request, JS(binary), false);
-    bool const forward = getBool(request, JS(forward), false);
+    constexpr std::string_view outerFuncName = __func__;
+    auto const maybeResponse = traverseTransactions(
+        context,
+        [&accountID, &outerFuncName](
+            std::shared_ptr<Backend::BackendInterface const> const& backend,
+            std::uint32_t const limit,
+            bool const forward,
+            std::optional<Backend::TransactionsCursor> const& cursorIn,
+            boost::asio::yield_context& yield) {
+            auto [txnsAndCursor, timeDiff] = util::timed([&]() {
+                return backend->fetchAccountTransactions(
+                    accountID, limit, forward, cursorIn, yield);
+            });
+            gLog.info() << outerFuncName << " db fetch took " << timeDiff
+                        << " milliseconds - num blobs = "
+                        << txnsAndCursor.txns.size();
+            return txnsAndCursor;
+        });
 
-    std::optional<Backend::TransactionsCursor> cursor;
-
-    if (request.contains(JS(marker)))
-    {
-        auto const& obj = request.at(JS(marker)).as_object();
-
-        std::optional<std::uint32_t> transactionIndex = {};
-        if (obj.contains(JS(seq)))
-        {
-            if (!obj.at(JS(seq)).is_int64())
-                return Status{
-                    Error::rpcINVALID_PARAMS, "transactionIndexNotInt"};
-
-            transactionIndex =
-                boost::json::value_to<std::uint32_t>(obj.at(JS(seq)));
-        }
-
-        std::optional<std::uint32_t> ledgerIndex = {};
-        if (obj.contains(JS(ledger)))
-        {
-            if (!obj.at(JS(ledger)).is_int64())
-                return Status{Error::rpcINVALID_PARAMS, "ledgerIndexNotInt"};
-
-            ledgerIndex =
-                boost::json::value_to<std::uint32_t>(obj.at(JS(ledger)));
-        }
-
-        if (!transactionIndex || !ledgerIndex)
-            return Status{Error::rpcINVALID_PARAMS, "missingLedgerOrSeq"};
-
-        cursor = {*ledgerIndex, *transactionIndex};
-    }
-
-    auto minIndex = context.range.minSequence;
-    if (request.contains(JS(ledger_index_min)))
-    {
-        auto& min = request.at(JS(ledger_index_min));
-
-        if (!min.is_int64())
-            return Status{Error::rpcINVALID_PARAMS, "ledgerSeqMinNotNumber"};
-
-        if (min.as_int64() != -1)
-        {
-            if (context.range.maxSequence < min.as_int64() ||
-                context.range.minSequence > min.as_int64())
-                return Status{
-                    Error::rpcINVALID_PARAMS, "ledgerSeqMinOutOfRange"};
-            else
-                minIndex = value_to<std::uint32_t>(min);
-        }
-
-        if (forward && !cursor)
-            cursor = {minIndex, 0};
-    }
-
-    auto maxIndex = context.range.maxSequence;
-    if (request.contains(JS(ledger_index_max)))
-    {
-        auto& max = request.at(JS(ledger_index_max));
-
-        if (!max.is_int64())
-            return Status{Error::rpcINVALID_PARAMS, "ledgerSeqMaxNotNumber"};
-
-        if (max.as_int64() != -1)
-        {
-            if (context.range.maxSequence < max.as_int64() ||
-                context.range.minSequence > max.as_int64())
-                return Status{
-                    Error::rpcINVALID_PARAMS, "ledgerSeqMaxOutOfRange"};
-            else
-                maxIndex = value_to<std::uint32_t>(max);
-        }
-
-        if (minIndex > maxIndex)
-            return Status{Error::rpcINVALID_PARAMS, "invalidIndex"};
-
-        if (!forward && !cursor)
-            cursor = {maxIndex, INT32_MAX};
-    }
-
-    if (request.contains(JS(ledger_index)) || request.contains(JS(ledger_hash)))
-    {
-        if (request.contains(JS(ledger_index_max)) ||
-            request.contains(JS(ledger_index_min)))
-            return Status{
-                Error::rpcINVALID_PARAMS, "containsLedgerSpecifierAndRange"};
-
-        auto v = ledgerInfoFromRequest(context);
-        if (auto status = std::get_if<Status>(&v))
-            return *status;
-
-        maxIndex = minIndex = std::get<ripple::LedgerInfo>(v).seq;
-    }
-
-    if (!cursor)
-    {
-        if (forward)
-            cursor = {minIndex, 0};
-        else
-            cursor = {maxIndex, INT32_MAX};
-    }
-
-    std::uint32_t limit;
-    if (auto const status = getLimit(context, limit); status)
-        return status;
-
-    if (request.contains(JS(limit)))
-        response[JS(limit)] = limit;
-
-    boost::json::array txns;
-    auto start = std::chrono::system_clock::now();
-    auto [blobs, retCursor] = context.backend->fetchAccountTransactions(
-        accountID, limit, forward, cursor, context.yield);
-
-    auto end = std::chrono::system_clock::now();
-    BOOST_LOG_TRIVIAL(info) << __func__ << " db fetch took "
-                            << ((end - start).count() / 1000000000.0)
-                            << " num blobs = " << blobs.size();
+    if (auto const status = std::get_if<Status>(&maybeResponse); status)
+        return *status;
+    auto response = std::get<boost::json::object>(maybeResponse);
 
     response[JS(account)] = ripple::to_string(accountID);
-
-    if (retCursor)
-    {
-        boost::json::object cursorJson;
-        cursorJson[JS(ledger)] = retCursor->ledgerSequence;
-        cursorJson[JS(seq)] = retCursor->transactionIndex;
-        response[JS(marker)] = cursorJson;
-    }
-
-    std::optional<size_t> maxReturnedIndex;
-    std::optional<size_t> minReturnedIndex;
-    for (auto const& txnPlusMeta : blobs)
-    {
-        if (txnPlusMeta.ledgerSequence < minIndex ||
-            txnPlusMeta.ledgerSequence > maxIndex)
-        {
-            BOOST_LOG_TRIVIAL(debug)
-                << __func__
-                << " skipping over transactions from incomplete ledger";
-            continue;
-        }
-
-        boost::json::object obj;
-
-        if (!binary)
-        {
-            auto [txn, meta] = toExpandedJson(txnPlusMeta);
-            obj[JS(meta)] = meta;
-            obj[JS(tx)] = txn;
-            obj[JS(tx)].as_object()[JS(ledger_index)] =
-                txnPlusMeta.ledgerSequence;
-            obj[JS(tx)].as_object()[JS(date)] = txnPlusMeta.date;
-        }
-        else
-        {
-            obj[JS(meta)] = ripple::strHex(txnPlusMeta.metadata);
-            obj[JS(tx_blob)] = ripple::strHex(txnPlusMeta.transaction);
-            obj[JS(ledger_index)] = txnPlusMeta.ledgerSequence;
-            obj[JS(date)] = txnPlusMeta.date;
-        }
-        obj[JS(validated)] = true;
-
-        txns.push_back(obj);
-        if (!minReturnedIndex || txnPlusMeta.ledgerSequence < *minReturnedIndex)
-            minReturnedIndex = txnPlusMeta.ledgerSequence;
-        if (!maxReturnedIndex || txnPlusMeta.ledgerSequence > *maxReturnedIndex)
-            maxReturnedIndex = txnPlusMeta.ledgerSequence;
-    }
-
-    assert(cursor);
-    if (!forward)
-    {
-        response[JS(ledger_index_min)] = cursor->ledgerSequence;
-        response[JS(ledger_index_max)] = maxIndex;
-    }
-    else
-    {
-        response[JS(ledger_index_max)] = cursor->ledgerSequence;
-        response[JS(ledger_index_min)] = minIndex;
-    }
-
-    response[JS(transactions)] = txns;
-
-    auto end2 = std::chrono::system_clock::now();
-    BOOST_LOG_TRIVIAL(info) << __func__ << " serialization took "
-                            << ((end2 - end).count() / 1000000000.0);
-
     return response;
-}  // namespace RPC
+}
 
 }  // namespace RPC

src/rpc/handlers/BookChanges.cpp

@@ -1,3 +1,22 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
 #include <ripple/app/ledger/Ledger.h>
 #include <ripple/basics/ToString.h>
 
@@ -12,6 +31,9 @@ using namespace ripple;
 
 namespace RPC {
 
+/**
+ * @brief Represents an entry in the book_changes' changes array.
+ */
 struct BookChange
 {
     STAmount sideAVolume;
@@ -22,190 +44,184 @@ struct BookChange
     STAmount closeRate;
 };
 
-class BookChangesHandler
+/**
+ * @brief Encapsulates the book_changes computations and transformations.
+ */
+class BookChanges final
 {
-    std::reference_wrapper<Context const> context_;
-    std::map<std::string, BookChange> tally_ = {};
-    std::optional<uint32_t> offerCancel_ = {};
-
 public:
-    ~BookChangesHandler() = default;
-    explicit BookChangesHandler(Context const& context)
-        : context_{std::cref(context)}
-    {
-    }
-
-    BookChangesHandler(BookChangesHandler const&) = delete;
-    BookChangesHandler(BookChangesHandler&&) = delete;
-    BookChangesHandler&
-    operator=(BookChangesHandler const&) = delete;
-    BookChangesHandler&
-    operator=(BookChangesHandler&&) = delete;
+    BookChanges() = delete;  // only accessed via static handle function
 
     /**
-     * @brief Handles the `book_change` request for given transactions
+     * @brief Computes all book_changes for the given transactions.
      *
-     * @param transactions The transactions to compute changes for
-     * @return std::vector<BookChange> The changes
+     * @param transactions The transactions to compute book changes for
+     * @return std::vector<BookChange> Book changes
      */
-    std::vector<BookChange>
-    handle(LedgerInfo const& ledger)
+    [[nodiscard]] static std::vector<BookChange>
+    compute(std::vector<Backend::TransactionAndMetadata> const& transactions)
     {
-        reset();
-
-        for (auto const transactions =
-                 context_.get().backend->fetchAllTransactionsInLedger(
-                     ledger.seq, context_.get().yield);
-             auto const& tx : transactions)
-        {
-            handleBookChange(tx);
-        }
-
-        // TODO: rewrite this with std::ranges when compilers catch up
-        std::vector<BookChange> changes;
-        std::transform(
-            std::make_move_iterator(std::begin(tally_)),
-            std::make_move_iterator(std::end(tally_)),
-            std::back_inserter(changes),
-            [](auto obj) { return obj.second; });
-        return changes;
+        return HandlerImpl{}(transactions);
     }
 
 private:
-    inline void
-    reset() noexcept
+    class HandlerImpl final
     {
-        tally_.clear();
-        offerCancel_ = std::nullopt;
-    }
+        std::map<std::string, BookChange> tally_ = {};
+        std::optional<uint32_t> offerCancel_ = {};
 
-    void
-    handleAffectedNode(STObject const& node)
-    {
-        auto const& metaType = node.getFName();
-        auto const nodeType = node.getFieldU16(sfLedgerEntryType);
-
-        // we only care about ltOFFER objects being modified or
-        // deleted
-        if (nodeType != ltOFFER || metaType == sfCreatedNode)
-            return;
-
-        // if either FF or PF are missing we can't compute
-        // but generally these are cancelled rather than crossed
-        // so skipping them is consistent
-        if (!node.isFieldPresent(sfFinalFields) ||
-            !node.isFieldPresent(sfPreviousFields))
-            return;
-
-        auto const& finalFields =
-            node.peekAtField(sfFinalFields).downcast<STObject>();
-        auto const& previousFields =
-            node.peekAtField(sfPreviousFields).downcast<STObject>();
-
-        // defensive case that should never be hit
-        if (!finalFields.isFieldPresent(sfTakerGets) ||
-            !finalFields.isFieldPresent(sfTakerPays) ||
-            !previousFields.isFieldPresent(sfTakerGets) ||
-            !previousFields.isFieldPresent(sfTakerPays))
-            return;
-
-        // filter out any offers deleted by explicit offer cancels
-        if (metaType == sfDeletedNode && offerCancel_ &&
-            finalFields.getFieldU32(sfSequence) == *offerCancel_)
-            return;
-
-        // compute the difference in gets and pays actually
-        // affected onto the offer
-        auto const deltaGets = finalFields.getFieldAmount(sfTakerGets) -
-            previousFields.getFieldAmount(sfTakerGets);
-        auto const deltaPays = finalFields.getFieldAmount(sfTakerPays) -
-            previousFields.getFieldAmount(sfTakerPays);
-
-        auto const g = to_string(deltaGets.issue());
-        auto const p = to_string(deltaPays.issue());
-
-        auto const noswap =
-            isXRP(deltaGets) ? true : (isXRP(deltaPays) ? false : (g < p));
-
-        auto first = noswap ? deltaGets : deltaPays;
-        auto second = noswap ? deltaPays : deltaGets;
-
-        // defensively programmed, should (probably) never happen
-        if (second == beast::zero)
-            return;
-
-        auto const rate = divide(first, second, noIssue());
-
-        if (first < beast::zero)
-            first = -first;
-
-        if (second < beast::zero)
-            second = -second;
-
-        auto const key = noswap ? (g + '|' + p) : (p + '|' + g);
-        if (tally_.contains(key))
+    public:
+        [[nodiscard]] std::vector<BookChange>
+        operator()(
+            std::vector<Backend::TransactionAndMetadata> const& transactions)
         {
-            auto& entry = tally_.at(key);
+            for (auto const& tx : transactions)
+                handleBookChange(tx);
 
-            entry.sideAVolume += first;
-            entry.sideBVolume += second;
-
-            if (entry.highRate < rate)
-                entry.highRate = rate;
-
-            if (entry.lowRate > rate)
-                entry.lowRate = rate;
-
-            entry.closeRate = rate;
+            // TODO: rewrite this with std::ranges when compilers catch up
+            std::vector<BookChange> changes;
+            std::transform(
+                std::make_move_iterator(std::begin(tally_)),
+                std::make_move_iterator(std::end(tally_)),
+                std::back_inserter(changes),
+                [](auto obj) { return obj.second; });
+            return changes;
         }
-        else
+
+    private:
+        void
+        handleAffectedNode(STObject const& node)
         {
-            // TODO: use paranthesized initialization when clang catches up
-            tally_[key] = {
-                first,   // sideAVolume
-                second,  // sideBVolume
-                rate,    // highRate
-                rate,    // lowRate
-                rate,    // openRate
-                rate,    // closeRate
-            };
+            auto const& metaType = node.getFName();
+            auto const nodeType = node.getFieldU16(sfLedgerEntryType);
+
+            // we only care about ltOFFER objects being modified or
+            // deleted
+            if (nodeType != ltOFFER || metaType == sfCreatedNode)
+                return;
+
+            // if either FF or PF are missing we can't compute
+            // but generally these are cancelled rather than crossed
+            // so skipping them is consistent
+            if (!node.isFieldPresent(sfFinalFields) ||
+                !node.isFieldPresent(sfPreviousFields))
+                return;
+
+            auto const& finalFields =
+                node.peekAtField(sfFinalFields).downcast<STObject>();
+            auto const& previousFields =
+                node.peekAtField(sfPreviousFields).downcast<STObject>();
+
+            // defensive case that should never be hit
+            if (!finalFields.isFieldPresent(sfTakerGets) ||
+                !finalFields.isFieldPresent(sfTakerPays) ||
+                !previousFields.isFieldPresent(sfTakerGets) ||
+                !previousFields.isFieldPresent(sfTakerPays))
+                return;
+
+            // filter out any offers deleted by explicit offer cancels
+            if (metaType == sfDeletedNode && offerCancel_ &&
+                finalFields.getFieldU32(sfSequence) == *offerCancel_)
+                return;
+
+            // compute the difference in gets and pays actually
+            // affected onto the offer
+            auto const deltaGets = finalFields.getFieldAmount(sfTakerGets) -
+                previousFields.getFieldAmount(sfTakerGets);
+            auto const deltaPays = finalFields.getFieldAmount(sfTakerPays) -
+                previousFields.getFieldAmount(sfTakerPays);
+
+            transformAndStore(deltaGets, deltaPays);
         }
-    }
 
-    void
-    handleBookChange(Backend::TransactionAndMetadata const& blob)
-    {
-        auto const [tx, meta] = deserializeTxPlusMeta(blob);
-        if (!tx || !meta || !tx->isFieldPresent(sfTransactionType))
-            return;
-
-        offerCancel_ = shouldCancelOffer(tx);
-        for (auto const& node : meta->getFieldArray(sfAffectedNodes))
-            handleAffectedNode(node);
-    }
-
-    std::optional<uint32_t>
-    shouldCancelOffer(std::shared_ptr<ripple::STTx const> const& tx) const
-    {
-        switch (tx->getFieldU16(sfTransactionType))
+        void
+        transformAndStore(
+            ripple::STAmount const& deltaGets,
+            ripple::STAmount const& deltaPays)
         {
-            // in future if any other ways emerge to cancel an offer
-            // this switch makes them easy to add
-            case ttOFFER_CANCEL:
-            case ttOFFER_CREATE:
-                if (tx->isFieldPresent(sfOfferSequence))
-                    return tx->getFieldU32(sfOfferSequence);
-            default:
-                return std::nullopt;
+            auto const g = to_string(deltaGets.issue());
+            auto const p = to_string(deltaPays.issue());
+
+            auto const noswap =
+                isXRP(deltaGets) ? true : (isXRP(deltaPays) ? false : (g < p));
+
+            auto first = noswap ? deltaGets : deltaPays;
+            auto second = noswap ? deltaPays : deltaGets;
+
+            // defensively programmed, should (probably) never happen
+            if (second == beast::zero)
+                return;
+
+            auto const rate = divide(first, second, noIssue());
+
+            if (first < beast::zero)
+                first = -first;
+
+            if (second < beast::zero)
+                second = -second;
+
+            auto const key = noswap ? (g + '|' + p) : (p + '|' + g);
+            if (tally_.contains(key))
+            {
+                auto& entry = tally_.at(key);
+
+                entry.sideAVolume += first;
+                entry.sideBVolume += second;
+
+                if (entry.highRate < rate)
+                    entry.highRate = rate;
+
+                if (entry.lowRate > rate)
+                    entry.lowRate = rate;
+
+                entry.closeRate = rate;
+            }
+            else
+            {
+                // TODO: use paranthesized initialization when clang catches up
+                tally_[key] = {
+                    first,   // sideAVolume
+                    second,  // sideBVolume
+                    rate,    // highRate
+                    rate,    // lowRate
+                    rate,    // openRate
+                    rate,    // closeRate
+                };
+            }
         }
-    }
+
+        void
+        handleBookChange(Backend::TransactionAndMetadata const& blob)
+        {
+            auto const [tx, meta] = deserializeTxPlusMeta(blob);
+            if (!tx || !meta || !tx->isFieldPresent(sfTransactionType))
+                return;
+
+            offerCancel_ = shouldCancelOffer(tx);
+            for (auto const& node : meta->getFieldArray(sfAffectedNodes))
+                handleAffectedNode(node);
+        }
+
+        std::optional<uint32_t>
+        shouldCancelOffer(std::shared_ptr<ripple::STTx const> const& tx) const
+        {
+            switch (tx->getFieldU16(sfTransactionType))
+            {
+                // in future if any other ways emerge to cancel an offer
+                // this switch makes them easy to add
+                case ttOFFER_CANCEL:
+                case ttOFFER_CREATE:
+                    if (tx->isFieldPresent(sfOfferSequence))
+                        return tx->getFieldU32(sfOfferSequence);
+                default:
+                    return std::nullopt;
+            }
+        }
+    };
 };
 
 void
-tag_invoke(
-    const json::value_from_tag&,
-    json::value& jv,
-    BookChange const& change)
+tag_invoke(json::value_from_tag, json::value& jv, BookChange const& change)
 {
     auto amountStr = [](STAmount const& amount) -> std::string {
         return isXRP(amount) ? to_string(amount.xrp())
@@ -228,6 +244,20 @@ tag_invoke(
     };
 }
 
+json::object const
+computeBookChanges(
+    ripple::LedgerInfo const& lgrInfo,
+    std::vector<Backend::TransactionAndMetadata> const& transactions)
+{
+    return {
+        {JS(type), "bookChanges"},
+        {JS(ledger_index), lgrInfo.seq},
+        {JS(ledger_hash), to_string(lgrInfo.hash)},
+        {JS(ledger_time), lgrInfo.closeTime.time_since_epoch().count()},
+        {JS(changes), json::value_from(BookChanges::compute(transactions))},
+    };
+}
+
 Result
 doBookChanges(Context const& context)
 {
@@ -237,14 +267,9 @@ doBookChanges(Context const& context)
         return *status;
 
     auto const lgrInfo = std::get<ripple::LedgerInfo>(info);
-    auto const changes = BookChangesHandler{context}.handle(lgrInfo);
-    return json::object{
-        {JS(type), "bookChanges"},
-        {JS(ledger_index), lgrInfo.seq},
-        {JS(ledger_hash), to_string(lgrInfo.hash)},
-        {JS(ledger_time), lgrInfo.closeTime.time_since_epoch().count()},
-        {JS(changes), json::value_from(changes)},
-    };
+    auto const transactions = context.backend->fetchAllTransactionsInLedger(
+        lgrInfo.seq, context.yield);
+    return computeBookChanges(lgrInfo, transactions);
 }
 
 }  // namespace RPC

src/rpc/handlers/BookOffers.cpp

@@ -1,16 +1,43 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
 #include <ripple/app/ledger/Ledger.h>
 #include <ripple/basics/StringUtilities.h>
 #include <ripple/protocol/ErrorCodes.h>
 #include <ripple/protocol/Indexes.h>
 #include <ripple/protocol/STLedgerEntry.h>
 #include <ripple/protocol/jss.h>
-#include <boost/json.hpp>
-#include <algorithm>
-#include <rpc/RPCHelpers.h>
-
 #include <backend/BackendInterface.h>
 #include <backend/DBHelpers.h>
-#include <backend/Pg.h>
+#include <log/Logger.h>
+#include <rpc/RPCHelpers.h>
+
+#include <boost/json.hpp>
+
+#include <algorithm>
+
+using namespace clio;
+
+// local to compilation unit loggers
+namespace {
+clio::Logger gLog{"RPC"};
+}  // namespace
 
 namespace RPC {
 
@@ -31,10 +58,10 @@ doBookOffers(Context const& context)
     if (request.contains("book"))
     {
         if (!request.at("book").is_string())
-            return Status{Error::rpcINVALID_PARAMS, "bookNotString"};
+            return Status{RippledError::rpcINVALID_PARAMS, "bookNotString"};
 
         if (!bookBase.parseHex(request.at("book").as_string().c_str()))
-            return Status{Error::rpcINVALID_PARAMS, "invalidBook"};
+            return Status{RippledError::rpcINVALID_PARAMS, "invalidBook"};
     }
     else
     {
@@ -65,11 +92,11 @@ doBookOffers(Context const& context)
         bookBase, lgrInfo.seq, limit, marker, context.yield);
     auto end = std::chrono::system_clock::now();
 
-    BOOST_LOG_TRIVIAL(warning)
-        << "Time loading books: "
-        << std::chrono::duration_cast<std::chrono::milliseconds>(end - start)
-               .count()
-        << " milliseconds - request = " << request;
+    gLog.warn() << "Time loading books: "
+                << std::chrono::duration_cast<std::chrono::milliseconds>(
+                       end - start)
+                       .count()
+                << " milliseconds - request = " << request;
 
     response[JS(ledger_hash)] = ripple::strHex(lgrInfo.hash);
     response[JS(ledger_index)] = lgrInfo.seq;
@@ -79,11 +106,11 @@ doBookOffers(Context const& context)
 
     auto end2 = std::chrono::system_clock::now();
 
-    BOOST_LOG_TRIVIAL(warning)
-        << "Time transforming to json: "
-        << std::chrono::duration_cast<std::chrono::milliseconds>(end2 - end)
-               .count()
-        << " milliseconds - request = " << request;
+    gLog.warn() << "Time transforming to json: "
+                << std::chrono::duration_cast<std::chrono::milliseconds>(
+                       end2 - end)
+                       .count()
+                << " milliseconds - request = " << request;
 
     if (retMarker)
         response["marker"] = ripple::strHex(*retMarker);

src/rpc/handlers/ChannelAuthorize.cpp

@@ -1,3 +1,22 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
 #include <ripple/basics/StringUtilities.h>
 #include <ripple/protocol/ErrorCodes.h>
 #include <ripple/protocol/PayChan.h>
@@ -28,13 +47,14 @@ doChannelAuthorize(Context const& context)
     boost::json::object response = {};
 
     if (!request.contains(JS(amount)))
-        return Status{Error::rpcINVALID_PARAMS, "missingAmount"};
+        return Status{RippledError::rpcINVALID_PARAMS, "missingAmount"};
 
     if (!request.at(JS(amount)).is_string())
-        return Status{Error::rpcINVALID_PARAMS, "amountNotString"};
+        return Status{RippledError::rpcINVALID_PARAMS, "amountNotString"};
 
     if (!request.contains(JS(key_type)) && !request.contains(JS(secret)))
-        return Status{Error::rpcINVALID_PARAMS, "missingKeyTypeOrSecret"};
+        return Status{
+            RippledError::rpcINVALID_PARAMS, "missingKeyTypeOrSecret"};
 
     auto v = keypairFromRequst(request);
     if (auto status = std::get_if<Status>(&v))
@@ -51,7 +71,8 @@ doChannelAuthorize(Context const& context)
         ripple::to_uint64(request.at(JS(amount)).as_string().c_str());
 
     if (!optDrops)
-        return Status{Error::rpcCHANNEL_AMT_MALFORMED, "couldNotParseAmount"};
+        return Status{
+            RippledError::rpcCHANNEL_AMT_MALFORMED, "couldNotParseAmount"};
 
     std::uint64_t drops = *optDrops;
 
@@ -66,7 +87,7 @@ doChannelAuthorize(Context const& context)
     }
     catch (std::exception&)
     {
-        return Status{Error::rpcINTERNAL};
+        return Status{RippledError::rpcINTERNAL};
     }
 
     return response;

src/rpc/handlers/ChannelVerify.cpp

@@ -1,3 +1,22 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
 #include <ripple/basics/StringUtilities.h>
 #include <ripple/protocol/ErrorCodes.h>
 #include <ripple/protocol/PayChan.h>
@@ -17,22 +36,22 @@ doChannelVerify(Context const& context)
     boost::json::object response = {};
 
     if (!request.contains(JS(amount)))
-        return Status{Error::rpcINVALID_PARAMS, "missingAmount"};
+        return Status{RippledError::rpcINVALID_PARAMS, "missingAmount"};
 
     if (!request.at(JS(amount)).is_string())
-        return Status{Error::rpcINVALID_PARAMS, "amountNotString"};
+        return Status{RippledError::rpcINVALID_PARAMS, "amountNotString"};
 
     if (!request.contains(JS(signature)))
-        return Status{Error::rpcINVALID_PARAMS, "missingSignature"};
+        return Status{RippledError::rpcINVALID_PARAMS, "missingSignature"};
 
     if (!request.at(JS(signature)).is_string())
-        return Status{Error::rpcINVALID_PARAMS, "signatureNotString"};
+        return Status{RippledError::rpcINVALID_PARAMS, "signatureNotString"};
 
     if (!request.contains(JS(public_key)))
-        return Status{Error::rpcINVALID_PARAMS, "missingPublicKey"};
+        return Status{RippledError::rpcINVALID_PARAMS, "missingPublicKey"};
 
     if (!request.at(JS(public_key)).is_string())
-        return Status{Error::rpcINVALID_PARAMS, "publicKeyNotString"};
+        return Status{RippledError::rpcINVALID_PARAMS, "publicKeyNotString"};
 
     std::optional<ripple::PublicKey> pk;
     {
@@ -45,12 +64,14 @@ doChannelVerify(Context const& context)
         {
             auto pkHex = ripple::strUnHex(strPk);
             if (!pkHex)
-                return Status{Error::rpcPUBLIC_MALFORMED, "malformedPublicKey"};
+                return Status{
+                    RippledError::rpcPUBLIC_MALFORMED, "malformedPublicKey"};
 
             auto const pkType =
                 ripple::publicKeyType(ripple::makeSlice(*pkHex));
             if (!pkType)
-                return Status{Error::rpcPUBLIC_MALFORMED, "invalidKeyType"};
+                return Status{
+                    RippledError::rpcPUBLIC_MALFORMED, "invalidKeyType"};
 
             pk.emplace(ripple::makeSlice(*pkHex));
         }
@@ -64,14 +85,15 @@ doChannelVerify(Context const& context)
         ripple::to_uint64(request.at(JS(amount)).as_string().c_str());
 
     if (!optDrops)
-        return Status{Error::rpcCHANNEL_AMT_MALFORMED, "couldNotParseAmount"};
+        return Status{
+            RippledError::rpcCHANNEL_AMT_MALFORMED, "couldNotParseAmount"};
 
     std::uint64_t drops = *optDrops;
 
     auto sig = ripple::strUnHex(request.at(JS(signature)).as_string().c_str());
 
     if (!sig || !sig->size())
-        return Status{Error::rpcINVALID_PARAMS, "invalidSignature"};
+        return Status{RippledError::rpcINVALID_PARAMS, "invalidSignature"};
 
     ripple::Serializer msg;
     ripple::serializePayChanAuthorization(

src/rpc/handlers/GatewayBalances.cpp

@@ -1,3 +1,22 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
 #include <backend/BackendInterface.h>
 #include <rpc/RPCHelpers.h>
 
@@ -25,7 +44,7 @@ doGatewayBalances(Context const& context)
     std::map<ripple::AccountID, std::vector<ripple::STAmount>> frozenBalances;
     std::set<ripple::AccountID> hotWallets;
 
-    if (request.contains("hot_wallet"))
+    if (request.contains(JS(hotwallet)))
     {
         auto getAccountID =
             [](auto const& j) -> std::optional<ripple::AccountID> {
@@ -44,7 +63,7 @@ doGatewayBalances(Context const& context)
             return {};
         };
 
-        auto const& hw = request.at("hot_wallet");
+        auto const& hw = request.at(JS(hotwallet));
         bool valid = true;
 
         // null is treated as a valid 0-sized array of hotwallet
@@ -79,7 +98,7 @@ doGatewayBalances(Context const& context)
     }
 
     // Traverse the cold wallet's trust lines
-    auto const addToResponse = [&](ripple::SLE const& sle) {
+    auto const addToResponse = [&](ripple::SLE&& sle) {
         if (sle.getType() == ripple::ltRIPPLE_STATE)
         {
             ripple::STAmount balance = sle.getFieldAmount(ripple::sfBalance);
@@ -110,7 +129,7 @@ doGatewayBalances(Context const& context)
             if (hotWallets.count(peer) > 0)
             {
                 // This is a specified hot wallet
-                hotBalances[peer].push_back(balance);
+                hotBalances[peer].push_back(-balance);
             }
             else if (balSign > 0)
             {
@@ -120,7 +139,7 @@ doGatewayBalances(Context const& context)
             else if (freeze)
             {
                 // An obligation the gateway has frozen
-                frozenBalances[peer].push_back(balance);
+                frozenBalances[peer].push_back(-balance);
             }
             else
             {
@@ -138,7 +157,7 @@ doGatewayBalances(Context const& context)
         return true;
     };
 
-    traverseOwnedNodes(
+    auto result = traverseOwnedNodes(
         *context.backend,
         accountID,
         lgrInfo.seq,
@@ -146,6 +165,8 @@ doGatewayBalances(Context const& context)
         {},
         context.yield,
         addToResponse);
+    if (auto status = std::get_if<RPC::Status>(&result))
+        return *status;
 
     if (!sums.empty())
     {
@@ -180,6 +201,13 @@ doGatewayBalances(Context const& context)
             return obj;
         };
 
+    auto containsHotWallet = [&](auto const& hw) {
+        return hotBalances.contains(hw);
+    };
+    if (not std::all_of(
+            hotWallets.begin(), hotWallets.end(), containsHotWallet))
+        return Status{RippledError::rpcINVALID_PARAMS, "invalidHotWallet"};
+
     if (auto balances = toJson(hotBalances); balances.size())
         response[JS(balances)] = balances;
     if (auto balances = toJson(frozenBalances); balances.size())

src/rpc/handlers/Ledger.cpp

@@ -1,3 +1,22 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
 #include <backend/BackendInterface.h>
 #include <rpc/RPCHelpers.h>
 
@@ -13,7 +32,7 @@ doLedger(Context const& context)
     if (params.contains(JS(binary)))
     {
         if (!params.at(JS(binary)).is_bool())
-            return Status{Error::rpcINVALID_PARAMS, "binaryFlagNotBool"};
+            return Status{RippledError::rpcINVALID_PARAMS, "binaryFlagNotBool"};
 
         binary = params.at(JS(binary)).as_bool();
     }
@@ -22,7 +41,8 @@ doLedger(Context const& context)
     if (params.contains(JS(transactions)))
     {
         if (!params.at(JS(transactions)).is_bool())
-            return Status{Error::rpcINVALID_PARAMS, "transactionsFlagNotBool"};
+            return Status{
+                RippledError::rpcINVALID_PARAMS, "transactionsFlagNotBool"};
 
         transactions = params.at(JS(transactions)).as_bool();
     }
@@ -31,7 +51,7 @@ doLedger(Context const& context)
     if (params.contains(JS(expand)))
     {
         if (!params.at(JS(expand)).is_bool())
-            return Status{Error::rpcINVALID_PARAMS, "expandFlagNotBool"};
+            return Status{RippledError::rpcINVALID_PARAMS, "expandFlagNotBool"};
 
         expand = params.at(JS(expand)).as_bool();
     }
@@ -40,11 +60,17 @@ doLedger(Context const& context)
     if (params.contains("diff"))
     {
         if (!params.at("diff").is_bool())
-            return Status{Error::rpcINVALID_PARAMS, "diffFlagNotBool"};
+            return Status{RippledError::rpcINVALID_PARAMS, "diffFlagNotBool"};
 
         diff = params.at("diff").as_bool();
     }
 
+    if (params.contains(JS(full)))
+        return Status{RippledError::rpcNOT_SUPPORTED};
+
+    if (params.contains(JS(accounts)))
+        return Status{RippledError::rpcNOT_SUPPORTED};
+
     auto v = ledgerInfoFromRequest(context);
     if (auto status = std::get_if<Status>(&v))
         return *status;

src/rpc/handlers/LedgerData.cpp

@@ -1,9 +1,30 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
 #include <ripple/app/ledger/LedgerToJson.h>
 #include <ripple/protocol/STLedgerEntry.h>
+#include <backend/BackendInterface.h>
+#include <log/Logger.h>
+#include <rpc/RPCHelpers.h>
+
 #include <boost/json.hpp>
 
-#include <backend/BackendInterface.h>
-#include <rpc/RPCHelpers.h>
 // Get state nodes from a ledger
 //   Inputs:
 //     limit:        integer, maximum number of entries
@@ -18,6 +39,13 @@
 //
 //
 
+using namespace clio;
+
+// local to compilation unit loggers
+namespace {
+clio::Logger gLog{"RPC"};
+}  // namespace
+
 namespace RPC {
 
 using boost::json::value_to;
@@ -41,7 +69,7 @@ doLedgerData(Context const& context)
     if (request.contains("out_of_order"))
     {
         if (!request.at("out_of_order").is_bool())
-            return Status{Error::rpcINVALID_PARAMS, "binaryFlagNotBool"};
+            return Status{RippledError::rpcINVALID_PARAMS, "binaryFlagNotBool"};
         outOfOrder = request.at("out_of_order").as_bool();
     }
 
@@ -55,19 +83,22 @@ doLedgerData(Context const& context)
             {
                 if (!request.at(JS(marker)).is_int64())
                     return Status{
-                        Error::rpcINVALID_PARAMS, "markerNotStringOrInt"};
+                        RippledError::rpcINVALID_PARAMS,
+                        "markerNotStringOrInt"};
                 diffMarker = value_to<uint32_t>(request.at(JS(marker)));
             }
             else
-                return Status{Error::rpcINVALID_PARAMS, "markerNotString"};
+                return Status{
+                    RippledError::rpcINVALID_PARAMS, "markerNotString"};
         }
         else
         {
-            BOOST_LOG_TRIVIAL(debug) << __func__ << " : parsing marker";
+            gLog.debug() << "Parsing marker";
 
             marker = ripple::uint256{};
             if (!marker->parseHex(request.at(JS(marker)).as_string().c_str()))
-                return Status{Error::rpcINVALID_PARAMS, "markerMalformed"};
+                return Status{
+                    RippledError::rpcINVALID_PARAMS, "markerMalformed"};
         }
     }
 
@@ -95,7 +126,6 @@ doLedgerData(Context const& context)
             header[JS(close_time_human)] = ripple::to_string(lgrInfo.closeTime);
             header[JS(close_time_resolution)] =
                 lgrInfo.closeTimeResolution.count();
-            header[JS(closed)] = true;
             header[JS(hash)] = ripple::strHex(lgrInfo.hash);
             header[JS(ledger_hash)] = ripple::strHex(lgrInfo.hash);
             header[JS(ledger_index)] = std::to_string(lgrInfo.seq);
@@ -108,6 +138,7 @@ doLedgerData(Context const& context)
             header[JS(transaction_hash)] = ripple::strHex(lgrInfo.txHash);
         }
 
+        header[JS(closed)] = true;
         response[JS(ledger)] = header;
     }
     else
@@ -115,7 +146,8 @@ doLedgerData(Context const& context)
         if (!outOfOrder &&
             !context.backend->fetchLedgerObject(
                 *marker, lgrInfo.seq, context.yield))
-            return Status{Error::rpcINVALID_PARAMS, "markerDoesNotExist"};
+            return Status{
+                RippledError::rpcINVALID_PARAMS, "markerDoesNotExist"};
     }
 
     response[JS(ledger_hash)] = ripple::strHex(lgrInfo.hash);
@@ -164,9 +196,8 @@ doLedgerData(Context const& context)
         std::chrono::duration_cast<std::chrono::microseconds>(end - start)
             .count();
 
-    BOOST_LOG_TRIVIAL(debug)
-        << __func__ << " number of results = " << results.size()
-        << " fetched in " << time << " microseconds";
+    gLog.debug() << "Number of results = " << results.size() << " fetched in "
+                 << time << " microseconds";
     boost::json::array objects;
     objects.reserve(results.size());
     for (auto const& [key, object] : results)
@@ -184,13 +215,14 @@ doLedgerData(Context const& context)
             objects.push_back(toJson(sle));
     }
     response[JS(state)] = std::move(objects);
+    if (outOfOrder)
+        response["cache_full"] = context.backend->cache().isFull();
     auto end2 = std::chrono::system_clock::now();
 
     time = std::chrono::duration_cast<std::chrono::microseconds>(end2 - end)
                .count();
-    BOOST_LOG_TRIVIAL(debug)
-        << __func__ << " number of results = " << results.size()
-        << " serialized in " << time << " microseconds";
+    gLog.debug() << "Number of results = " << results.size()
+                 << " serialized in " << time << " microseconds";
 
     return response;
 }

src/rpc/handlers/LedgerEntry.cpp

@@ -1,3 +1,22 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
 #include <ripple/protocol/Indexes.h>
 #include <ripple/protocol/STLedgerEntry.h>
 #include <boost/json.hpp>
@@ -29,34 +48,38 @@ doLedgerEntry(Context const& context)
     auto lgrInfo = std::get<ripple::LedgerInfo>(v);
 
     ripple::uint256 key;
+
+    // Note: according to docs, only 1 of the below should be specified at any
+    // time. see https://xrpl.org/ledger_entry.html#ledger_entry
     if (request.contains(JS(index)))
     {
         if (!request.at(JS(index)).is_string())
-            return Status{Error::rpcINVALID_PARAMS, "indexNotString"};
+            return Status{RippledError::rpcINVALID_PARAMS, "indexNotString"};
 
         if (!key.parseHex(request.at(JS(index)).as_string().c_str()))
-            return Status{Error::rpcINVALID_PARAMS, "malformedIndex"};
+            return Status{ClioError::rpcMALFORMED_REQUEST};
     }
     else if (request.contains(JS(account_root)))
     {
         if (!request.at(JS(account_root)).is_string())
-            return Status{Error::rpcINVALID_PARAMS, "account_rootNotString"};
+            return Status{
+                RippledError::rpcINVALID_PARAMS, "account_rootNotString"};
 
         auto const account = ripple::parseBase58<ripple::AccountID>(
             request.at(JS(account_root)).as_string().c_str());
         if (!account || account->isZero())
-            return Status{Error::rpcINVALID_PARAMS, "malformedAddress"};
+            return Status{ClioError::rpcMALFORMED_ADDRESS};
         else
             key = ripple::keylet::account(*account).key;
     }
     else if (request.contains(JS(check)))
     {
         if (!request.at(JS(check)).is_string())
-            return Status{Error::rpcINVALID_PARAMS, "checkNotString"};
+            return Status{RippledError::rpcINVALID_PARAMS, "checkNotString"};
 
         if (!key.parseHex(request.at(JS(check)).as_string().c_str()))
         {
-            return Status{Error::rpcINVALID_PARAMS, "checkMalformed"};
+            return Status{RippledError::rpcINVALID_PARAMS, "checkMalformed"};
         }
     }
     else if (request.contains(JS(deposit_preauth)))
@@ -68,7 +91,8 @@ doLedgerEntry(Context const& context)
                     request.at(JS(deposit_preauth)).as_string().c_str()))
             {
                 return Status{
-                    Error::rpcINVALID_PARAMS, "deposit_preauthMalformed"};
+                    RippledError::rpcINVALID_PARAMS,
+                    "deposit_preauthMalformed"};
             }
         }
         else if (
@@ -78,7 +102,7 @@ doLedgerEntry(Context const& context)
                  .at(JS(owner))
                  .is_string())
         {
-            return Status{Error::rpcINVALID_PARAMS, "ownerNotString"};
+            return Status{RippledError::rpcINVALID_PARAMS, "malformedOwner"};
         }
         else if (
             !request.at(JS(deposit_preauth))
@@ -89,7 +113,8 @@ doLedgerEntry(Context const& context)
                  .at(JS(authorized))
                  .is_string())
         {
-            return Status{Error::rpcINVALID_PARAMS, "authorizedNotString"};
+            return Status{
+                RippledError::rpcINVALID_PARAMS, "authorizedNotString"};
         }
         else
         {
@@ -103,9 +128,11 @@ doLedgerEntry(Context const& context)
                 deposit_preauth.at(JS(authorized)).as_string().c_str());
 
             if (!owner)
-                return Status{Error::rpcINVALID_PARAMS, "malformedOwner"};
+                return Status{
+                    RippledError::rpcINVALID_PARAMS, "malformedOwner"};
             else if (!authorized)
-                return Status{Error::rpcINVALID_PARAMS, "malformedAuthorized"};
+                return Status{
+                    RippledError::rpcINVALID_PARAMS, "malformedAuthorized"};
             else
                 key = ripple::keylet::depositPreauth(*owner, *authorized).key;
         }
@@ -115,18 +142,20 @@ doLedgerEntry(Context const& context)
         if (!request.at(JS(directory)).is_object())
         {
             if (!request.at(JS(directory)).is_string())
-                return Status{Error::rpcINVALID_PARAMS, "directoryNotString"};
+                return Status{
+                    RippledError::rpcINVALID_PARAMS, "directoryNotString"};
 
             if (!key.parseHex(request.at(JS(directory)).as_string().c_str()))
             {
-                return Status{Error::rpcINVALID_PARAMS, "malformedDirectory"};
+                return Status{
+                    RippledError::rpcINVALID_PARAMS, "malformedDirectory"};
             }
         }
         else if (
             request.at(JS(directory)).as_object().contains(JS(sub_index)) &&
             !request.at(JS(directory)).as_object().at(JS(sub_index)).is_int64())
         {
-            return Status{Error::rpcINVALID_PARAMS, "sub_indexNotInt"};
+            return Status{RippledError::rpcINVALID_PARAMS, "sub_indexNotInt"};
         }
         else
         {
@@ -144,13 +173,14 @@ doLedgerEntry(Context const& context)
                 {
                     // May not specify both dir_root and owner.
                     return Status{
-                        Error::rpcINVALID_PARAMS,
+                        RippledError::rpcINVALID_PARAMS,
                         "mayNotSpecifyBothDirRootAndOwner"};
                 }
                 else if (!uDirRoot.parseHex(
                              directory.at(JS(dir_root)).as_string().c_str()))
                 {
-                    return Status{Error::rpcINVALID_PARAMS, "malformedDirRoot"};
+                    return Status{
+                        RippledError::rpcINVALID_PARAMS, "malformedDirRoot"};
                 }
                 else
                 {
@@ -164,7 +194,7 @@ doLedgerEntry(Context const& context)
 
                 if (!ownerID)
                 {
-                    return Status{Error::rpcINVALID_PARAMS, "malformedAddress"};
+                    return Status{ClioError::rpcMALFORMED_ADDRESS};
                 }
                 else
                 {
@@ -176,7 +206,7 @@ doLedgerEntry(Context const& context)
             else
             {
                 return Status{
-                    Error::rpcINVALID_PARAMS, "missingOwnerOrDirRoot"};
+                    RippledError::rpcINVALID_PARAMS, "missingOwnerOrDirRoot"};
             }
         }
     }
@@ -185,19 +215,20 @@ doLedgerEntry(Context const& context)
         if (!request.at(JS(escrow)).is_object())
         {
             if (!key.parseHex(request.at(JS(escrow)).as_string().c_str()))
-                return Status{Error::rpcINVALID_PARAMS, "malformedEscrow"};
+                return Status{
+                    RippledError::rpcINVALID_PARAMS, "malformedEscrow"};
         }
         else if (
             !request.at(JS(escrow)).as_object().contains(JS(owner)) ||
             !request.at(JS(escrow)).as_object().at(JS(owner)).is_string())
         {
-            return Status{Error::rpcINVALID_PARAMS, "malformedOwner"};
+            return Status{RippledError::rpcINVALID_PARAMS, "malformedOwner"};
         }
         else if (
             !request.at(JS(escrow)).as_object().contains(JS(seq)) ||
             !request.at(JS(escrow)).as_object().at(JS(seq)).is_int64())
         {
-            return Status{Error::rpcINVALID_PARAMS, "malformedSeq"};
+            return Status{RippledError::rpcINVALID_PARAMS, "malformedSeq"};
         }
         else
         {
@@ -209,7 +240,7 @@ doLedgerEntry(Context const& context)
                                                            .c_str());
 
             if (!id)
-                return Status{Error::rpcINVALID_PARAMS, "malformedOwner"};
+                return Status{ClioError::rpcMALFORMED_ADDRESS};
             else
             {
                 std::uint32_t seq =
@@ -223,19 +254,20 @@ doLedgerEntry(Context const& context)
         if (!request.at(JS(offer)).is_object())
         {
             if (!key.parseHex(request.at(JS(offer)).as_string().c_str()))
-                return Status{Error::rpcINVALID_PARAMS, "malformedOffer"};
+                return Status{
+                    RippledError::rpcINVALID_PARAMS, "malformedOffer"};
         }
         else if (
             !request.at(JS(offer)).as_object().contains(JS(account)) ||
             !request.at(JS(offer)).as_object().at(JS(account)).is_string())
         {
-            return Status{Error::rpcINVALID_PARAMS, "malformedAccount"};
+            return Status{RippledError::rpcINVALID_PARAMS, "malformedAccount"};
         }
         else if (
             !request.at(JS(offer)).as_object().contains(JS(seq)) ||
             !request.at(JS(offer)).as_object().at(JS(seq)).is_int64())
         {
-            return Status{Error::rpcINVALID_PARAMS, "malformedSeq"};
+            return Status{RippledError::rpcINVALID_PARAMS, "malformedSeq"};
         }
         else
         {
@@ -244,7 +276,7 @@ doLedgerEntry(Context const& context)
                 offer.at(JS(account)).as_string().c_str());
 
             if (!id)
-                return Status{Error::rpcINVALID_PARAMS, "malformedAccount"};
+                return Status{ClioError::rpcMALFORMED_ADDRESS};
             else
             {
                 std::uint32_t seq =
@@ -256,15 +288,18 @@ doLedgerEntry(Context const& context)
     else if (request.contains(JS(payment_channel)))
     {
         if (!request.at(JS(payment_channel)).is_string())
-            return Status{Error::rpcINVALID_PARAMS, "paymentChannelNotString"};
+            return Status{
+                RippledError::rpcINVALID_PARAMS, "paymentChannelNotString"};
 
         if (!key.parseHex(request.at(JS(payment_channel)).as_string().c_str()))
-            return Status{Error::rpcINVALID_PARAMS, "malformedPaymentChannel"};
+            return Status{
+                RippledError::rpcINVALID_PARAMS, "malformedPaymentChannel"};
     }
     else if (request.contains(JS(ripple_state)))
     {
         if (!request.at(JS(ripple_state)).is_object())
-            return Status{Error::rpcINVALID_PARAMS, "rippleStateNotObject"};
+            return Status{
+                RippledError::rpcINVALID_PARAMS, "rippleStateNotObject"};
 
         ripple::Currency currency;
         boost::json::object const& state =
@@ -273,7 +308,7 @@ doLedgerEntry(Context const& context)
         if (!state.contains(JS(currency)) ||
             !state.at(JS(currency)).is_string())
         {
-            return Status{Error::rpcINVALID_PARAMS, "malformedCurrency"};
+            return Status{RippledError::rpcINVALID_PARAMS, "currencyNotString"};
         }
 
         if (!state.contains(JS(accounts)) ||
@@ -284,7 +319,7 @@ doLedgerEntry(Context const& context)
             (state.at(JS(accounts)).as_array().at(0).as_string() ==
              state.at(JS(accounts)).as_array().at(1).as_string()))
         {
-            return Status{Error::rpcINVALID_PARAMS, "malformedAccounts"};
+            return Status{RippledError::rpcINVALID_PARAMS, "malformedAccounts"};
         }
 
         auto const id1 = ripple::parseBase58<ripple::AccountID>(
@@ -293,11 +328,13 @@ doLedgerEntry(Context const& context)
             state.at(JS(accounts)).as_array().at(1).as_string().c_str());
 
         if (!id1 || !id2)
-            return Status{Error::rpcINVALID_PARAMS, "malformedAccounts"};
+            return Status{
+                ClioError::rpcMALFORMED_ADDRESS, "malformedAddresses"};
 
         else if (!ripple::to_currency(
                      currency, state.at(JS(currency)).as_string().c_str()))
-            return Status{Error::rpcINVALID_PARAMS, "malformedCurrency"};
+            return Status{
+                ClioError::rpcMALFORMED_CURRENCY, "malformedCurrency"};
 
         key = ripple::keylet::line(*id1, *id2, currency).key;
     }
@@ -306,35 +343,37 @@ doLedgerEntry(Context const& context)
         if (!request.at(JS(ticket)).is_object())
         {
             if (!request.at(JS(ticket)).is_string())
-                return Status{Error::rpcINVALID_PARAMS, "ticketNotString"};
+                return Status{
+                    ClioError::rpcMALFORMED_REQUEST, "ticketNotString"};
 
             if (!key.parseHex(request.at(JS(ticket)).as_string().c_str()))
-                return Status{Error::rpcINVALID_PARAMS, "malformedTicket"};
+                return Status{
+                    ClioError::rpcMALFORMED_REQUEST, "malformedTicket"};
         }
         else if (
-            !request.at(JS(ticket)).as_object().contains(JS(account)) ||
-            !request.at(JS(ticket)).as_object().at(JS(account)).is_string())
+            !request.at(JS(ticket)).as_object().contains(JS(owner)) ||
+            !request.at(JS(ticket)).as_object().at(JS(owner)).is_string())
         {
-            return Status{Error::rpcINVALID_PARAMS, "malformedTicketAccount"};
+            return Status{ClioError::rpcMALFORMED_REQUEST};
         }
         else if (
             !request.at(JS(ticket)).as_object().contains(JS(ticket_seq)) ||
             !request.at(JS(ticket)).as_object().at(JS(ticket_seq)).is_int64())
         {
-            return Status{Error::rpcINVALID_PARAMS, "malformedTicketSeq"};
+            return Status{
+                ClioError::rpcMALFORMED_REQUEST, "malformedTicketSeq"};
         }
         else
         {
             auto const id =
                 ripple::parseBase58<ripple::AccountID>(request.at(JS(ticket))
                                                            .as_object()
-                                                           .at(JS(account))
+                                                           .at(JS(owner))
                                                            .as_string()
                                                            .c_str());
 
             if (!id)
-                return Status{
-                    Error::rpcINVALID_PARAMS, "malformedTicketAccount"};
+                return Status{ClioError::rpcMALFORMED_OWNER};
             else
             {
                 std::uint32_t seq = request.at(JS(offer))
@@ -348,7 +387,7 @@ doLedgerEntry(Context const& context)
     }
     else
     {
-        return Status{Error::rpcINVALID_PARAMS, "unknownOption"};
+        return Status{RippledError::rpcINVALID_PARAMS, "unknownOption"};
     }
 
     auto dbResponse =

src/rpc/handlers/LedgerRange.cpp

@@ -1,3 +1,21 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
 
 #include <backend/BackendInterface.h>
 #include <rpc/RPCHelpers.h>
@@ -12,7 +30,7 @@ doLedgerRange(Context const& context)
     auto range = context.backend->fetchLedgerRange();
     if (!range)
     {
-        return Status{Error::rpcNOT_READY, "rangeNotFound"};
+        return Status{RippledError::rpcNOT_READY, "rangeNotFound"};
     }
     else
     {

src/rpc/handlers/NFTHistory.cpp

@@ -0,0 +1,70 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include <log/Logger.h>
+#include <rpc/RPCHelpers.h>
+#include <util/Profiler.h>
+
+using namespace clio;
+
+// local to compilation unit loggers
+namespace {
+clio::Logger gLog{"RPC"};
+}  // namespace
+
+namespace RPC {
+
+Result
+doNFTHistory(Context const& context)
+{
+    auto const maybeTokenID = getNFTID(context.params);
+    if (auto const status = std::get_if<Status>(&maybeTokenID); status)
+        return *status;
+    auto const tokenID = std::get<ripple::uint256>(maybeTokenID);
+
+    constexpr std::string_view outerFuncName = __func__;
+    auto const maybeResponse = traverseTransactions(
+        context,
+        [&tokenID, &outerFuncName](
+            std::shared_ptr<Backend::BackendInterface const> const& backend,
+            std::uint32_t const limit,
+            bool const forward,
+            std::optional<Backend::TransactionsCursor> const& cursorIn,
+            boost::asio::yield_context& yield)
+            -> Backend::TransactionsAndCursor {
+            auto const [txnsAndCursor, timeDiff] =
+                util::timed([&, &tokenID = tokenID]() {
+                    return backend->fetchNFTTransactions(
+                        tokenID, limit, forward, cursorIn, yield);
+                });
+            gLog.info() << outerFuncName << " db fetch took " << timeDiff
+                        << " milliseconds - num blobs = "
+                        << txnsAndCursor.txns.size();
+            return txnsAndCursor;
+        });
+
+    if (auto const status = std::get_if<Status>(&maybeResponse); status)
+        return *status;
+    auto response = std::get<boost::json::object>(maybeResponse);
+
+    response[JS(nft_id)] = ripple::to_string(tokenID);
+    return response;
+}
+
+}  // namespace RPC

src/rpc/handlers/NFTInfo.cpp

@@ -1,3 +1,22 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
 #include <ripple/app/tx/impl/details/NFTokenUtils.h>
 #include <ripple/protocol/Indexes.h>
 #include <boost/json.hpp>
@@ -45,7 +64,8 @@ getURI(Backend::NFT const& dbResponse, Context const& context)
 
         if (!blob || blob->size() == 0)
             return Status{
-                Error::rpcINTERNAL, "Cannot find NFTokenPage for this NFT"};
+                RippledError::rpcINTERNAL,
+                "Cannot find NFTokenPage for this NFT"};
 
         sle = ripple::STLedgerEntry(
             ripple::SerialIter{blob->data(), blob->size()}, nextKey);
@@ -57,7 +77,7 @@ getURI(Backend::NFT const& dbResponse, Context const& context)
 
     if (!sle)
         return Status{
-            Error::rpcINTERNAL, "Cannot find NFTokenPage for this NFT"};
+            RippledError::rpcINTERNAL, "Cannot find NFTokenPage for this NFT"};
 
     auto const nfts = sle->getFieldArray(ripple::sfNFTokens);
     auto const nft = std::find_if(
@@ -70,7 +90,7 @@ getURI(Backend::NFT const& dbResponse, Context const& context)
 
     if (nft == nfts.end())
         return Status{
-            Error::rpcINTERNAL, "Cannot find NFTokenPage for this NFT"};
+            RippledError::rpcINTERNAL, "Cannot find NFTokenPage for this NFT"};
 
     ripple::Blob const uriField = nft->getFieldVL(ripple::sfURI);
 
@@ -89,29 +109,20 @@ doNFTInfo(Context const& context)
     auto request = context.params;
     boost::json::object response = {};
 
-    if (!request.contains("nft_id"))
-        return Status{Error::rpcINVALID_PARAMS, "Missing nft_id"};
-
-    auto const& jsonTokenID = request.at("nft_id");
-    if (!jsonTokenID.is_string())
-        return Status{Error::rpcINVALID_PARAMS, "nft_id is not a string"};
-
-    ripple::uint256 tokenID;
-    if (!tokenID.parseHex(jsonTokenID.as_string().c_str()))
-        return Status{Error::rpcINVALID_PARAMS, "Malformed nft_id"};
-
-    // We only need to fetch the ledger header because the ledger hash is
-    // supposed to be included in the response. The ledger sequence is specified
-    // in the request
-    auto v = ledgerInfoFromRequest(context);
-    if (auto status = std::get_if<Status>(&v))
+    auto const maybeTokenID = getNFTID(request);
+    if (auto const status = std::get_if<Status>(&maybeTokenID); status)
         return *status;
-    ripple::LedgerInfo lgrInfo = std::get<ripple::LedgerInfo>(v);
+    auto const tokenID = std::get<ripple::uint256>(maybeTokenID);
+
+    auto const maybeLedgerInfo = ledgerInfoFromRequest(context);
+    if (auto status = std::get_if<Status>(&maybeLedgerInfo); status)
+        return *status;
+    auto const lgrInfo = std::get<ripple::LedgerInfo>(maybeLedgerInfo);
 
     std::optional<Backend::NFT> dbResponse =
         context.backend->fetchNFT(tokenID, lgrInfo.seq, context.yield);
     if (!dbResponse)
-        return Status{Error::rpcOBJECT_NOT_FOUND, "NFT not found"};
+        return Status{RippledError::rpcOBJECT_NOT_FOUND, "NFT not found"};
 
     response["nft_id"] = ripple::strHex(dbResponse->tokenID);
     response["ledger_index"] = dbResponse->ledgerSequence;
@@ -130,10 +141,10 @@ doNFTInfo(Context const& context)
     {
         auto const maybeURI = getURI(*dbResponse, context);
         // An error occurred
-        if (Status const* status = std::get_if<Status>(&maybeURI))
+        if (Status const* status = std::get_if<Status>(&maybeURI); status)
             return *status;
         // A URI was found
-        if (std::string const* uri = std::get_if<std::string>(&maybeURI))
+        if (std::string const* uri = std::get_if<std::string>(&maybeURI); uri)
             response["uri"] = *uri;
         // A URI was not found, explicitly set to null
         else

src/rpc/handlers/NFTOffers.cpp

@@ -1,37 +1,66 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
 #include <ripple/app/ledger/Ledger.h>
 #include <ripple/basics/StringUtilities.h>
 #include <ripple/protocol/ErrorCodes.h>
 #include <ripple/protocol/Indexes.h>
 #include <ripple/protocol/STLedgerEntry.h>
 #include <ripple/protocol/jss.h>
-#include <boost/json.hpp>
-#include <algorithm>
 #include <rpc/RPCHelpers.h>
 
+#include <boost/json.hpp>
+
+#include <algorithm>
+
+namespace json = boost::json;
+
+namespace ripple {
+
+void
+tag_invoke(json::value_from_tag, json::value& jv, SLE const& offer)
+{
+    auto amount = ::RPC::toBoostJson(
+        offer.getFieldAmount(sfAmount).getJson(JsonOptions::none));
+
+    json::object obj = {
+        {JS(nft_offer_index), to_string(offer.key())},
+        {JS(flags), offer[sfFlags]},
+        {JS(owner), toBase58(offer.getAccountID(sfOwner))},
+        {JS(amount), std::move(amount)},
+    };
+
+    if (offer.isFieldPresent(sfDestination))
+        obj.insert_or_assign(
+            JS(destination), toBase58(offer.getAccountID(sfDestination)));
+
+    if (offer.isFieldPresent(sfExpiration))
+        obj.insert_or_assign(JS(expiration), offer.getFieldU32(sfExpiration));
+
+    jv = std::move(obj);
+}
+
+}  // namespace ripple
+
 namespace RPC {
 
-static void
-appendNftOfferJson(ripple::SLE const& offer, boost::json::array& offers)
-{
-    offers.push_back(boost::json::object_kind);
-    boost::json::object& obj(offers.back().as_object());
-
-    obj[JS(index)] = ripple::to_string(offer.key());
-    obj[JS(flags)] = (offer)[ripple::sfFlags];
-    obj[JS(owner)] = ripple::toBase58(offer.getAccountID(ripple::sfOwner));
-
-    if (offer.isFieldPresent(ripple::sfDestination))
-        obj[JS(destination)] =
-            ripple::toBase58(offer.getAccountID(ripple::sfDestination));
-
-    if (offer.isFieldPresent(ripple::sfExpiration))
-        obj[JS(expiration)] = offer.getFieldU32(ripple::sfExpiration);
-
-    obj[JS(amount)] = toBoostJson(offer.getFieldAmount(ripple::sfAmount)
-                                      .getJson(ripple::JsonOptions::none));
-}
-
-static Result
+Result
 enumerateNFTOffers(
     Context const& context,
     ripple::uint256 const& tokenid,
@@ -48,21 +77,20 @@ enumerateNFTOffers(
     // TODO: just check for existence without pulling
     if (!context.backend->fetchLedgerObject(
             directory.key, lgrInfo.seq, context.yield))
-        return Status{Error::rpcOBJECT_NOT_FOUND, "notFound"};
+        return Status{RippledError::rpcOBJECT_NOT_FOUND, "notFound"};
 
     std::uint32_t limit;
     if (auto const status = getLimit(context, limit); status)
         return status;
 
     boost::json::object response = {};
+    boost::json::array jsonOffers = {};
     response[JS(nft_id)] = ripple::to_string(tokenid);
-    response[JS(offers)] = boost::json::value(boost::json::array_kind);
-
-    auto& jsonOffers = response[JS(offers)].as_array();
 
     std::vector<ripple::SLE> offers;
-    std::uint64_t reserve(limit);
+    auto reserve = limit;
     ripple::uint256 cursor;
+    uint64_t startHint = 0;
 
     if (request.contains(JS(marker)))
     {
@@ -71,21 +99,22 @@ enumerateNFTOffers(
         auto const& marker(request.at(JS(marker)));
 
         if (!marker.is_string())
-            return Status{Error::rpcINVALID_PARAMS, "markerNotString"};
+            return Status{RippledError::rpcINVALID_PARAMS, "markerNotString"};
 
         if (!cursor.parseHex(marker.as_string().c_str()))
-            return Status{Error::rpcINVALID_PARAMS, "malformedCursor"};
+            return Status{RippledError::rpcINVALID_PARAMS, "malformedCursor"};
 
         auto const sle =
             read(ripple::keylet::nftoffer(cursor), lgrInfo, context);
 
-        if (!sle || tokenid != sle->getFieldH256(ripple::sfNFTokenID))
-            return Status{Error::rpcOBJECT_NOT_FOUND, "notFound"};
+        if (!sle ||
+            sle->getFieldU16(ripple::sfLedgerEntryType) !=
+                ripple::ltNFTOKEN_OFFER ||
+            tokenid != sle->getFieldH256(ripple::sfNFTokenID))
+            return Status{RippledError::rpcINVALID_PARAMS};
 
-        if (tokenid != sle->getFieldH256(ripple::sfNFTokenID))
-            return Status{Error::rpcINVALID_PARAMS, "invalidTokenid"};
-
-        appendNftOfferJson(*sle, jsonOffers);
+        startHint = sle->getFieldU64(ripple::sfNFTokenOfferNode);
+        jsonOffers.push_back(json::value_from(*sle));
         offers.reserve(reserve);
     }
     else
@@ -98,15 +127,15 @@ enumerateNFTOffers(
         *context.backend,
         directory,
         cursor,
-        0,
+        startHint,
         lgrInfo.seq,
-        limit,
+        reserve,
         {},
         context.yield,
-        [&offers](ripple::SLE const& offer) {
+        [&offers](ripple::SLE&& offer) {
             if (offer.getType() == ripple::ltNFTOKEN_OFFER)
             {
-                offers.emplace_back(offer);
+                offers.push_back(std::move(offer));
                 return true;
             }
 
@@ -123,32 +152,23 @@ enumerateNFTOffers(
         offers.pop_back();
     }
 
-    for (auto const& offer : offers)
-        appendNftOfferJson(offer, jsonOffers);
+    std::transform(
+        std::cbegin(offers),
+        std::cend(offers),
+        std::back_inserter(jsonOffers),
+        [](auto const& offer) {
+            // uses tag_invoke at the top of this file
+            return json::value_from(offer);
+        });
 
+    response.insert_or_assign(JS(offers), std::move(jsonOffers));
     return response;
 }
 
-std::variant<ripple::uint256, Status>
-getTokenid(boost::json::object const& request)
-{
-    if (!request.contains(JS(nft_id)))
-        return Status{Error::rpcINVALID_PARAMS, "missingTokenid"};
-
-    if (!request.at(JS(nft_id)).is_string())
-        return Status{Error::rpcINVALID_PARAMS, "tokenidNotString"};
-
-    ripple::uint256 tokenid;
-    if (!tokenid.parseHex(request.at(JS(nft_id)).as_string().c_str()))
-        return Status{Error::rpcINVALID_PARAMS, "malformedCursor"};
-
-    return tokenid;
-}
-
 Result
 doNFTOffers(Context const& context, bool sells)
 {
-    auto const v = getTokenid(context.params);
+    auto const v = getNFTID(context.params);
     if (auto const status = std::get_if<Status>(&v))
         return *status;
 

src/rpc/handlers/NoRippleCheck.cpp

@@ -1,3 +1,22 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
 #include <ripple/protocol/TxFlags.h>
 #include <rpc/RPCHelpers.h>
 
@@ -31,7 +50,8 @@ doNoRippleCheck(Context const& context)
         if (role == "gateway")
             roleGateway = true;
         else if (role != "user")
-            return Status{Error::rpcINVALID_PARAMS, "role field is invalid"};
+            return Status{
+                RippledError::rpcINVALID_PARAMS, "role field is invalid"};
     }
 
     std::uint32_t limit = 300;
@@ -100,7 +120,7 @@ doNoRippleCheck(Context const& context)
          &accountSeq,
          &limit,
          &accountID,
-         &problems](auto const& ownedItem) {
+         &problems](ripple::SLE&& ownedItem) {
             if (ownedItem.getType() == ripple::ltRIPPLE_STATE)
             {
                 bool const bLow = accountID ==

src/rpc/handlers/Random.cpp

@@ -1,8 +1,27 @@
-// rngfill.h doesn't compile without this include
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
 #include <cassert>
 
 #include <ripple/beast/utility/rngfill.h>
 #include <ripple/crypto/csprng.h>
+
 #include <rpc/RPCHelpers.h>
 
 namespace RPC {

src/rpc/handlers/ServerInfo.cpp

@@ -1,3 +1,21 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
 
 #include <backend/BackendInterface.h>
 #include <etl/ETLSource.h>
@@ -16,7 +34,7 @@ doServerInfo(Context const& context)
     if (!range)
     {
         return Status{
-            Error::rpcNOT_READY,
+            RippledError::rpcNOT_READY,
             "emptyDatabase",
             "The server has no data in the database"};
     }
@@ -27,7 +45,7 @@ doServerInfo(Context const& context)
     auto fees = context.backend->fetchFees(lgrInfo->seq, context.yield);
 
     if (!lgrInfo || !fees)
-        return Status{Error::rpcINTERNAL};
+        return Status{RippledError::rpcINTERNAL};
 
     auto age = std::chrono::duration_cast<std::chrono::seconds>(
                    std::chrono::system_clock::now().time_since_epoch())
@@ -47,8 +65,7 @@ doServerInfo(Context const& context)
 
     if (admin)
     {
-        info[JS(counters)] = boost::json::object{};
-        info[JS(counters)].as_object()[JS(rpc)] = context.counters.report();
+        info[JS(counters)] = context.counters.report();
         info[JS(counters)].as_object()["subscriptions"] =
             context.subscriptions->report();
     }

src/rpc/handlers/Subscribe.cpp

@@ -1,3 +1,22 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
 #include <boost/json.hpp>
 #include <subscriptions/SubscriptionManager.h>
 #include <webserver/WsBase.h>
@@ -12,22 +31,20 @@ static std::unordered_set<std::string> validCommonStreams{
     "transactions",
     "transactions_proposed",
     "validations",
-    "manifests"};
+    "manifests",
+    "book_changes"};
 
 Status
 validateStreams(boost::json::object const& request)
 {
-    boost::json::array const& streams = request.at(JS(streams)).as_array();
-
-    for (auto const& stream : streams)
+    for (auto const& streams = request.at(JS(streams)).as_array();
+         auto const& stream : streams)
     {
         if (!stream.is_string())
-            return Status{Error::rpcINVALID_PARAMS, "streamNotString"};
+            return Status{RippledError::rpcINVALID_PARAMS, "streamNotString"};
 
-        std::string s = stream.as_string().c_str();
-
-        if (validCommonStreams.find(s) == validCommonStreams.end())
-            return Status{Error::rpcINVALID_PARAMS, "invalidStream" + s};
+        if (!validCommonStreams.contains(stream.as_string().c_str()))
+            return Status{RippledError::rpcSTREAM_MALFORMED};
     }
 
     return OK;
@@ -57,6 +74,8 @@ subscribeToStreams(
             manager.subValidation(session);
         else if (s == "manifests")
             manager.subManifest(session);
+        else if (s == "book_changes")
+            manager.subBookChanges(session);
         else
             assert(false);
     }
@@ -85,6 +104,8 @@ unsubscribeToStreams(
             manager.unsubValidation(session);
         else if (s == "manifests")
             manager.unsubManifest(session);
+        else if (s == "book_changes")
+            manager.unsubBookChanges(session);
         else
             assert(false);
     }
@@ -96,13 +117,10 @@ validateAccounts(boost::json::array const& accounts)
     for (auto const& account : accounts)
     {
         if (!account.is_string())
-            return Status{Error::rpcINVALID_PARAMS, "accountNotString"};
+            return Status{RippledError::rpcINVALID_PARAMS, "accountNotString"};
 
-        std::string s = account.as_string().c_str();
-        auto id = accountFromStringStrict(s);
-
-        if (!id)
-            return Status{Error::rpcINVALID_PARAMS, "invalidAccount" + s};
+        if (!accountFromStringStrict(account.as_string().c_str()))
+            return Status{RippledError::rpcACT_MALFORMED, "Account malformed."};
     }
 
     return OK;
@@ -213,7 +231,7 @@ validateAndGetBooks(
     std::shared_ptr<Backend::BackendInterface const> const& backend)
 {
     if (!request.at(JS(books)).is_array())
-        return Status{Error::rpcINVALID_PARAMS, "booksNotArray"};
+        return Status{RippledError::rpcINVALID_PARAMS, "booksNotArray"};
     boost::json::array const& books = request.at(JS(books)).as_array();
 
     std::vector<ripple::Book> booksToSub;
@@ -292,16 +310,10 @@ doSubscribe(Context const& context)
 {
     auto request = context.params;
 
-    if (!request.contains(JS(streams)) && !request.contains(JS(accounts)) &&
-        !request.contains(JS(accounts_proposed)) &&
-        !request.contains(JS(books)))
-        return Status{
-            Error::rpcINVALID_PARAMS, "does not contain valid subscription"};
-
     if (request.contains(JS(streams)))
     {
         if (!request.at(JS(streams)).is_array())
-            return Status{Error::rpcINVALID_PARAMS, "streamsNotArray"};
+            return Status{RippledError::rpcINVALID_PARAMS, "streamsNotArray"};
 
         auto status = validateStreams(request);
 
@@ -311,31 +323,38 @@ doSubscribe(Context const& context)
 
     if (request.contains(JS(accounts)))
     {
-        if (!request.at(JS(accounts)).is_array())
-            return Status{Error::rpcINVALID_PARAMS, "accountsNotArray"};
+        auto const& jsonAccounts = request.at(JS(accounts));
+        if (!jsonAccounts.is_array())
+            return Status{RippledError::rpcINVALID_PARAMS, "accountsNotArray"};
 
-        boost::json::array accounts = request.at(JS(accounts)).as_array();
-        auto status = validateAccounts(accounts);
+        auto const& accounts = jsonAccounts.as_array();
+        if (accounts.empty())
+            return Status{RippledError::rpcACT_MALFORMED, "Account malformed."};
 
+        auto const status = validateAccounts(accounts);
         if (status)
             return status;
     }
 
     if (request.contains(JS(accounts_proposed)))
     {
-        if (!request.at(JS(accounts_proposed)).is_array())
-            return Status{Error::rpcINVALID_PARAMS, "accountsProposedNotArray"};
+        auto const& jsonAccounts = request.at(JS(accounts_proposed));
+        if (!jsonAccounts.is_array())
+            return Status{
+                RippledError::rpcINVALID_PARAMS, "accountsProposedNotArray"};
 
-        boost::json::array accounts =
-            request.at(JS(accounts_proposed)).as_array();
-        auto status = validateAccounts(accounts);
+        auto const& accounts = jsonAccounts.as_array();
+        if (accounts.empty())
+            return Status{RippledError::rpcACT_MALFORMED, "Account malformed."};
 
+        auto const status = validateAccounts(accounts);
         if (status)
             return status;
     }
 
     std::vector<ripple::Book> books;
-    boost::json::array snapshot;
+    boost::json::object response;
+
     if (request.contains(JS(books)))
     {
         auto parsed =
@@ -346,10 +365,9 @@ doSubscribe(Context const& context)
             std::get<std::pair<std::vector<ripple::Book>, boost::json::array>>(
                 parsed);
         books = std::move(bks);
-        snapshot = std::move(snap);
+        response[JS(offers)] = std::move(snap);
     }
 
-    boost::json::object response;
     if (request.contains(JS(streams)))
         response = subscribeToStreams(
             context.yield, request, context.session, *context.subscriptions);
@@ -364,8 +382,6 @@ doSubscribe(Context const& context)
     if (request.contains(JS(books)))
         subscribeToBooks(books, context.session, *context.subscriptions);
 
-    if (snapshot.size())
-        response[JS(offers)] = snapshot;
     return response;
 }
 
@@ -374,16 +390,10 @@ doUnsubscribe(Context const& context)
 {
     auto request = context.params;
 
-    if (!request.contains(JS(streams)) && !request.contains(JS(accounts)) &&
-        !request.contains(JS(accounts_proposed)) &&
-        !request.contains(JS(books)))
-        return Status{
-            Error::rpcINVALID_PARAMS, "does not contain valid subscription"};
-
     if (request.contains(JS(streams)))
     {
         if (!request.at(JS(streams)).is_array())
-            return Status{Error::rpcINVALID_PARAMS, "streamsNotArray"};
+            return Status{RippledError::rpcINVALID_PARAMS, "streamsNotArray"};
 
         auto status = validateStreams(request);
 
@@ -393,25 +403,31 @@ doUnsubscribe(Context const& context)
 
     if (request.contains(JS(accounts)))
     {
-        if (!request.at(JS(accounts)).is_array())
-            return Status{Error::rpcINVALID_PARAMS, "accountsNotArray"};
+        auto const& jsonAccounts = request.at(JS(accounts));
+        if (!jsonAccounts.is_array())
+            return Status{RippledError::rpcINVALID_PARAMS, "accountsNotArray"};
 
-        boost::json::array accounts = request.at(JS(accounts)).as_array();
-        auto status = validateAccounts(accounts);
+        auto const& accounts = jsonAccounts.as_array();
+        if (accounts.empty())
+            return Status{RippledError::rpcACT_MALFORMED, "Account malformed."};
 
+        auto const status = validateAccounts(accounts);
         if (status)
             return status;
     }
 
     if (request.contains(JS(accounts_proposed)))
     {
-        if (!request.at(JS(accounts_proposed)).is_array())
-            return Status{Error::rpcINVALID_PARAMS, "accountsProposedNotArray"};
+        auto const& jsonAccounts = request.at(JS(accounts_proposed));
+        if (!jsonAccounts.is_array())
+            return Status{
+                RippledError::rpcINVALID_PARAMS, "accountsProposedNotArray"};
 
-        boost::json::array accounts =
-            request.at(JS(accounts_proposed)).as_array();
-        auto status = validateAccounts(accounts);
+        auto const& accounts = jsonAccounts.as_array();
+        if (accounts.empty())
+            return Status{RippledError::rpcACT_MALFORMED, "Account malformed."};
 
+        auto const status = validateAccounts(accounts);
         if (status)
             return status;
     }
@@ -445,8 +461,7 @@ doUnsubscribe(Context const& context)
     if (request.contains("books"))
         unsubscribeToBooks(books, context.session, *context.subscriptions);
 
-    boost::json::object response = {{"status", "success"}};
-    return response;
+    return boost::json::object{};
 }
 
 }  // namespace RPC

src/rpc/handlers/TransactionEntry.cpp

@@ -1,3 +1,22 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
 #include <rpc/RPCHelpers.h>
 
 namespace RPC {
@@ -14,7 +33,7 @@ doTransactionEntry(Context const& context)
 
     ripple::uint256 hash;
     if (!hash.parseHex(getRequiredString(context.params, JS(tx_hash))))
-        return Status{Error::rpcINVALID_PARAMS, "malformedTransaction"};
+        return Status{RippledError::rpcINVALID_PARAMS, "malformedTransaction"};
 
     auto dbResponse = context.backend->fetchTransaction(hash, context.yield);
     // Note: transaction_entry is meant to only search a specified ledger for
@@ -28,7 +47,7 @@ doTransactionEntry(Context const& context)
     // is in a different ledger than the one specified.
     if (!dbResponse || dbResponse->ledgerSequence != lgrInfo.seq)
         return Status{
-            Error::rpcTXN_NOT_FOUND,
+            RippledError::rpcTXN_NOT_FOUND,
             "transactionNotFound",
             "Transaction not found."};
 

src/rpc/handlers/Tx.cpp

@@ -1,5 +1,23 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
 #include <backend/BackendInterface.h>
-#include <backend/Pg.h>
 #include <rpc/RPCHelpers.h>
 
 namespace RPC {
@@ -15,31 +33,52 @@ doTx(Context const& context)
     boost::json::object response = {};
 
     if (!request.contains(JS(transaction)))
-        return Status{Error::rpcINVALID_PARAMS, "specifyTransaction"};
+        return Status{RippledError::rpcINVALID_PARAMS, "specifyTransaction"};
 
     if (!request.at(JS(transaction)).is_string())
-        return Status{Error::rpcINVALID_PARAMS, "transactionNotString"};
+        return Status{RippledError::rpcINVALID_PARAMS, "transactionNotString"};
 
     ripple::uint256 hash;
     if (!hash.parseHex(request.at(JS(transaction)).as_string().c_str()))
-        return Status{Error::rpcINVALID_PARAMS, "malformedTransaction"};
+        return Status{RippledError::rpcINVALID_PARAMS, "malformedTransaction"};
 
     bool binary = false;
     if (request.contains(JS(binary)))
     {
         if (!request.at(JS(binary)).is_bool())
-            return Status{Error::rpcINVALID_PARAMS, "binaryFlagNotBool"};
+            return Status{RippledError::rpcINVALID_PARAMS, "binaryFlagNotBool"};
 
         binary = request.at(JS(binary)).as_bool();
     }
+    auto minLedger = getUInt(request, JS(min_ledger));
+    auto maxLedger = getUInt(request, JS(max_ledger));
+    bool rangeSupplied = minLedger && maxLedger;
+
+    if (rangeSupplied)
+    {
+        if (*minLedger > *maxLedger)
+            return Status{RippledError::rpcINVALID_LGR_RANGE};
+        if (*maxLedger - *minLedger > 1000)
+            return Status{RippledError::rpcEXCESSIVE_LGR_RANGE};
+    }
 
     auto range = context.backend->fetchLedgerRange();
     if (!range)
-        return Status{Error::rpcNOT_READY};
+        return Status{RippledError::rpcNOT_READY};
 
     auto dbResponse = context.backend->fetchTransaction(hash, context.yield);
     if (!dbResponse)
-        return Status{Error::rpcTXN_NOT_FOUND};
+    {
+        if (rangeSupplied)
+        {
+            bool searchedAll = range->maxSequence >= *maxLedger &&
+                range->minSequence <= *minLedger;
+            boost::json::object extra;
+            extra["searched_all"] = searchedAll;
+            return Status{RippledError::rpcTXN_NOT_FOUND, std::move(extra)};
+        }
+        return Status{RippledError::rpcTXN_NOT_FOUND};
+    }
 
     if (!binary)
     {

src/subscriptions/Message.h

@@ -1,5 +1,23 @@
-#ifndef CLIO_SUBSCRIPTION_MESSAGE_H
-#define CLIO_SUBSCRIPTION_MESSAGE_H
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
 
 #include <string>
 
@@ -36,5 +54,3 @@ public:
         return message_.size();
     }
 };
-
-#endif  // CLIO_SUBSCRIPTION_MESSAGE_H

src/subscriptions/SubscriptionManager.cpp

@@ -1,3 +1,22 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
 #include <rpc/RPCHelpers.h>
 #include <subscriptions/SubscriptionManager.h>
 #include <webserver/WsBase.h>
@@ -70,7 +89,7 @@ Subscription::unsubscribe(std::shared_ptr<WsBase> const& session)
 }
 
 void
-Subscription::publish(std::shared_ptr<Message>& message)
+Subscription::publish(std::shared_ptr<Message> const& message)
 {
     boost::asio::post(strand_, [this, message]() {
         sendToSubscribers(message, subscribers_, subCount_);
@@ -155,7 +174,9 @@ SubscriptionManager::subLedger(
     boost::asio::yield_context& yield,
     std::shared_ptr<WsBase> session)
 {
-    ledgerSubscribers_.subscribe(session);
+    subscribeHelper(session, ledgerSubscribers_, [this](session_ptr session) {
+        unsubLedger(session);
+    });
 
     auto ledgerRange = backend_->fetchLedgerRange();
     assert(ledgerRange);
@@ -185,7 +206,9 @@ SubscriptionManager::unsubLedger(std::shared_ptr<WsBase> session)
 void
 SubscriptionManager::subTransactions(std::shared_ptr<WsBase> session)
 {
-    txSubscribers_.subscribe(session);
+    subscribeHelper(session, txSubscribers_, [this](session_ptr session) {
+        unsubTransactions(session);
+    });
 }
 
 void
@@ -199,12 +222,13 @@ SubscriptionManager::subAccount(
     ripple::AccountID const& account,
     std::shared_ptr<WsBase>& session)
 {
-    accountSubscribers_.subscribe(session, account);
-
-    std::unique_lock lk(cleanupMtx_);
-    cleanupFuncs_[session].emplace_back([this, account](session_ptr session) {
-        unsubAccount(account, session);
-    });
+    subscribeHelper(
+        session,
+        account,
+        accountSubscribers_,
+        [this, account](session_ptr session) {
+            unsubAccount(account, session);
+        });
 }
 
 void
@@ -220,11 +244,10 @@ SubscriptionManager::subBook(
     ripple::Book const& book,
     std::shared_ptr<WsBase> session)
 {
-    bookSubscribers_.subscribe(session, book);
-
-    std::unique_lock lk(cleanupMtx_);
-    cleanupFuncs_[session].emplace_back(
-        [this, book](session_ptr session) { unsubBook(book, session); });
+    subscribeHelper(
+        session, book, bookSubscribers_, [this, book](session_ptr session) {
+            unsubBook(book, session);
+        });
 }
 
 void
@@ -235,6 +258,21 @@ SubscriptionManager::unsubBook(
     bookSubscribers_.unsubscribe(session, book);
 }
 
+void
+SubscriptionManager::subBookChanges(std::shared_ptr<WsBase> session)
+{
+    subscribeHelper(
+        session, bookChangesSubscribers_, [this](session_ptr session) {
+            unsubBookChanges(session);
+        });
+}
+
+void
+SubscriptionManager::unsubBookChanges(std::shared_ptr<WsBase> session)
+{
+    bookChangesSubscribers_.unsubscribe(session);
+}
+
 void
 SubscriptionManager::pubLedger(
     ripple::LedgerInfo const& lgrInfo,
@@ -345,6 +383,20 @@ SubscriptionManager::pubTransaction(
     }
 }
 
+void
+SubscriptionManager::pubBookChanges(
+    ripple::LedgerInfo const& lgrInfo,
+    std::vector<Backend::TransactionAndMetadata> const& transactions)
+{
+    if (bookChangesSubscribers_.empty())
+        return;
+
+    auto const json = RPC::computeBookChanges(lgrInfo, transactions);
+    auto const bookChangesMsg =
+        std::make_shared<Message>(boost::json::serialize(json));
+    bookChangesSubscribers_.publish(bookChangesMsg);
+}
+
 void
 SubscriptionManager::forwardProposedTransaction(
     boost::json::object const& response)
@@ -378,13 +430,21 @@ SubscriptionManager::subProposedAccount(
     ripple::AccountID const& account,
     std::shared_ptr<WsBase> session)
 {
-    accountProposedSubscribers_.subscribe(session, account);
+    subscribeHelper(
+        session,
+        account,
+        accountProposedSubscribers_,
+        [this, account](session_ptr session) {
+            unsubProposedAccount(account, session);
+        });
 }
 
 void
 SubscriptionManager::subManifest(std::shared_ptr<WsBase> session)
 {
-    manifestSubscribers_.subscribe(session);
+    subscribeHelper(session, manifestSubscribers_, [this](session_ptr session) {
+        unsubManifest(session);
+    });
 }
 
 void
@@ -396,7 +456,10 @@ SubscriptionManager::unsubManifest(std::shared_ptr<WsBase> session)
 void
 SubscriptionManager::subValidation(std::shared_ptr<WsBase> session)
 {
-    validationsSubscribers_.subscribe(session);
+    subscribeHelper(
+        session, validationsSubscribers_, [this](session_ptr session) {
+            unsubValidation(session);
+        });
 }
 
 void
@@ -416,7 +479,10 @@ SubscriptionManager::unsubProposedAccount(
 void
 SubscriptionManager::subProposedTransactions(std::shared_ptr<WsBase> session)
 {
-    txProposedSubscribers_.subscribe(session);
+    subscribeHelper(
+        session, txProposedSubscribers_, [this](session_ptr session) {
+            unsubProposedTransactions(session);
+        });
 }
 
 void
@@ -424,11 +490,33 @@ SubscriptionManager::unsubProposedTransactions(std::shared_ptr<WsBase> session)
 {
     txProposedSubscribers_.unsubscribe(session);
 }
+void
+SubscriptionManager::subscribeHelper(
+    std::shared_ptr<WsBase>& session,
+    Subscription& subs,
+    CleanupFunction&& func)
+{
+    subs.subscribe(session);
+    std::scoped_lock lk(cleanupMtx_);
+    cleanupFuncs_[session].push_back(std::move(func));
+}
+template <typename Key>
+void
+SubscriptionManager::subscribeHelper(
+    std::shared_ptr<WsBase>& session,
+    Key const& k,
+    SubscriptionMap<Key>& subs,
+    CleanupFunction&& func)
+{
+    subs.subscribe(session, k);
+    std::scoped_lock lk(cleanupMtx_);
+    cleanupFuncs_[session].push_back(std::move(func));
+}
 
 void
 SubscriptionManager::cleanup(std::shared_ptr<WsBase> session)
 {
-    std::unique_lock lk(cleanupMtx_);
+    std::scoped_lock lk(cleanupMtx_);
     if (!cleanupFuncs_.contains(session))
         return;
 

src/subscriptions/SubscriptionManager.h

@@ -1,10 +1,31 @@
-#ifndef SUBSCRIPTION_MANAGER_H
-#define SUBSCRIPTION_MANAGER_H
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
 
 #include <backend/BackendInterface.h>
-#include <memory>
+#include <config/Config.h>
+#include <log/Logger.h>
 #include <subscriptions/Message.h>
 
+#include <memory>
+
 class WsBase;
 
 class Subscription
@@ -31,13 +52,19 @@ public:
     unsubscribe(std::shared_ptr<WsBase> const& session);
 
     void
-    publish(std::shared_ptr<Message>& message);
+    publish(std::shared_ptr<Message> const& message);
 
     std::uint64_t
-    count()
+    count() const
     {
         return subCount_.load();
     }
+
+    bool
+    empty() const
+    {
+        return count() == 0;
+    }
 };
 
 template <class Key>
@@ -80,6 +107,7 @@ public:
 class SubscriptionManager
 {
     using session_ptr = std::shared_ptr<WsBase>;
+    clio::Logger log_{"Subscriptions"};
 
     std::vector<std::thread> workers_;
     boost::asio::io_context ioc_;
@@ -90,6 +118,7 @@ class SubscriptionManager
     Subscription txProposedSubscribers_;
     Subscription manifestSubscribers_;
     Subscription validationsSubscribers_;
+    Subscription bookChangesSubscribers_;
 
     SubscriptionMap<ripple::AccountID> accountSubscribers_;
     SubscriptionMap<ripple::AccountID> accountProposedSubscribers_;
@@ -100,17 +129,10 @@ class SubscriptionManager
 public:
     static std::shared_ptr<SubscriptionManager>
     make_SubscriptionManager(
-        boost::json::object const& config,
+        clio::Config const& config,
         std::shared_ptr<Backend::BackendInterface const> const& b)
     {
-        auto numThreads = 1;
-
-        if (config.contains("subscription_workers") &&
-            config.at("subscription_workers").is_int64())
-        {
-            numThreads = config.at("subscription_workers").as_int64();
-        }
-
+        auto numThreads = config.valueOr<uint64_t>("subscription_workers", 1);
         return std::make_shared<SubscriptionManager>(numThreads, b);
     }
 
@@ -122,6 +144,7 @@ public:
         , txProposedSubscribers_(ioc_)
         , manifestSubscribers_(ioc_)
         , validationsSubscribers_(ioc_)
+        , bookChangesSubscribers_(ioc_)
         , accountSubscribers_(ioc_)
         , accountProposedSubscribers_(ioc_)
         , bookSubscribers_(ioc_)
@@ -132,8 +155,8 @@ public:
         // We will eventually want to clamp this to be the number of strands,
         // since adding more threads than we have strands won't see any
         // performance benefits
-        BOOST_LOG_TRIVIAL(info) << "Starting subscription manager with "
-                                << numThreads << " workers";
+        log_.info() << "Starting subscription manager with " << numThreads
+                    << " workers";
 
         workers_.reserve(numThreads);
         for (auto i = numThreads; i > 0; --i)
@@ -159,6 +182,11 @@ public:
         std::string const& ledgerRange,
         std::uint32_t txnCount);
 
+    void
+    pubBookChanges(
+        ripple::LedgerInfo const& lgrInfo,
+        std::vector<Backend::TransactionAndMetadata> const& transactions);
+
     void
     unsubLedger(session_ptr session);
 
@@ -185,6 +213,12 @@ public:
     void
     unsubBook(ripple::Book const& book, session_ptr session);
 
+    void
+    subBookChanges(std::shared_ptr<WsBase> session);
+
+    void
+    unsubBookChanges(std::shared_ptr<WsBase> session);
+
     void
     subManifest(session_ptr session);
 
@@ -234,6 +268,7 @@ public:
         counts["account"] = accountSubscribers_.count();
         counts["accounts_proposed"] = accountProposedSubscribers_.count();
         counts["books"] = bookSubscribers_.count();
+        counts["book_changes"] = bookChangesSubscribers_.count();
 
         return counts;
     }
@@ -242,16 +277,29 @@ private:
     void
     sendAll(std::string const& pubMsg, std::unordered_set<session_ptr>& subs);
 
+    using CleanupFunction = std::function<void(session_ptr)>;
+
+    void
+    subscribeHelper(
+        std::shared_ptr<WsBase>& session,
+        Subscription& subs,
+        CleanupFunction&& func);
+
+    template <typename Key>
+    void
+    subscribeHelper(
+        std::shared_ptr<WsBase>& session,
+        Key const& k,
+        SubscriptionMap<Key>& subs,
+        CleanupFunction&& func);
+
     /**
      * This is how we chose to cleanup subscriptions that have been closed.
      * Each time we add a subscriber, we add the opposite lambda that
      * unsubscribes that subscriber when cleanup is called with the session that
      * closed.
      */
-    using CleanupFunction = std::function<void(session_ptr)>;
     std::mutex cleanupMtx_;
     std::unordered_map<session_ptr, std::vector<CleanupFunction>>
         cleanupFuncs_ = {};
 };
-
-#endif  // SUBSCRIPTION_MANAGER_H

src/util/Expected.h

@@ -0,0 +1,264 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2023, the clio developers.
+    Copyright (c) 2021 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+/*
+ * NOTE:
+ *
+ * This entire file is taken from rippled and modified slightly to fit this
+ * codebase as well as fixing the original issue that made this necessary.
+ *
+ * The reason is that currently there is no easy way to injest the fix that is
+ * required to make this implementation correctly work with boost::json::value.
+ * Since this will be replaced by `std::expected` as soon as possible there is
+ * not much harm done in doing it this way.
+ */
+
+#pragma once
+
+#include <ripple/basics/contract.h>
+#include <boost/outcome.hpp>
+#include <stdexcept>
+#include <type_traits>
+
+namespace util {
+
+/** Expected is an approximation of std::expected (hoped for in C++23)
+
+    See: http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2021/p0323r10.html
+
+    The implementation is entirely based on boost::outcome_v2::result.
+*/
+
+// Exception thrown by an invalid access to Expected.
+struct bad_expected_access : public std::runtime_error
+{
+    bad_expected_access() : runtime_error("bad expected access")
+    {
+    }
+};
+
+namespace detail {
+
+// Custom policy for Expected.  Always throw on an invalid access.
+struct throw_policy : public boost::outcome_v2::policy::base
+{
+    template <class Impl>
+    static constexpr void
+    wide_value_check(Impl&& self)
+    {
+        if (!base::_has_value(std::forward<Impl>(self)))
+            ripple::Throw<bad_expected_access>();
+    }
+
+    template <class Impl>
+    static constexpr void
+    wide_error_check(Impl&& self)
+    {
+        if (!base::_has_error(std::forward<Impl>(self)))
+            ripple::Throw<bad_expected_access>();
+    }
+
+    template <class Impl>
+    static constexpr void
+    wide_exception_check(Impl&& self)
+    {
+        if (!base::_has_exception(std::forward<Impl>(self)))
+            ripple::Throw<bad_expected_access>();
+    }
+};
+
+}  // namespace detail
+
+// Definition of Unexpected, which is used to construct the unexpected
+// return type of an Expected.
+template <class E>
+class Unexpected
+{
+public:
+    static_assert(!std::is_same<E, void>::value, "E must not be void");
+
+    Unexpected() = delete;
+
+    constexpr explicit Unexpected(E const& e) : val_(e)
+    {
+    }
+
+    constexpr explicit Unexpected(E&& e) : val_(std::move(e))
+    {
+    }
+
+    constexpr const E&
+    value() const&
+    {
+        return val_;
+    }
+
+    constexpr E&
+    value() &
+    {
+        return val_;
+    }
+
+    constexpr E&&
+    value() &&
+    {
+        return std::move(val_);
+    }
+
+    constexpr const E&&
+    value() const&&
+    {
+        return std::move(val_);
+    }
+
+private:
+    E val_;
+};
+
+// Unexpected deduction guide that converts array to const*.
+template <typename E, std::size_t N>
+Unexpected(E (&)[N]) -> Unexpected<E const*>;
+
+// Definition of Expected.  All of the machinery comes from boost::result.
+template <class T, class E>
+class [[nodiscard]] Expected
+    : private boost::outcome_v2::result<T, E, detail::throw_policy>
+{
+    using Base = boost::outcome_v2::result<T, E, detail::throw_policy>;
+
+public:
+    template <
+        typename U,
+        typename = std::enable_if_t<std::is_convertible_v<U, T>>>
+    constexpr Expected(U r) : Base(T(std::forward<U>(r)))
+    {
+    }
+
+    template <
+        typename U,
+        typename = std::enable_if_t<std::is_convertible_v<U, E>>>
+    constexpr Expected(Unexpected<U> e) : Base(E(std::forward<U>(e.value())))
+    {
+    }
+
+    constexpr bool
+    has_value() const
+    {
+        return Base::has_value();
+    }
+
+    constexpr T const&
+    value() const
+    {
+        return Base::value();
+    }
+
+    constexpr T&
+    value()
+    {
+        return Base::value();
+    }
+
+    constexpr E const&
+    error() const
+    {
+        return Base::error();
+    }
+
+    constexpr E&
+    error()
+    {
+        return Base::error();
+    }
+
+    constexpr explicit operator bool() const
+    {
+        return has_value();
+    }
+
+    // Add operator* and operator-> so the Expected API looks a bit more like
+    // what std::expected is likely to look like.  See:
+    // http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2021/p0323r10.html
+    [[nodiscard]] constexpr T&
+    operator*()
+    {
+        return this->value();
+    }
+
+    [[nodiscard]] constexpr T const&
+    operator*() const
+    {
+        return this->value();
+    }
+
+    [[nodiscard]] constexpr T*
+    operator->()
+    {
+        return &this->value();
+    }
+
+    [[nodiscard]] constexpr T const*
+    operator->() const
+    {
+        return &this->value();
+    }
+};
+
+// Specialization of Expected<void, E>.  Allows returning either success
+// (without a value) or the reason for the failure.
+template <class E>
+class [[nodiscard]] Expected<void, E>
+    : private boost::outcome_v2::result<void, E, detail::throw_policy>
+{
+    using Base = boost::outcome_v2::result<void, E, detail::throw_policy>;
+
+public:
+    // The default constructor makes a successful Expected<void, E>.
+    // This aligns with std::expected behavior proposed in P0323R10.
+    constexpr Expected() : Base(boost::outcome_v2::success())
+    {
+    }
+
+    template <
+        typename U,
+        typename = std::enable_if_t<std::is_convertible_v<U, E>>>
+    constexpr Expected(Unexpected<U> e) : Base(E(std::forward<U>(e.value())))
+    {
+    }
+
+    constexpr E const&
+    error() const
+    {
+        return Base::error();
+    }
+
+    constexpr E&
+    error()
+    {
+        return Base::error();
+    }
+
+    constexpr explicit operator bool() const
+    {
+        return Base::has_value();
+    }
+};
+
+}  // namespace util

src/util/Profiler.h

@@ -0,0 +1,59 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include <chrono>
+#include <type_traits>
+#include <utility>
+
+namespace util {
+
+/**
+ * @brief Profiler function to measure the time consuming
+ * @param func function object, can be a lamdba or function wrapper
+ * @return return a pair if function wrapper has return value: result of
+ * function wrapper and the elapsed time(ms) during executing the given
+ * function only return the elapsed time if function wrapper does not have
+ * return value
+ */
+template <typename U = std::chrono::milliseconds, typename F>
+[[nodiscard]] auto
+timed(F&& func)
+{
+    auto start = std::chrono::system_clock::now();
+
+    if constexpr (std::is_same_v<decltype(func()), void>)
+    {
+        func();
+        return std::chrono::duration_cast<U>(
+                   std::chrono::system_clock::now() - start)
+            .count();
+    }
+    else
+    {
+        auto ret = func();
+        auto elapsed = std::chrono::duration_cast<U>(
+                           std::chrono::system_clock::now() - start)
+                           .count();
+        return std::make_pair(ret, elapsed);
+    }
+}
+
+}  // namespace util