Upgrade xrpl to 2.2.0 (#1443)

Fixes #1156

.githooks/ensure_release_tag

@@ -1,20 +0,0 @@
-#!/bin/bash
-
-# Pushing a release branch requires an annotated tag at the released commit
-branch=$(git rev-parse --abbrev-ref HEAD)
-
-if [[ $branch =~ master ]]; then
-  # check if HEAD commit is tagged
-  if ! git describe --exact-match HEAD; then
-    echo "Commits to master must be tagged"
-    exit 1
-  fi
-elif [[ $branch =~ release/* ]]; then
-  IFS=/ read -r branch rel_ver <<< ${branch}
-  tag=$(git describe --tags --abbrev=0)
-  if [[ "${rel_ver}" != "${tag}" ]]; then
-    echo "release/${rel_ver} branches must have annotated tag ${rel_ver}"
-    echo "git tag -am\"${rel_ver}\" ${rel_ver}"
-    exit 1
-  fi
-fi

.githooks/pre-commit

@@ -17,12 +17,28 @@ if [[ "17.0.0" > "$version" ]]; then
             Please fix paths and run again.
 -----------------------------------------------------------------------------
 
+EOF
+    exit 3
+fi
+
+# check there is no .h headers, only .hpp
+wrong_headers=$(find $sources -name "*.h" | sed 's/^/        - /')
+if [[ ! -z "$wrong_headers" ]]; then
+    cat <<EOF
+
+                                    ERROR
+-----------------------------------------------------------------------------
+       Found .h headers in the source code. Please rename them to .hpp:
+
+$wrong_headers
+-----------------------------------------------------------------------------
+
 EOF
     exit 2
 fi
 
 function grep_code {
-    grep -l "${1}" ${sources} -r --include \*.h --include \*.cpp
+    grep -l "${1}" ${sources} -r --include \*.hpp --include \*.cpp
 }
 
 if [[ "$OSTYPE" == "darwin"* ]]; then
@@ -42,7 +58,7 @@ else
 fi
 
 first=$(git diff $sources)
-find $sources -type f \( -name '*.cpp' -o -name '*.h' -o -name '*.ipp' \) -print0 | xargs -0 $formatter
+find $sources -type f \( -name '*.cpp' -o -name '*.hpp' -o -name '*.ipp' \) -print0 | xargs -0 $formatter
 second=$(git diff $sources)
 changes=$(diff <(echo "$first") <(echo "$second") | wc -l | sed -e 's/^[[:space:]]*//')
 
@@ -59,4 +75,3 @@ EOF
     exit 1
 fi
 
-.githooks/ensure_release_tag

.github/actions/code_coverage/action.yml

@@ -6,7 +6,7 @@ runs:
     - name: Run tests
       shell: bash
       run: |
-        build/clio_tests --gtest_filter="-BackendCassandraBaseTest*:BackendCassandraTest*:BackendCassandraFactoryTestWithDB*"
+        build/clio_tests --backend_host=scylladb
 
     - name: Run gcovr
       shell: bash
@@ -21,9 +21,13 @@ runs:
         retention-days: 30
 
     - name: Upload coverage report
-      uses: codecov/codecov-action@v3
+      uses: wandalen/wretry.action@v1.3.0
       with:
-        files: build/coverage_report.xml
-        fail_ci_if_error: true
-        verbose: true
-        token: ${{ env.CODECOV_TOKEN }}
+        action: codecov/codecov-action@v3
+        with: |
+          files: build/coverage_report.xml 
+          fail_ci_if_error: true
+          verbose: true
+          token: ${{ env.CODECOV_TOKEN }}
+        attempt_limit: 5
+        attempt_delay: 10000

.github/workflows/build.yml

@@ -6,7 +6,6 @@ on:
     branches: [master, release/*, develop]
   workflow_dispatch:
 
- 
 jobs:
   lint:
     name: Check format
@@ -33,21 +32,30 @@ jobs:
               image: rippleci/clio_ci:latest
             build_type: Debug
             code_coverage: true
-          - os: macOS
-            build_type: Release
-            code_coverage: false
+          # - os: macOS
+          #   build_type: Release
+          #   code_coverage: false
     runs-on: [self-hosted, "${{ matrix.os }}"]
     container: ${{ matrix.container }}
 
+    services:
+      scylladb:
+        image: ${{ (matrix.code_coverage) && 'scylladb/scylla' || '' }}
+        options: >-
+          --health-cmd "cqlsh -e 'describe cluster'" 
+          --health-interval 10s 
+          --health-timeout 5s 
+          --health-retries 5
+
     steps:
       - uses: actions/checkout@v4
         with:
-            fetch-depth: 0
+          fetch-depth: 0
 
       - name: Prepare runner
         uses: ./.github/actions/prepare_runner
         with:
-            disable_ccache: false
+          disable_ccache: false
 
       - name: Setup conan
         uses: ./.github/actions/setup_conan
@@ -90,7 +98,7 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           name: clio_server_${{ runner.os }}_${{ matrix.build_type }}
-          path: build/clio_server      
+          path: build/clio_server
 
       - name: Upload clio_tests
         if: ${{ !matrix.code_coverage }}
@@ -119,7 +127,6 @@ jobs:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         uses: ./.github/actions/code_coverage
 
-
   test:
     name: Run Tests
     needs: build
@@ -130,7 +137,7 @@ jobs:
           - os: heavy
             container:
               image: rippleci/clio_ci:latest
-          - os: macOS
+          # - os: macOS
     runs-on: [self-hosted, "${{ matrix.os }}"]
     container: ${{ matrix.container }}
 

.github/workflows/clang-tidy.yml

@@ -1,13 +1,14 @@
 name: Clang-tidy check
 on:
   schedule:
-    - cron:  '0 6 * * 1-5'
+    - cron: "0 6 * * 1-5"
   workflow_dispatch:
   pull_request:
     branches: [develop]
     paths:
       - .clang_tidy
       - .github/workflows/clang-tidy.yml
+  workflow_call:
 
 jobs:
   clang_tidy:
@@ -22,12 +23,12 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-            fetch-depth: 0
+          fetch-depth: 0
 
       - name: Prepare runner
         uses: ./.github/actions/prepare_runner
         with:
-            disable_ccache: true
+          disable_ccache: true
 
       - name: Setup conan
         uses: ./.github/actions/setup_conan
@@ -58,6 +59,12 @@ jobs:
         run: |
           run-clang-tidy-17 -p build -j ${{ steps.number_of_threads.outputs.threads_number }} -fix -quiet 1>output.txt
 
+      - name: Run pre-commit hook
+        if: ${{ steps.run_clang_tidy.outcome != 'success' }}
+        continue-on-error: true
+        shell: bash
+        run: ./.githooks/pre-commit
+
       - name: Print issues found
         if: ${{ steps.run_clang_tidy.outcome != 'success' }}
         shell: bash
@@ -94,14 +101,14 @@ jobs:
           GH_REPO: ${{ github.repository }}
           GH_TOKEN: ${{ github.token }}
         with:
-          commit-message: '[CI] clang-tidy auto fixes'
+          commit-message: "[CI] clang-tidy auto fixes"
           committer: Clio CI <skuznetsov@ripple.com>
-          branch: 'clang_tidy/autofix'
+          branch: "clang_tidy/autofix"
           branch-suffix: timestamp
           delete-branch: true
-          title: '[CI] clang-tidy auto fixes'
-          body: 'Fixes #${{ steps.create_issue.outputs.created_issue }}. Please review and commit clang-tidy fixes.'
-          reviewers: 'cindyyan317,godexsoft,kuznetsss'
+          title: "[CI] clang-tidy auto fixes"
+          body: "Fixes #${{ steps.create_issue.outputs.created_issue }}. Please review and commit clang-tidy fixes."
+          reviewers: "cindyyan317,godexsoft,kuznetsss"
 
       - name: Fail the job
         if: ${{ steps.run_clang_tidy.outcome != 'success' }}

.github/workflows/clang-tidy_on_fix_merged.yml

@@ -0,0 +1,29 @@
+name: Restart clang-tidy workflow
+on:
+  push:
+    branches: [develop]
+  workflow_dispatch:
+
+jobs:
+  restart_clang_tidy:
+    runs-on: ubuntu-20.04
+
+    permissions:
+      actions: write
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Check last commit matches clang-tidy auto fixes
+        id: check
+        shell: bash
+        run: |
+          passed=$(if [[ $(git log -1 --pretty=format:%s | grep '\[CI\] clang-tidy auto fixes') ]]; then echo 'true' ; else echo 'false' ; fi)
+          echo "passed=$passed" >> $GITHUB_OUTPUT
+
+      - name: Run clang-tidy workflow
+        if: ${{ contains(steps.check.outputs.passed, 'true') }}
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          GH_REPO: ${{ github.repository }}
+        run: gh workflow run clang-tidy.yml

.github/workflows/nightly.yml

@@ -11,8 +11,8 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - os: macOS
-            build_type: Release
+          # - os: macOS
+          #   build_type: Release
           - os: heavy
             build_type: Release
             container:
@@ -76,8 +76,8 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - os: macOS
-            build_type: Release
+          # - os: macOS
+          #   build_type: Release
           - os: heavy
             build_type: Release
           - os: heavy

.gitignore

@@ -1,5 +1,6 @@
 *clio*.log
 /build*/
+.devcontainer
 .build
 .cache
 .vscode

CMake/Build.cpp.in

@@ -17,7 +17,7 @@
 */
 //==============================================================================
 
-#include "main/Build.h"
+#include "main/Build.hpp"
 
 #include <string>
 

CMake/Settings.cmake

@@ -17,6 +17,13 @@ set(COMPILER_FLAGS
   -pedantic
   -Wpedantic
   -Wunused
+   # FIXME: The following bunch are needed for gcc12 atm.
+   -Wno-missing-requires
+   -Wno-restrict
+   -Wno-null-dereference
+   -Wno-maybe-uninitialized
+   -Wno-unknown-warning-option # and this to work with clang
+   # TODO: Address these and others in https://github.com/XRPLF/clio/issues/1273
 )
 
 #TODO: reenable when we change CI #884

CMakeLists.txt

@@ -163,8 +163,14 @@ target_sources (clio PRIVATE
   src/util/prometheus/OStream.cpp
   src/util/prometheus/Prometheus.cpp
   src/util/Random.cpp
+  src/util/requests/RequestBuilder.cpp
+  src/util/requests/Types.cpp
+  src/util/requests/WsConnection.cpp
+  src/util/requests/impl/SslContext.cpp
   src/util/Taggable.cpp
   src/util/TerminationHandler.cpp
+  src/util/TxUtils.cpp
+  src/util/LedgerUtils.cpp
 )
 
 # Clio server
@@ -187,10 +193,15 @@ if (tests)
     unittests/ProfilerTests.cpp
     unittests/JsonUtilTests.cpp
     unittests/DOSGuardTests.cpp
+    unittests/util/TestGlobals.cpp
     unittests/util/AssertTests.cpp
     unittests/util/BatchingTests.cpp
+    unittests/util/TestHttpServer.cpp
     unittests/util/TestObject.cpp
+    unittests/util/TestWsServer.cpp
+    unittests/util/TxUtilTests.cpp
     unittests/util/StringUtils.cpp
+    unittests/util/LedgerUtilsTests.cpp
     unittests/util/prometheus/CounterTests.cpp
     unittests/util/prometheus/GaugeTests.cpp
     unittests/util/prometheus/HistogramTests.cpp
@@ -199,6 +210,9 @@ if (tests)
     unittests/util/prometheus/MetricBuilderTests.cpp
     unittests/util/prometheus/MetricsFamilyTests.cpp
     unittests/util/prometheus/OStreamTests.cpp
+    unittests/util/requests/RequestBuilderTests.cpp
+    unittests/util/requests/SslContextTests.cpp
+    unittests/util/requests/WsConnectionTests.cpp
     # ETL
     unittests/etl/ExtractionDataPipeTests.cpp
     unittests/etl/ExtractorTests.cpp
@@ -253,6 +267,7 @@ if (tests)
     unittests/rpc/handlers/AMMInfoTests.cpp
     # Backend
     unittests/data/BackendFactoryTests.cpp
+    unittests/data/BackendInterfaceTests.cpp
     unittests/data/BackendCountersTests.cpp
     unittests/data/cassandra/BaseTests.cpp
     unittests/data/cassandra/BackendTests.cpp

Doxyfile

@@ -1,6 +1,6 @@
 PROJECT_NAME           = "Clio"
 INPUT                  = ../src ../unittests
-EXCLUDE_PATTERNS       = *Test*.cpp *Test*.h
+EXCLUDE_PATTERNS       = *Test*.cpp *Test*.hpp
 RECURSIVE              = YES
 HAVE_DOT               = YES
 

README.md

@@ -86,6 +86,8 @@ conan remote add --insert 0 conan-non-prod http://18.143.149.228:8081/artifactor
 ```
 Now you should be able to download prebuilt `xrpl` package on some platforms.
 
+You might need to edit the `~/.conan/remotes.json` file to ensure that this newly added artifactory is listed last. Otherwise you might see compilation errors when building the project with gcc version 13 (or newer).
+
 2. Remove old packages you may have cached: 
 ```sh 
 conan remove -f xrpl

conanfile.py

@@ -23,10 +23,10 @@ class Clio(ConanFile):
         'boost/1.82.0',
         'cassandra-cpp-driver/2.17.0',
         'fmt/10.1.1',
-        'protobuf/3.21.12',
+        'protobuf/3.21.9',
         'grpc/1.50.1',
         'openssl/1.1.1u',
-        'xrpl/2.0.0',
+        'xrpl/2.2.0',
         'libbacktrace/cci.20210118'
     ]
 

metrics.py

@@ -1,183 +0,0 @@
-#!/usr/bin/python3
-import argparse
-
-from datetime import datetime
-
-def getTime(line):
-    bracketOpen = line.find("[")
-    bracketClose = line.find("]")
-    timestampSub = line[bracketOpen+1:bracketClose]
-    timestamp = datetime.strptime(timestampSub, '%Y-%m-%d %H:%M:%S.%f')
-    return timestamp.timestamp()
-
-def parseAccountTx(filename):
-
-
-    with open(filename) as f:
-        totalProcTime = 0.0
-        totalTxnTime = 0.0
-        numCalls = 0
-        for line in f:
-            if "executed stored_procedure" in line:
-                idx = line.find("in ")
-                idx = idx + 3
-                idx2 = line.find("num")
-                procTime = float(line[idx:idx2])
-                totalProcTime += procTime
-            if "fetchTransactions fetched" in line:
-                idx = line.find("took ")
-                idx = idx + 5
-                txnTime = float(line[idx:])
-                totalTxnTime += txnTime
-                numCalls = numCalls + 1
-        print(totalProcTime)
-        print(totalProcTime/numCalls)
-        print(totalTxnTime)
-        print(totalTxnTime/numCalls)
-            
-
-
-
-
-def parseLogs(filename, interval, minTxnCount = 0):
-
-    with open(filename) as f:
-
-        totalTime = 0
-        totalTxns = 0
-        totalObjs = 0
-        totalLoadTime = 0
-
-
-        start = 0
-        end = 0
-        totalLedgers = 0
-
-        intervalTime = 0
-        intervalTxns = 0
-        intervalObjs = 0
-        intervalLoadTime = 0
-
-        intervalStart = 0
-        intervalEnd = 0
-        intervalLedgers = 0
-        ledgersPerSecond = 0
-
-        print("ledgers, transactions, objects, loadTime, loadTime/ledger, ledgers/sec, txns/sec, objs/sec")
-        for line in f:
-            if "Load phase" in line:
-                sequenceIdx = line.find("Sequence : ")
-                hashIdx =  line.find(" Hash :")
-                sequence = line[sequenceIdx + len("Sequence : "):hashIdx]
-                txnCountSubstr = "txn count = "
-                objCountSubstr = ". object count = "
-                loadTimeSubstr = ". load time = "
-                txnsSubstr = ". load txns per second = "
-                objsSubstr = ". load objs per second = "
-                txnCountIdx = line.find(txnCountSubstr)
-                objCountIdx = line.find(objCountSubstr)
-                loadTimeIdx = line.find(loadTimeSubstr)
-                txnsIdx = line.find(txnsSubstr)
-                objsIdx = line.find(objsSubstr)
-                txnCount = line[txnCountIdx + len(txnCountSubstr):objCountIdx]
-                objCount = line[objCountIdx + len(objCountSubstr):loadTimeIdx]
-                loadTime = line[loadTimeIdx + len(loadTimeSubstr):txnsIdx]
-                txnsPerSecond = line[txnsIdx + len(txnsSubstr):objsIdx]
-                objsPerSecond = line[objsIdx + len(objsSubstr):-1]
-                if int(txnCount) >= minTxnCount:
-                    totalTime += float(loadTime);
-                    totalTxns += float(txnCount)
-                    totalObjs += float(objCount)
-                    intervalTime += float(loadTime)
-                    intervalTxns += float(txnCount)
-                    intervalObjs += float(objCount)
-
-                totalLoadTime += float(loadTime)
-                intervalLoadTime += float(loadTime)
-                
-
-                if start == 0:
-                    start = getTime(line)
-
-
-                prevEnd = end
-                end = getTime(line)
-
-                if intervalStart == 0:
-                    intervalStart = getTime(line)
-
-                intervalEnd = getTime(line)
-
-                totalLedgers+=1
-                intervalLedgers+=1
-                ledgersPerSecond = 0
-                if end != start:
-                    ledgersPerSecond = float(totalLedgers) / float((end - start))
-                intervalLedgersPerSecond = 0
-                if intervalEnd != intervalStart:
-                    intervalLedgersPerSecond = float(intervalLedgers) / float((intervalEnd - intervalStart))
-
-
-
-                if int(sequence) % interval == 0:
-
-                   # print("Sequence = " + sequence + " : [time, txCount, objCount, txPerSec, objsPerSec]")
-                   # print(loadTime + " , " 
-                   #     + txnCount + " , " 
-                   #     + objCount + " , " 
-                   #     + txnsPerSecond + " , " 
-                   #     + objsPerSecond)
-                   # print("Interval Aggregate ( " + str(interval) + " ) [ledgers, txns, objects, elapsedTime, ledgersPerSec, avgLoadTime, txPerSec, objsPerSec]: ")
-                    print(str(intervalLedgers) + " , " 
-                        + str(intervalTxns) + " , "
-                        + str(intervalObjs) + " , "
-                        + str(intervalLoadTime) + " , " 
-                        + str(intervalLoadTime/intervalLedgers) + " , " 
-                        + str(intervalLedgers/intervalLoadTime) + " , " 
-                        + str(intervalTxns/intervalLoadTime) + " , " 
-                        + str(intervalObjs/intervalLoadTime))
-                  #  print("Total Aggregate: [ledgers, txns, objects, elapsedTime, ledgersPerSec, avgLoadTime, txPerSec, objsPerSec]")
-                  #  print(str(totalLedgers) + " , " 
-                  #      + str(totalTxns) + " , "
-                  #      + str(totalObjs) + " , "
-                  #      + str(end-start) + " , " 
-                  #      + str(ledgersPerSecond) + " , " 
-                  #      + str(totalLoadTime/totalLedgers) + " , " 
-                  #      + str(totalTxns/totalTime) + " , " 
-                  #      + str(totalObjs/totalTime))
-                    if int(sequence) % interval == 0:
-                        intervalTime = 0
-                        intervalTxns = 0
-                        intervalObjs = 0
-                        intervalStart = 0
-                        intervalEnd = 0
-                        intervalLedgers = 0
-                        intervalLoadTime = 0
-        print("Total Aggregate: [ledgers, elapsedTime, ledgersPerSec, avgLoadTime, txPerSec, objsPerSec]")
-        print(totalLedgers)
-        print(totalLoadTime)
-        print(str(totalLedgers) + " : " 
-            + str(end-start) + " : " 
-            + str(ledgersPerSecond) + " : " 
-            + str(totalLoadTime/totalLedgers) + " : " 
-            + str(totalTxns/totalTime) + " : " 
-            + str(totalObjs/totalTime))
-
-
-    
-
-parser = argparse.ArgumentParser(description='parses logs')
-parser.add_argument("--filename")
-parser.add_argument("--interval",default=100000)
-parser.add_argument("--minTxnCount",default=0)
-parser.add_argument("--account_tx",default=False)
-
-args = parser.parse_args()
-
-def run(args):
-    if args.account_tx:
-        parseAccountTx(args.filename)
-    else:
-        parseLogs(args.filename, int(args.interval))
-
-run(args)

src/data/BackendCounters.cpp

@@ -17,11 +17,11 @@
 */
 //==============================================================================
 
-#include "data/BackendCounters.h"
+#include "data/BackendCounters.hpp"
 
-#include "util/Assert.h"
-#include "util/prometheus/Label.h"
-#include "util/prometheus/Prometheus.h"
+#include "util/Assert.hpp"
+#include "util/prometheus/Label.hpp"
+#include "util/prometheus/Prometheus.hpp"
 
 #include <boost/json/object.hpp>
 

src/data/BackendCounters.hpp

@@ -19,14 +19,17 @@
 
 #pragma once
 
-#include "util/prometheus/Prometheus.h"
+#include "util/prometheus/Counter.hpp"
+#include "util/prometheus/Gauge.hpp"
+#include "util/prometheus/Histogram.hpp"
 
 #include <boost/json/object.hpp>
 
-#include <atomic>
+#include <chrono>
+#include <cstdint>
 #include <functional>
 #include <memory>
-#include <utility>
+#include <string>
 
 namespace data {
 

src/data/BackendFactory.hpp

@@ -19,12 +19,18 @@
 
 #pragma once
 
-#include "data/BackendInterface.h"
-#include "data/CassandraBackend.h"
-#include "util/config/Config.h"
-#include "util/log/Logger.h"
+#include "data/BackendInterface.hpp"
+#include "data/CassandraBackend.hpp"
+#include "data/cassandra/SettingsProvider.hpp"
+#include "util/config/Config.hpp"
+#include "util/log/Logger.hpp"
 
 #include <boost/algorithm/string.hpp>
+#include <boost/algorithm/string/predicate.hpp>
+
+#include <memory>
+#include <stdexcept>
+#include <string>
 
 namespace data {
 

src/data/BackendInterface.cpp

@@ -17,11 +17,11 @@
 */
 //==============================================================================
 
-#include "data/BackendInterface.h"
+#include "data/BackendInterface.hpp"
 
-#include "data/Types.h"
-#include "util/Assert.h"
-#include "util/log/Logger.h"
+#include "data/Types.hpp"
+#include "util/Assert.hpp"
+#include "util/log/Logger.hpp"
 
 #include <boost/asio/spawn.hpp>
 #include <ripple/basics/base_uint.h>
@@ -344,14 +344,42 @@ BackendInterface::fetchFees(std::uint32_t const seq, boost::asio::yield_context
     ripple::SerialIter it(bytes->data(), bytes->size());
     ripple::SLE const sle{it, key};
 
-    if (sle.getFieldIndex(ripple::sfBaseFee) != -1)
-        fees.base = sle.getFieldU64(ripple::sfBaseFee);
+    // XRPFees amendment introduced new fields for fees calculations.
+    // New fields are set and the old fields are removed via `set_fees` tx.
+    // Fallback to old fields if `set_fees` was not yet used to update the fields on this tx.
+    auto hasNewFields = false;
+    {
+        auto const baseFeeXRP = sle.at(~ripple::sfBaseFeeDrops);
+        auto const reserveBaseXRP = sle.at(~ripple::sfReserveBaseDrops);
+        auto const reserveIncrementXRP = sle.at(~ripple::sfReserveIncrementDrops);
 
-    if (sle.getFieldIndex(ripple::sfReserveBase) != -1)
-        fees.reserve = sle.getFieldU32(ripple::sfReserveBase);
+        if (baseFeeXRP)
+            fees.base = baseFeeXRP->xrp();
 
-    if (sle.getFieldIndex(ripple::sfReserveIncrement) != -1)
-        fees.increment = sle.getFieldU32(ripple::sfReserveIncrement);
+        if (reserveBaseXRP)
+            fees.reserve = reserveBaseXRP->xrp();
+
+        if (reserveIncrementXRP)
+            fees.increment = reserveIncrementXRP->xrp();
+
+        hasNewFields = baseFeeXRP || reserveBaseXRP || reserveIncrementXRP;
+    }
+
+    if (not hasNewFields) {
+        // Fallback to old fields
+        auto const baseFee = sle.at(~ripple::sfBaseFee);
+        auto const reserveBase = sle.at(~ripple::sfReserveBase);
+        auto const reserveIncrement = sle.at(~ripple::sfReserveIncrement);
+
+        if (baseFee)
+            fees.base = baseFee.value();
+
+        if (reserveBase)
+            fees.reserve = reserveBase.value();
+
+        if (reserveIncrement)
+            fees.increment = reserveIncrement.value();
+    }
 
     return fees;
 }

src/data/BackendInterface.hpp

@@ -19,19 +19,32 @@
 
 #pragma once
 
-#include "data/DBHelpers.h"
-#include "data/LedgerCache.h"
-#include "data/Types.h"
-#include "util/config/Config.h"
-#include "util/log/Logger.h"
+#include "data/DBHelpers.hpp"
+#include "data/LedgerCache.hpp"
+#include "data/Types.hpp"
+#include "util/log/Logger.hpp"
 
+#include <boost/asio/executor_work_guard.hpp>
+#include <boost/asio/io_context.hpp>
 #include <boost/asio/spawn.hpp>
 #include <boost/json.hpp>
+#include <boost/json/object.hpp>
+#include <boost/utility/result_of.hpp>
+#include <ripple/basics/base_uint.h>
+#include <ripple/protocol/AccountID.h>
 #include <ripple/protocol/Fees.h>
 #include <ripple/protocol/LedgerHeader.h>
 
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <exception>
+#include <optional>
+#include <shared_mutex>
+#include <string>
 #include <thread>
 #include <type_traits>
+#include <vector>
 
 namespace data {
 

src/data/CassandraBackend.hpp

@@ -19,21 +19,43 @@
 
 #pragma once
 
-#include "data/BackendInterface.h"
-#include "data/cassandra/Concepts.h"
-#include "data/cassandra/Handle.h"
-#include "data/cassandra/Schema.h"
-#include "data/cassandra/SettingsProvider.h"
-#include "data/cassandra/impl/ExecutionStrategy.h"
-#include "util/Assert.h"
-#include "util/LedgerUtils.h"
-#include "util/Profiler.h"
-#include "util/log/Logger.h"
+#include "data/BackendInterface.hpp"
+#include "data/DBHelpers.hpp"
+#include "data/Types.hpp"
+#include "data/cassandra/Concepts.hpp"
+#include "data/cassandra/Handle.hpp"
+#include "data/cassandra/Schema.hpp"
+#include "data/cassandra/SettingsProvider.hpp"
+#include "data/cassandra/Types.hpp"
+#include "data/cassandra/impl/ExecutionStrategy.hpp"
+#include "util/Assert.hpp"
+#include "util/LedgerUtils.hpp"
+#include "util/Profiler.hpp"
+#include "util/log/Logger.hpp"
 
 #include <boost/asio/spawn.hpp>
+#include <boost/json/object.hpp>
+#include <cassandra.h>
+#include <ripple/basics/Blob.h>
+#include <ripple/basics/base_uint.h>
+#include <ripple/basics/strHex.h>
+#include <ripple/protocol/AccountID.h>
 #include <ripple/protocol/LedgerHeader.h>
 #include <ripple/protocol/nft.h>
 
+#include <atomic>
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <iterator>
+#include <limits>
+#include <optional>
+#include <stdexcept>
+#include <string>
+#include <tuple>
+#include <utility>
+#include <vector>
+
 namespace data::cassandra {
 
 /**

src/data/DBHelpers.hpp

@@ -20,16 +20,25 @@
 /** @file */
 #pragma once
 
-#include "data/Types.h"
-#include "util/Assert.h"
+#include "util/Assert.hpp"
 
 #include <boost/container/flat_set.hpp>
+#include <ripple/basics/Blob.h>
 #include <ripple/basics/Log.h>
 #include <ripple/basics/StringUtilities.h>
+#include <ripple/basics/base_uint.h>
+#include <ripple/protocol/AccountID.h>
 #include <ripple/protocol/SField.h>
 #include <ripple/protocol/STAccount.h>
+#include <ripple/protocol/STLedgerEntry.h>
+#include <ripple/protocol/Serializer.h>
 #include <ripple/protocol/TxMeta.h>
 
+#include <cstddef>
+#include <cstdint>
+#include <optional>
+#include <string>
+
 /**
  * @brief Struct used to keep track of what to write to account_transactions/account_tx tables.
  */

src/data/LedgerCache.cpp

@@ -17,10 +17,10 @@
 */
 //==============================================================================
 
-#include "data/LedgerCache.h"
+#include "data/LedgerCache.hpp"
 
-#include "data/Types.h"
-#include "util/Assert.h"
+#include "data/Types.hpp"
+#include "util/Assert.hpp"
 
 #include <ripple/basics/base_uint.h>
 
@@ -40,6 +40,16 @@ LedgerCache::latestLedgerSequence() const
     return latestSeq_;
 }
 
+void
+LedgerCache::waitUntilCacheContainsSeq(uint32_t seq)
+{
+    if (disabled_)
+        return;
+    std::unique_lock lock(mtx_);
+    cv_.wait(lock, [this, seq] { return latestSeq_ >= seq; });
+    return;
+}
+
 void
 LedgerCache::update(std::vector<LedgerObject> const& objs, uint32_t seq, bool isBackground)
 {
@@ -72,6 +82,7 @@ LedgerCache::update(std::vector<LedgerObject> const& objs, uint32_t seq, bool is
                     deletes_.insert(obj.key);
             }
         }
+        cv_.notify_all();
     }
 }
 

src/data/LedgerCache.hpp

@@ -19,16 +19,23 @@
 
 #pragma once
 
-#include "data/Types.h"
-#include "util/prometheus/Prometheus.h"
+#include "data/Types.hpp"
+#include "util/prometheus/Counter.hpp"
+#include "util/prometheus/Label.hpp"
+#include "util/prometheus/Prometheus.hpp"
 
 #include <ripple/basics/base_uint.h>
 #include <ripple/basics/hardened_hash.h>
 
+#include <atomic>
+#include <condition_variable>
+#include <cstddef>
+#include <cstdint>
+#include <functional>
 #include <map>
-#include <mutex>
+#include <optional>
 #include <shared_mutex>
-#include <utility>
+#include <unordered_set>
 #include <vector>
 
 namespace data {
@@ -67,6 +74,7 @@ class LedgerCache {
     std::map<ripple::uint256, CacheEntry> map_;
 
     mutable std::shared_mutex mtx_;
+    std::condition_variable_any cv_;
     uint32_t latestSeq_ = 0;
     std::atomic_bool full_ = false;
     std::atomic_bool disabled_ = false;
@@ -164,6 +172,9 @@ public:
      */
     float
     getSuccessorHitRate() const;
+
+    void
+    waitUntilCacheContainsSeq(uint32_t seq);
 };
 
 }  // namespace data

src/data/Types.hpp

@@ -22,7 +22,10 @@
 #include <ripple/basics/base_uint.h>
 #include <ripple/protocol/AccountID.h>
 
+#include <cstdint>
 #include <optional>
+#include <string_view>
+#include <tuple>
 #include <utility>
 #include <vector>
 

src/data/cassandra/Concepts.hpp

@@ -19,15 +19,20 @@
 
 #pragma once
 
-#include "data/cassandra/Types.h"
+#include "data/cassandra/Types.hpp"
 
+#include <boost/asio/io_context.hpp>
 #include <boost/asio/spawn.hpp>
 #include <boost/json.hpp>
+#include <boost/json/object.hpp>
 
 #include <chrono>
 #include <concepts>
+#include <cstdint>
 #include <optional>
 #include <string>
+#include <utility>
+#include <vector>
 
 namespace data::cassandra {
 

src/data/cassandra/Error.hpp

@@ -21,6 +21,8 @@
 
 #include <cassandra.h>
 
+#include <cstdint>
+#include <ostream>
 #include <string>
 #include <utility>
 

src/data/cassandra/Handle.cpp

@@ -17,9 +17,9 @@
 */
 //==============================================================================
 
-#include "data/cassandra/Handle.h"
+#include "data/cassandra/Handle.hpp"
 
-#include "data/cassandra/Types.h"
+#include "data/cassandra/Types.hpp"
 
 #include <cassandra.h>
 

src/data/cassandra/Handle.hpp

@@ -19,22 +19,20 @@
 
 #pragma once
 
-#include "data/cassandra/Error.h"
-#include "data/cassandra/Types.h"
-#include "data/cassandra/impl/Batch.h"
-#include "data/cassandra/impl/Cluster.h"
-#include "data/cassandra/impl/Future.h"
-#include "data/cassandra/impl/ManagedObject.h"
-#include "data/cassandra/impl/Result.h"
-#include "data/cassandra/impl/Session.h"
-#include "data/cassandra/impl/Statement.h"
-#include "util/Expected.h"
+#include "data/cassandra/Error.hpp"
+#include "data/cassandra/Types.hpp"
+#include "data/cassandra/impl/Batch.hpp"
+#include "data/cassandra/impl/Cluster.hpp"
+#include "data/cassandra/impl/Future.hpp"
+#include "data/cassandra/impl/ManagedObject.hpp"
+#include "data/cassandra/impl/Result.hpp"
+#include "data/cassandra/impl/Session.hpp"
+#include "data/cassandra/impl/Statement.hpp"
 
 #include <cassandra.h>
 
-#include <chrono>
-#include <compare>
-#include <iterator>
+#include <functional>
+#include <string_view>
 #include <vector>
 
 namespace data::cassandra {

src/data/cassandra/Schema.hpp

@@ -19,16 +19,19 @@
 
 #pragma once
 
-#include "data/cassandra/Concepts.h"
-#include "data/cassandra/Handle.h"
-#include "data/cassandra/SettingsProvider.h"
-#include "data/cassandra/Types.h"
-#include "util/Expected.h"
-#include "util/config/Config.h"
-#include "util/log/Logger.h"
+#include "data/cassandra/Concepts.hpp"
+#include "data/cassandra/Handle.hpp"
+#include "data/cassandra/Types.hpp"
+#include "util/log/Logger.hpp"
 
 #include <fmt/compile.h>
 
+#include <functional>
+#include <memory>
+#include <string>
+#include <string_view>
+#include <vector>
+
 namespace data::cassandra {
 
 template <SomeSettingsProvider SettingsProviderType>

src/data/cassandra/SettingsProvider.cpp

@@ -17,12 +17,12 @@
 */
 //==============================================================================
 
-#include "data/cassandra/SettingsProvider.h"
+#include "data/cassandra/SettingsProvider.hpp"
 
-#include "data/cassandra/Types.h"
-#include "data/cassandra/impl/Cluster.h"
-#include "util/Constants.h"
-#include "util/config/Config.h"
+#include "data/cassandra/Types.hpp"
+#include "data/cassandra/impl/Cluster.hpp"
+#include "util/Constants.hpp"
+#include "util/config/Config.hpp"
 
 #include <boost/json/conversion.hpp>
 #include <boost/json/value.hpp>

src/data/cassandra/SettingsProvider.hpp

@@ -19,11 +19,14 @@
 
 #pragma once
 
-#include "data/cassandra/Handle.h"
-#include "data/cassandra/Types.h"
-#include "util/Expected.h"
-#include "util/config/Config.h"
-#include "util/log/Logger.h"
+#include "data/cassandra/Handle.hpp"
+#include "data/cassandra/Types.hpp"
+#include "util/config/Config.hpp"
+#include "util/log/Logger.hpp"
+
+#include <cstdint>
+#include <optional>
+#include <string>
 
 namespace data::cassandra {
 

src/data/cassandra/Types.hpp

@@ -19,9 +19,9 @@
 
 #pragma once
 
-#include "util/Expected.h"
+#include "util/Expected.hpp"
 
-#include <string>
+#include <cstdint>
 
 namespace data::cassandra {
 

src/data/cassandra/impl/AsyncExecutor.hpp

@@ -19,17 +19,20 @@
 
 #pragma once
 
-#include "data/cassandra/Concepts.h"
-#include "data/cassandra/Handle.h"
-#include "data/cassandra/Types.h"
-#include "data/cassandra/impl/RetryPolicy.h"
-#include "util/Expected.h"
-#include "util/log/Logger.h"
+#include "data/cassandra/Concepts.hpp"
+#include "data/cassandra/Handle.hpp"
+#include "data/cassandra/Types.hpp"
+#include "data/cassandra/impl/RetryPolicy.hpp"
+#include "util/log/Logger.hpp"
 
 #include <boost/asio.hpp>
+#include <boost/asio/io_context.hpp>
 
 #include <functional>
 #include <memory>
+#include <mutex>
+#include <optional>
+#include <utility>
 
 namespace data::cassandra::detail {
 

src/data/cassandra/impl/Batch.cpp

@@ -17,13 +17,13 @@
 */
 //==============================================================================
 
-#include "data/cassandra/impl/Batch.h"
+#include "data/cassandra/impl/Batch.hpp"
 
-#include "data/cassandra/Error.h"
-#include "data/cassandra/Types.h"
-#include "data/cassandra/impl/ManagedObject.h"
-#include "data/cassandra/impl/Statement.h"
-#include "util/Expected.h"
+#include "data/cassandra/Error.hpp"
+#include "data/cassandra/Types.hpp"
+#include "data/cassandra/impl/ManagedObject.hpp"
+#include "data/cassandra/impl/Statement.hpp"
+#include "util/Expected.hpp"
 
 #include <cassandra.h>
 

src/data/cassandra/impl/Batch.hpp

@@ -19,11 +19,13 @@
 
 #pragma once
 
-#include "data/cassandra/Types.h"
-#include "data/cassandra/impl/ManagedObject.h"
+#include "data/cassandra/Types.hpp"
+#include "data/cassandra/impl/ManagedObject.hpp"
 
 #include <cassandra.h>
 
+#include <vector>
+
 namespace data::cassandra::detail {
 
 struct Batch : public ManagedObject<CassBatch> {

src/data/cassandra/impl/Cluster.cpp

@@ -17,11 +17,11 @@
 */
 //==============================================================================
 
-#include "data/cassandra/impl/Cluster.h"
+#include "data/cassandra/impl/Cluster.hpp"
 
-#include "data/cassandra/impl/ManagedObject.h"
-#include "data/cassandra/impl/SslContext.h"
-#include "util/log/Logger.h"
+#include "data/cassandra/impl/ManagedObject.hpp"
+#include "data/cassandra/impl/SslContext.hpp"
+#include "util/log/Logger.hpp"
 
 #include <cassandra.h>
 #include <fmt/core.h>

src/data/cassandra/impl/Cluster.hpp

@@ -19,8 +19,8 @@
 
 #pragma once
 
-#include "data/cassandra/impl/ManagedObject.h"
-#include "util/log/Logger.h"
+#include "data/cassandra/impl/ManagedObject.hpp"
+#include "util/log/Logger.hpp"
 
 #include <cassandra.h>
 

src/data/cassandra/impl/Collection.hpp

@@ -19,13 +19,16 @@
 
 #pragma once
 
-#include "data/cassandra/impl/ManagedObject.h"
+#include "data/cassandra/impl/ManagedObject.hpp"
 
 #include <cassandra.h>
 #include <ripple/basics/base_uint.h>
 
+#include <cstdint>
+#include <stdexcept>
 #include <string>
 #include <string_view>
+#include <vector>
 
 namespace data::cassandra::detail {
 

src/data/cassandra/impl/ExecutionStrategy.hpp

@@ -19,26 +19,35 @@
 
 #pragma once
 
-#include "data/BackendCounters.h"
-#include "data/BackendInterface.h"
-#include "data/cassandra/Handle.h"
-#include "data/cassandra/Types.h"
-#include "data/cassandra/impl/AsyncExecutor.h"
-#include "util/Assert.h"
-#include "util/Batching.h"
-#include "util/Expected.h"
-#include "util/log/Logger.h"
+#include "data/BackendCounters.hpp"
+#include "data/BackendInterface.hpp"
+#include "data/cassandra/Handle.hpp"
+#include "data/cassandra/Types.hpp"
+#include "data/cassandra/impl/AsyncExecutor.hpp"
+#include "util/Assert.hpp"
+#include "util/Batching.hpp"
+#include "util/log/Logger.hpp"
 
 #include <boost/asio.hpp>
+#include <boost/asio/associated_executor.hpp>
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/io_service.hpp>
 #include <boost/asio/spawn.hpp>
+#include <boost/json/object.hpp>
 
 #include <atomic>
+#include <chrono>
 #include <condition_variable>
+#include <cstddef>
+#include <cstdint>
 #include <functional>
 #include <memory>
 #include <mutex>
 #include <optional>
+#include <stdexcept>
 #include <thread>
+#include <type_traits>
+#include <vector>
 
 namespace data::cassandra::detail {
 

src/data/cassandra/impl/Future.cpp

@@ -17,12 +17,12 @@
 */
 //==============================================================================
 
-#include "data/cassandra/impl/Future.h"
+#include "data/cassandra/impl/Future.hpp"
 
-#include "data/cassandra/Error.h"
-#include "data/cassandra/Types.h"
-#include "data/cassandra/impl/ManagedObject.h"
-#include "data/cassandra/impl/Result.h"
+#include "data/cassandra/Error.hpp"
+#include "data/cassandra/Types.hpp"
+#include "data/cassandra/impl/ManagedObject.hpp"
+#include "data/cassandra/impl/Result.hpp"
 
 #include <cassandra.h>
 

src/data/cassandra/impl/Future.hpp

@@ -19,11 +19,14 @@
 
 #pragma once
 
-#include "data/cassandra/Types.h"
-#include "data/cassandra/impl/ManagedObject.h"
+#include "data/cassandra/Types.hpp"
+#include "data/cassandra/impl/ManagedObject.hpp"
 
 #include <cassandra.h>
 
+#include <functional>
+#include <memory>
+
 namespace data::cassandra::detail {
 
 struct Future : public ManagedObject<CassFuture> {

src/data/cassandra/impl/ManagedObject.hpp

@@ -20,6 +20,7 @@
 #pragma once
 
 #include <memory>
+#include <stdexcept>
 
 namespace data::cassandra::detail {
 

src/data/cassandra/impl/Result.cpp

@@ -17,9 +17,9 @@
 */
 //==============================================================================
 
-#include "data/cassandra/impl/Result.h"
+#include "data/cassandra/impl/Result.hpp"
 
-#include "data/cassandra/impl/ManagedObject.h"
+#include "data/cassandra/impl/ManagedObject.hpp"
 
 #include <cassandra.h>
 

src/data/cassandra/impl/Result.hpp

@@ -19,17 +19,25 @@
 
 #pragma once
 
-#include "data/cassandra/impl/ManagedObject.h"
-#include "data/cassandra/impl/Tuple.h"
-#include "util/Expected.h"
+#include "data/cassandra/impl/ManagedObject.hpp"
+#include "data/cassandra/impl/Tuple.hpp"
 
 #include <cassandra.h>
 #include <ripple/basics/base_uint.h>
 #include <ripple/protocol/AccountID.h>
 
-#include <compare>
+#include <cstddef>
+#include <cstdint>
+#include <functional>
 #include <iterator>
+#include <optional>
+#include <stdexcept>
+#include <string>
+#include <string_view>
 #include <tuple>
+#include <type_traits>
+#include <utility>
+#include <vector>
 
 namespace data::cassandra::detail {
 

src/data/cassandra/impl/RetryPolicy.hpp

@@ -19,10 +19,10 @@
 
 #pragma once
 
-#include "data/cassandra/Handle.h"
-#include "data/cassandra/Types.h"
-#include "util/Expected.h"
-#include "util/log/Logger.h"
+#include "data/cassandra/Handle.hpp"
+#include "data/cassandra/Types.hpp"
+#include "util/Expected.hpp"
+#include "util/log/Logger.hpp"
 
 #include <boost/asio.hpp>
 

src/data/cassandra/impl/Session.hpp

@@ -19,7 +19,7 @@
 
 #pragma once
 
-#include "data/cassandra/impl/ManagedObject.h"
+#include "data/cassandra/impl/ManagedObject.hpp"
 
 #include <cassandra.h>
 

src/data/cassandra/impl/SslContext.cpp

@@ -17,9 +17,9 @@
 */
 //==============================================================================
 
-#include "data/cassandra/impl/SslContext.h"
+#include "data/cassandra/impl/SslContext.hpp"
 
-#include "data/cassandra/impl/ManagedObject.h"
+#include "data/cassandra/impl/ManagedObject.hpp"
 
 #include <cassandra.h>
 

src/data/cassandra/impl/SslContext.hpp

@@ -19,7 +19,7 @@
 
 #pragma once
 
-#include "data/cassandra/impl/ManagedObject.h"
+#include "data/cassandra/impl/ManagedObject.hpp"
 
 #include <cassandra.h>
 

src/data/cassandra/impl/Statement.hpp

@@ -19,20 +19,25 @@
 
 #pragma once
 
-#include "data/cassandra/Types.h"
-#include "data/cassandra/impl/Collection.h"
-#include "data/cassandra/impl/ManagedObject.h"
-#include "data/cassandra/impl/Tuple.h"
-#include "util/Expected.h"
+#include "data/cassandra/Types.hpp"
+#include "data/cassandra/impl/Collection.hpp"
+#include "data/cassandra/impl/ManagedObject.hpp"
+#include "data/cassandra/impl/Tuple.hpp"
 
 #include <cassandra.h>
 #include <fmt/core.h>
 #include <ripple/basics/base_uint.h>
+#include <ripple/protocol/AccountID.h>
 #include <ripple/protocol/STAccount.h>
 
-#include <chrono>
-#include <compare>
-#include <iterator>
+#include <cstddef>
+#include <cstdint>
+#include <stdexcept>
+#include <string>
+#include <string_view>
+#include <tuple>
+#include <type_traits>
+#include <vector>
 
 namespace data::cassandra::detail {
 

src/data/cassandra/impl/Tuple.cpp

@@ -17,9 +17,9 @@
 */
 //==============================================================================
 
-#include "data/cassandra/impl/Tuple.h"
+#include "data/cassandra/impl/Tuple.hpp"
 
-#include "data/cassandra/impl/ManagedObject.h"
+#include "data/cassandra/impl/ManagedObject.hpp"
 
 #include <cassandra.h>
 

src/data/cassandra/impl/Tuple.hpp

@@ -19,15 +19,19 @@
 
 #pragma once
 
-#include "data/cassandra/impl/ManagedObject.h"
+#include "data/cassandra/impl/ManagedObject.hpp"
 
 #include <cassandra.h>
 #include <ripple/basics/base_uint.h>
 
+#include <cstddef>
+#include <cstdint>
 #include <functional>
+#include <stdexcept>
 #include <string>
 #include <string_view>
 #include <tuple>
+#include <type_traits>
 
 namespace data::cassandra::detail {
 

src/etl/ETLHelpers.hpp

@@ -20,15 +20,19 @@
 /** @file */
 #pragma once
 
-#include "util/Assert.h"
+#include "util/Assert.hpp"
 
 #include <ripple/basics/base_uint.h>
 
+#include <chrono>
 #include <condition_variable>
+#include <cstddef>
+#include <cstdint>
+#include <memory>
 #include <mutex>
 #include <optional>
 #include <queue>
-#include <sstream>
+#include <vector>
 
 namespace etl {
 /**

src/etl/ETLService.cpp

@@ -17,13 +17,13 @@
 */
 //==============================================================================
 
-#include "etl/ETLService.h"
+#include "etl/ETLService.hpp"
 
-#include "data/BackendInterface.h"
-#include "util/Assert.h"
-#include "util/Constants.h"
-#include "util/config/Config.h"
-#include "util/log/Logger.h"
+#include "data/BackendInterface.hpp"
+#include "util/Assert.hpp"
+#include "util/Constants.hpp"
+#include "util/config/Config.hpp"
+#include "util/log/Logger.hpp"
 
 #include <boost/asio/io_context.hpp>
 #include <ripple/beast/core/CurrentThreadName.h>
@@ -47,6 +47,10 @@ ETLService::runETLPipeline(uint32_t startSequence, uint32_t numExtractors)
     if (finishSequence_ && startSequence > *finishSequence_)
         return {};
 
+    LOG(log_.debug()) << "Wait for cache containing seq " << startSequence - 1
+                      << " current cache last seq =" << backend_->cache().latestLedgerSequence();
+    backend_->cache().waitUntilCacheContainsSeq(startSequence - 1);
+
     LOG(log_.debug()) << "Starting etl pipeline";
     state_.isWriting = true;
 

src/etl/ETLService.hpp

@@ -19,27 +19,37 @@
 
 #pragma once
 
-#include "data/BackendInterface.h"
-#include "data/LedgerCache.h"
-#include "etl/LoadBalancer.h"
-#include "etl/Source.h"
-#include "etl/SystemState.h"
-#include "etl/impl/AmendmentBlock.h"
-#include "etl/impl/CacheLoader.h"
-#include "etl/impl/ExtractionDataPipe.h"
-#include "etl/impl/Extractor.h"
-#include "etl/impl/LedgerFetcher.h"
-#include "etl/impl/LedgerLoader.h"
-#include "etl/impl/LedgerPublisher.h"
-#include "etl/impl/Transformer.h"
-#include "feed/SubscriptionManager.h"
-#include "util/log/Logger.h"
+#include "data/BackendInterface.hpp"
+#include "data/LedgerCache.hpp"
+#include "etl/ETLHelpers.hpp"
+#include "etl/ETLState.hpp"
+#include "etl/LoadBalancer.hpp"
+#include "etl/Source.hpp"
+#include "etl/SystemState.hpp"
+#include "etl/impl/AmendmentBlock.hpp"
+#include "etl/impl/CacheLoader.hpp"
+#include "etl/impl/ExtractionDataPipe.hpp"
+#include "etl/impl/Extractor.hpp"
+#include "etl/impl/LedgerFetcher.hpp"
+#include "etl/impl/LedgerLoader.hpp"
+#include "etl/impl/LedgerPublisher.hpp"
+#include "etl/impl/Transformer.hpp"
+#include "feed/SubscriptionManager.hpp"
+#include "util/log/Logger.hpp"
 
+#include <boost/asio/io_context.hpp>
 #include <boost/asio/steady_timer.hpp>
+#include <boost/json/object.hpp>
 #include <grpcpp/grpcpp.h>
+#include <org/xrpl/rpc/v1/get_ledger.pb.h>
 #include <ripple/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
 
+#include <cstddef>
+#include <cstdint>
 #include <memory>
+#include <optional>
+#include <string>
+#include <thread>
 
 struct AccountTransactionsData;
 struct NFTTransactionsData;

src/etl/ETLState.cpp

@@ -17,9 +17,9 @@
 */
 //==============================================================================
 
-#include "etl/ETLState.h"
+#include "etl/ETLState.hpp"
 
-#include "rpc/JS.h"
+#include "rpc/JS.hpp"
 
 #include <boost/json/conversion.hpp>
 #include <boost/json/value.hpp>

src/etl/ETLState.hpp

@@ -19,9 +19,12 @@
 
 #pragma once
 
-#include "data/BackendInterface.h"
+#include "data/BackendInterface.hpp"
 
 #include <boost/json.hpp>
+#include <boost/json/conversion.hpp>
+#include <boost/json/value.hpp>
+#include <boost/json/value_to.hpp>
 
 #include <cstdint>
 #include <optional>

src/etl/LoadBalancer.cpp

@@ -17,17 +17,17 @@
 */
 //==============================================================================
 
-#include "etl/LoadBalancer.h"
+#include "etl/LoadBalancer.hpp"
 
-#include "data/BackendInterface.h"
-#include "etl/ETLHelpers.h"
-#include "etl/ETLService.h"
-#include "etl/ETLState.h"
-#include "etl/ProbingSource.h"
-#include "etl/Source.h"
-#include "util/Assert.h"
-#include "util/Random.h"
-#include "util/log/Logger.h"
+#include "data/BackendInterface.hpp"
+#include "etl/ETLHelpers.hpp"
+#include "etl/ETLService.hpp"
+#include "etl/ETLState.hpp"
+#include "etl/ProbingSource.hpp"
+#include "etl/Source.hpp"
+#include "util/Assert.hpp"
+#include "util/Random.hpp"
+#include "util/log/Logger.hpp"
 
 #include <boost/asio/io_context.hpp>
 #include <boost/asio/spawn.hpp>

src/etl/LoadBalancer.hpp

@@ -19,17 +19,30 @@
 
 #pragma once
 
-#include "data/BackendInterface.h"
-#include "etl/ETLHelpers.h"
-#include "etl/ETLState.h"
-#include "feed/SubscriptionManager.h"
-#include "util/config/Config.h"
-#include "util/log/Logger.h"
+#include "data/BackendInterface.hpp"
+#include "etl/ETLHelpers.hpp"
+#include "etl/ETLState.hpp"
+#include "feed/SubscriptionManager.hpp"
+#include "util/config/Config.hpp"
+#include "util/log/Logger.hpp"
 
 #include <boost/asio.hpp>
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/spawn.hpp>
+#include <boost/json/object.hpp>
+#include <boost/json/value.hpp>
 #include <grpcpp/grpcpp.h>
+#include <org/xrpl/rpc/v1/get_ledger.pb.h>
+#include <org/xrpl/rpc/v1/ledger.pb.h>
 #include <ripple/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
 
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
+
 namespace etl {
 class Source;
 class ProbingSource;
@@ -99,26 +112,6 @@ public:
         std::shared_ptr<NetworkValidatedLedgers> validatedLedgers
     );
 
-    /**
-     * @brief A factory function for the ETL source.
-     *
-     * @param config The configuration to use
-     * @param ioc The io_context to run on
-     * @param backend BackendInterface implementation
-     * @param subscriptions Subscription manager
-     * @param validatedLedgers The network validated ledgers datastructure
-     * @param balancer The load balancer
-     */
-    static std::unique_ptr<Source>
-    make_Source(
-        util::Config const& config,
-        boost::asio::io_context& ioc,
-        std::shared_ptr<BackendInterface> backend,
-        std::shared_ptr<feed::SubscriptionManager> subscriptions,
-        std::shared_ptr<NetworkValidatedLedgers> validatedLedgers,
-        LoadBalancer& balancer
-    );
-
     ~LoadBalancer();
 
     /**
@@ -187,6 +180,26 @@ public:
     getETLState() noexcept;
 
 private:
+    /**
+     * @brief A factory function for the ETL source.
+     *
+     * @param config The configuration to use
+     * @param ioc The io_context to run on
+     * @param backend BackendInterface implementation
+     * @param subscriptions Subscription manager
+     * @param validatedLedgers The network validated ledgers datastructure
+     * @param balancer The load balancer
+     */
+    static std::unique_ptr<Source>
+    make_Source(
+        util::Config const& config,
+        boost::asio::io_context& ioc,
+        std::shared_ptr<BackendInterface> backend,
+        std::shared_ptr<feed::SubscriptionManager> subscriptions,
+        std::shared_ptr<NetworkValidatedLedgers> validatedLedgers,
+        LoadBalancer& balancer
+    );
+
     /**
      * @brief Execute a function on a randomly selected source.
      *

src/etl/NFTHelpers.cpp

@@ -17,7 +17,7 @@
 */
 //==============================================================================
 
-#include "data/DBHelpers.h"
+#include "data/DBHelpers.hpp"
 
 #include <fmt/core.h>
 #include <ripple/basics/base_uint.h>
@@ -358,4 +358,4 @@ getNFTDataFromObj(std::uint32_t const seq, std::string const& key, std::string c
 
     return nfts;
 }
-}  // namespace etl
+}  // namespace etl

src/etl/NFTHelpers.hpp

@@ -20,11 +20,17 @@
 /** @file */
 #pragma once
 
-#include "data/DBHelpers.h"
+#include "data/DBHelpers.hpp"
 
 #include <ripple/protocol/STTx.h>
 #include <ripple/protocol/TxMeta.h>
 
+#include <cstdint>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
+
 namespace etl {
 
 /**
@@ -48,4 +54,4 @@ getNFTDataFromTx(ripple::TxMeta const& txMeta, ripple::STTx const& sttx);
 std::vector<NFTsData>
 getNFTDataFromObj(std::uint32_t seq, std::string const& key, std::string const& blob);
 
-}  // namespace etl
+}  // namespace etl

src/etl/ProbingSource.cpp

@@ -17,15 +17,15 @@
 */
 //==============================================================================
 
-#include "etl/ProbingSource.h"
+#include "etl/ProbingSource.hpp"
 
-#include "data/BackendInterface.h"
-#include "etl/ETLHelpers.h"
-#include "etl/LoadBalancer.h"
-#include "etl/Source.h"
-#include "feed/SubscriptionManager.h"
-#include "util/config/Config.h"
-#include "util/log/Logger.h"
+#include "data/BackendInterface.hpp"
+#include "etl/ETLHelpers.hpp"
+#include "etl/LoadBalancer.hpp"
+#include "etl/Source.hpp"
+#include "feed/SubscriptionManager.hpp"
+#include "util/config/Config.hpp"
+#include "util/log/Logger.hpp"
 
 #include <boost/asio/io_context.hpp>
 #include <boost/asio/spawn.hpp>

src/etl/ProbingSource.hpp

@@ -19,17 +19,33 @@
 
 #pragma once
 
-#include "etl/Source.h"
-#include "util/config/Config.h"
-#include "util/log/Logger.h"
+#include "data/BackendInterface.hpp"
+#include "etl/ETLHelpers.hpp"
+#include "etl/LoadBalancer.hpp"
+#include "etl/Source.hpp"
+#include "util/config/Config.hpp"
+#include "util/log/Logger.hpp"
 
 #include <boost/asio.hpp>
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/spawn.hpp>
+#include <boost/asio/ssl/context.hpp>
 #include <boost/beast/core.hpp>
 #include <boost/beast/core/string.hpp>
 #include <boost/beast/ssl.hpp>
 #include <boost/beast/websocket.hpp>
+#include <boost/json/object.hpp>
+#include <boost/uuid/uuid.hpp>
+#include <grpcpp/support/status.h>
+#include <org/xrpl/rpc/v1/get_ledger.pb.h>
 
+#include <cstdint>
+#include <memory>
 #include <mutex>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
 
 namespace etl {
 

src/etl/Source.cpp

@@ -17,9 +17,9 @@
 */
 //==============================================================================
 
-#include "etl/Source.h"
+#include "etl/Source.hpp"
 
-#include "util/log/Logger.h"
+#include "util/log/Logger.hpp"
 
 #include <boost/asio/ip/tcp.hpp>
 #include <boost/asio/post.hpp>

src/etl/Source.hpp

@@ -19,28 +19,77 @@
 
 #pragma once
 
-#include "data/BackendInterface.h"
-#include "etl/ETLHelpers.h"
-#include "etl/LoadBalancer.h"
-#include "etl/impl/AsyncData.h"
-#include "etl/impl/ForwardCache.h"
-#include "feed/SubscriptionManager.h"
-#include "util/Assert.h"
-#include "util/config/Config.h"
-#include "util/log/Logger.h"
+#include "data/BackendInterface.hpp"
+#include "etl/ETLHelpers.hpp"
+#include "etl/LoadBalancer.hpp"
+#include "etl/impl/AsyncData.hpp"
+#include "etl/impl/ForwardCache.hpp"
+#include "feed/SubscriptionManager.hpp"
+#include "util/Assert.hpp"
+#include "util/config/Config.hpp"
+#include "util/log/Logger.hpp"
 
 #include <boost/algorithm/string.hpp>
+#include <boost/algorithm/string/classification.hpp>
+#include <boost/algorithm/string/split.hpp>
 #include <boost/asio.hpp>
+#include <boost/asio/associated_executor.hpp>
+#include <boost/asio/buffer.hpp>
+#include <boost/asio/error.hpp>
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/ip/address.hpp>
+#include <boost/asio/ip/tcp.hpp>
+#include <boost/asio/spawn.hpp>
+#include <boost/asio/ssl/context.hpp>
+#include <boost/asio/ssl/error.hpp>
+#include <boost/asio/steady_timer.hpp>
+#include <boost/asio/strand.hpp>
 #include <boost/beast/core.hpp>
+#include <boost/beast/core/buffers_to_string.hpp>
+#include <boost/beast/core/error.hpp>
+#include <boost/beast/core/flat_buffer.hpp>
 #include <boost/beast/core/string.hpp>
+#include <boost/beast/core/tcp_stream.hpp>
+#include <boost/beast/http/field.hpp>
 #include <boost/beast/ssl.hpp>
+#include <boost/beast/ssl/ssl_stream.hpp>
+#include <boost/beast/version.hpp>
 #include <boost/beast/websocket.hpp>
+#include <boost/beast/websocket/rfc6455.hpp>
+#include <boost/beast/websocket/stream.hpp>
+#include <boost/beast/websocket/stream_base.hpp>
+#include <boost/json/object.hpp>
+#include <boost/json/parse.hpp>
+#include <boost/json/serialize.hpp>
+#include <boost/lexical_cast.hpp>
+#include <boost/uuid/random_generator.hpp>
 #include <boost/uuid/uuid.hpp>
 #include <boost/uuid/uuid_generators.hpp>
+#include <grpcpp/client_context.h>
 #include <grpcpp/grpcpp.h>
+#include <grpcpp/security/credentials.h>
+#include <grpcpp/support/channel_arguments.h>
+#include <grpcpp/support/status.h>
+#include <openssl/err.h>
+#include <org/xrpl/rpc/v1/get_ledger.pb.h>
+#include <ripple/basics/base_uint.h>
 #include <ripple/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
 
+#include <algorithm>
+#include <atomic>
+#include <chrono>
+#include <cmath>
+#include <cstddef>
+#include <cstdint>
+#include <exception>
+#include <functional>
+#include <memory>
+#include <mutex>
+#include <optional>
+#include <sstream>
+#include <string>
 #include <utility>
+#include <vector>
 
 namespace feed {
 class SubscriptionManager;
@@ -314,7 +363,7 @@ public:
         boost::json::object response;
 
         namespace beast = boost::beast;
-        namespace http = boost::beast::http;
+        namespace http = beast::http;
         namespace websocket = beast::websocket;
         namespace net = boost::asio;
         using tcp = boost::asio::ip::tcp;

src/etl/impl/AmendmentBlock.hpp

@@ -19,8 +19,8 @@
 
 #pragma once
 
-#include "etl/SystemState.h"
-#include "util/log/Logger.h"
+#include "etl/SystemState.hpp"
+#include "util/log/Logger.hpp"
 
 #include <boost/asio/io_context.hpp>
 #include <boost/asio/post.hpp>

src/etl/impl/AsyncData.hpp

@@ -19,14 +19,27 @@
 
 #pragma once
 
-#include "data/BackendInterface.h"
-#include "etl/NFTHelpers.h"
-#include "util/Assert.h"
-#include "util/log/Logger.h"
+#include "data/BackendInterface.hpp"
+#include "data/Types.hpp"
+#include "etl/NFTHelpers.hpp"
+#include "util/Assert.hpp"
+#include "util/log/Logger.hpp"
 
+#include <grpcpp/client_context.h>
 #include <grpcpp/grpcpp.h>
+#include <grpcpp/support/status.h>
+#include <org/xrpl/rpc/v1/get_ledger_data.pb.h>
+#include <ripple/basics/base_uint.h>
+#include <ripple/basics/strHex.h>
 #include <ripple/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
 
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
+
 namespace etl::detail {
 
 class AsyncCallData {

src/etl/impl/CacheLoader.hpp

@@ -19,20 +19,48 @@
 
 #pragma once
 
-#include "data/BackendInterface.h"
-#include "util/log/Logger.h"
+#include "data/BackendInterface.hpp"
+#include "data/Types.hpp"
+#include "util/Assert.hpp"
+#include "util/log/Logger.hpp"
 
+#include <boost/algorithm/hex.hpp>
 #include <boost/algorithm/string.hpp>
+#include <boost/algorithm/string/predicate.hpp>
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/ip/tcp.hpp>
 #include <boost/asio/spawn.hpp>
 #include <boost/beast/core.hpp>
+#include <boost/beast/core/buffers_to_string.hpp>
+#include <boost/beast/core/error.hpp>
+#include <boost/beast/core/flat_buffer.hpp>
 #include <boost/beast/core/string.hpp>
+#include <boost/beast/core/tcp_stream.hpp>
 #include <boost/beast/websocket.hpp>
+#include <boost/beast/websocket/stream.hpp>
+#include <boost/json/object.hpp>
+#include <boost/json/parse.hpp>
+#include <boost/json/value.hpp>
 #include <grpcpp/grpcpp.h>
+#include <ripple/basics/base_uint.h>
+#include <ripple/basics/strHex.h>
 #include <ripple/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
 
+#include <atomic>
 #include <chrono>
-#include <mutex>
+#include <cstddef>
+#include <cstdint>
+#include <exception>
+#include <functional>
+#include <iterator>
+#include <memory>
+#include <optional>
+#include <random>
+#include <sstream>
+#include <string>
 #include <thread>
+#include <utility>
+#include <vector>
 
 namespace etl::detail {
 

src/etl/impl/ExtractionDataPipe.hpp

@@ -19,10 +19,13 @@
 
 #pragma once
 
-#include "etl/ETLHelpers.h"
-#include "util/log/Logger.h"
+#include "etl/ETLHelpers.hpp"
+#include "util/log/Logger.hpp"
 
+#include <cstddef>
+#include <cstdint>
 #include <memory>
+#include <optional>
 #include <vector>
 
 namespace etl::detail {

src/etl/impl/Extractor.hpp

@@ -19,15 +19,18 @@
 
 #pragma once
 
-#include "etl/SystemState.h"
-#include "util/Assert.h"
-#include "util/Profiler.h"
-#include "util/log/Logger.h"
+#include "etl/SystemState.hpp"
+#include "util/Assert.hpp"
+#include "util/Profiler.hpp"
+#include "util/log/Logger.hpp"
 
 #include <ripple/beast/core/CurrentThreadName.h>
 
 #include <chrono>
-#include <mutex>
+#include <cstdint>
+#include <functional>
+#include <memory>
+#include <optional>
 #include <thread>
 #include <utility>
 

src/etl/impl/ForwardCache.cpp

@@ -17,11 +17,11 @@
 */
 //==============================================================================
 
-#include "etl/impl/ForwardCache.h"
+#include "etl/impl/ForwardCache.hpp"
 
-#include "etl/Source.h"
-#include "rpc/RPCHelpers.h"
-#include "util/log/Logger.h"
+#include "etl/Source.hpp"
+#include "rpc/RPCHelpers.hpp"
+#include "util/log/Logger.hpp"
 
 #include <boost/asio/spawn.hpp>
 #include <boost/json/object.hpp>

src/etl/impl/ForwardCache.hpp

@@ -19,16 +19,19 @@
 
 #pragma once
 
-#include "data/BackendInterface.h"
-#include "etl/ETLHelpers.h"
-#include "util/config/Config.h"
-#include "util/log/Logger.h"
+#include "util/config/Config.hpp"
+#include "util/log/Logger.hpp"
 
 #include <boost/asio.hpp>
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/strand.hpp>
 #include <boost/json.hpp>
+#include <boost/json/object.hpp>
 
-#include <atomic>
-#include <mutex>
+#include <cstdint>
+#include <optional>
+#include <shared_mutex>
+#include <string>
 #include <unordered_map>
 
 namespace etl {

src/etl/impl/LedgerFetcher.hpp

@@ -19,14 +19,14 @@
 
 #pragma once
 
-#include "data/BackendInterface.h"
-#include "etl/Source.h"
-#include "util/log/Logger.h"
+#include "data/BackendInterface.hpp"
+#include "util/log/Logger.hpp"
 
 #include <grpcpp/grpcpp.h>
 #include <ripple/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
 
-#include <optional>
+#include <cstdint>
+#include <memory>
 #include <utility>
 
 namespace etl::detail {
@@ -89,9 +89,15 @@ public:
     {
         LOG(log_.debug()) << "Attempting to fetch ledger with sequence = " << sequence;
 
-        auto response = loadBalancer_->fetchLedger(
-            sequence, true, !backend_->cache().isFull() || backend_->cache().latestLedgerSequence() >= sequence
-        );
+        auto const isCacheFull = backend_->cache().isFull();
+        auto const isLedgerCached = backend_->cache().latestLedgerSequence() >= sequence;
+        if (isLedgerCached) {
+            LOG(log_.info()) << sequence << " is already cached, the current latest seq in cache is "
+                             << backend_->cache().latestLedgerSequence() << " and the cache is "
+                             << (isCacheFull ? "full" : "not full");
+        }
+
+        auto response = loadBalancer_->fetchLedger(sequence, true, !isCacheFull || isLedgerCached);
         if (response)
             LOG(log_.trace()) << "GetLedger reply = " << response->DebugString();
 

src/etl/impl/LedgerLoader.hpp

@@ -19,19 +19,34 @@
 
 #pragma once
 
-#include "data/BackendInterface.h"
-#include "etl/NFTHelpers.h"
-#include "etl/SystemState.h"
-#include "etl/impl/LedgerFetcher.h"
-#include "util/Assert.h"
-#include "util/LedgerUtils.h"
-#include "util/Profiler.h"
-#include "util/log/Logger.h"
+#include "data/BackendInterface.hpp"
+#include "data/DBHelpers.hpp"
+#include "data/Types.hpp"
+#include "etl/NFTHelpers.hpp"
+#include "etl/SystemState.hpp"
+#include "etl/impl/LedgerFetcher.hpp"
+#include "util/Assert.hpp"
+#include "util/LedgerUtils.hpp"
+#include "util/Profiler.hpp"
+#include "util/log/Logger.hpp"
 
+#include <ripple/basics/base_uint.h>
+#include <ripple/basics/strHex.h>
 #include <ripple/beast/core/CurrentThreadName.h>
+#include <ripple/protocol/LedgerHeader.h>
+#include <ripple/protocol/STTx.h>
+#include <ripple/protocol/Serializer.h>
+#include <ripple/protocol/TxMeta.h>
 
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <functional>
 #include <memory>
+#include <optional>
+#include <string>
 #include <utility>
+#include <vector>
 
 /**
  * @brief Account transactions, NFT transactions and NFT data bundled togeher.
@@ -206,7 +221,7 @@ public:
                         if (isBookDir(cur->key, cur->blob)) {
                             auto base = getBookBase(cur->key);
                             // make sure the base is not an actual object
-                            if (!backend_->cache().get(cur->key, sequence)) {
+                            if (!backend_->cache().get(base, sequence)) {
                                 auto succ = backend_->cache().getSuccessor(base, sequence);
                                 ASSERT(succ.has_value(), "Book base {} must have a successor", ripple::strHex(base));
                                 if (succ->key == cur->key) {

src/etl/impl/LedgerPublisher.hpp

@@ -19,17 +19,34 @@
 
 #pragma once
 
-#include "data/BackendInterface.h"
-#include "etl/SystemState.h"
-#include "feed/SubscriptionManager.h"
-#include "util/Assert.h"
-#include "util/LedgerUtils.h"
-#include "util/log/Logger.h"
+#include "data/BackendInterface.hpp"
+#include "data/DBHelpers.hpp"
+#include "data/Types.hpp"
+#include "etl/SystemState.hpp"
+#include "util/Assert.hpp"
+#include "util/log/Logger.hpp"
 
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/strand.hpp>
+#include <ripple/basics/chrono.h>
+#include <ripple/protocol/Fees.h>
 #include <ripple/protocol/LedgerHeader.h>
+#include <ripple/protocol/SField.h>
+#include <ripple/protocol/STObject.h>
+#include <ripple/protocol/Serializer.h>
 
 #include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <functional>
+#include <memory>
+#include <mutex>
+#include <optional>
+#include <shared_mutex>
+#include <string>
+#include <thread>
 #include <utility>
+#include <vector>
 
 namespace etl::detail {
 

src/etl/impl/Transformer.hpp

@@ -19,23 +19,35 @@
 
 #pragma once
 
-#include "data/BackendInterface.h"
-#include "etl/SystemState.h"
-#include "etl/impl/AmendmentBlock.h"
-#include "etl/impl/LedgerLoader.h"
-#include "util/Assert.h"
-#include "util/LedgerUtils.h"
-#include "util/Profiler.h"
-#include "util/log/Logger.h"
+#include "data/BackendInterface.hpp"
+#include "data/DBHelpers.hpp"
+#include "data/Types.hpp"
+#include "etl/SystemState.hpp"
+#include "etl/impl/AmendmentBlock.hpp"
+#include "etl/impl/LedgerLoader.hpp"
+#include "util/Assert.hpp"
+#include "util/LedgerUtils.hpp"
+#include "util/Profiler.hpp"
+#include "util/log/Logger.hpp"
 
 #include <grpcpp/grpcpp.h>
+#include <ripple/basics/base_uint.h>
+#include <ripple/basics/strHex.h>
 #include <ripple/beast/core/CurrentThreadName.h>
 #include <ripple/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
+#include <ripple/protocol/LedgerHeader.h>
 
 #include <chrono>
+#include <cstdint>
+#include <functional>
 #include <memory>
+#include <optional>
+#include <set>
+#include <stdexcept>
+#include <string>
 #include <thread>
 #include <utility>
+#include <vector>
 
 namespace etl::detail {
 

src/feed/SubscriptionManager.cpp

@@ -17,10 +17,10 @@
 */
 //==============================================================================
 
-#include "feed/SubscriptionManager.h"
+#include "feed/SubscriptionManager.hpp"
 
-#include "data/Types.h"
-#include "feed/Types.h"
+#include "data/Types.hpp"
+#include "feed/Types.hpp"
 
 #include <boost/asio/spawn.hpp>
 #include <boost/json/object.hpp>

src/feed/SubscriptionManager.hpp

@@ -19,15 +19,15 @@
 
 #pragma once
 
-#include "data/BackendInterface.h"
-#include "data/Types.h"
-#include "feed/Types.h"
-#include "feed/impl/BookChangesFeed.h"
-#include "feed/impl/ForwardFeed.h"
-#include "feed/impl/LedgerFeed.h"
-#include "feed/impl/ProposedTransactionFeed.h"
-#include "feed/impl/TransactionFeed.h"
-#include "util/log/Logger.h"
+#include "data/BackendInterface.hpp"
+#include "data/Types.hpp"
+#include "feed/Types.hpp"
+#include "feed/impl/BookChangesFeed.hpp"
+#include "feed/impl/ForwardFeed.hpp"
+#include "feed/impl/LedgerFeed.hpp"
+#include "feed/impl/ProposedTransactionFeed.hpp"
+#include "feed/impl/TransactionFeed.hpp"
+#include "util/log/Logger.hpp"
 
 #include <boost/asio/executor_work_guard.hpp>
 #include <boost/asio/io_context.hpp>

src/feed/Types.hpp

@@ -19,7 +19,7 @@
 
 #pragma once
 
-#include "web/interface/ConnectionBase.h"
+#include "web/interface/ConnectionBase.hpp"
 
 #include <memory>
 

src/feed/impl/BookChangesFeed.hpp

@@ -19,9 +19,9 @@
 
 #pragma once
 
-#include "data/Types.h"
-#include "feed/impl/SingleFeedBase.h"
-#include "rpc/BookChangesHelper.h"
+#include "data/Types.hpp"
+#include "feed/impl/SingleFeedBase.hpp"
+#include "rpc/BookChangesHelper.hpp"
 
 #include <boost/asio/io_context.hpp>
 #include <boost/json/serialize.hpp>

src/feed/impl/ForwardFeed.hpp

@@ -19,7 +19,7 @@
 
 #pragma once
 
-#include "feed/impl/SingleFeedBase.h"
+#include "feed/impl/SingleFeedBase.hpp"
 
 #include <boost/json/object.hpp>
 #include <boost/json/serialize.hpp>

src/feed/impl/LedgerFeed.cpp

@@ -17,13 +17,13 @@
 */
 //==============================================================================
 
-#include "feed/impl/LedgerFeed.h"
+#include "feed/impl/LedgerFeed.hpp"
 
-#include "data/BackendInterface.h"
-#include "feed/Types.h"
-#include "feed/impl/SingleFeedBase.h"
-#include "rpc/RPCHelpers.h"
-#include "util/Assert.h"
+#include "data/BackendInterface.hpp"
+#include "feed/Types.hpp"
+#include "feed/impl/SingleFeedBase.hpp"
+#include "rpc/RPCHelpers.hpp"
+#include "util/Assert.hpp"
 
 #include <boost/asio/spawn.hpp>
 #include <boost/json/object.hpp>

src/feed/impl/LedgerFeed.hpp

@@ -19,9 +19,9 @@
 
 #pragma once
 
-#include "data/BackendInterface.h"
-#include "feed/Types.h"
-#include "feed/impl/SingleFeedBase.h"
+#include "data/BackendInterface.hpp"
+#include "feed/Types.hpp"
+#include "feed/impl/SingleFeedBase.hpp"
 
 #include <boost/asio/io_context.hpp>
 #include <boost/asio/spawn.hpp>

src/feed/impl/ProposedTransactionFeed.cpp

@@ -17,11 +17,11 @@
 */
 //==============================================================================
 
-#include "feed/impl/ProposedTransactionFeed.h"
+#include "feed/impl/ProposedTransactionFeed.hpp"
 
-#include "feed/Types.h"
-#include "rpc/RPCHelpers.h"
-#include "util/log/Logger.h"
+#include "feed/Types.hpp"
+#include "rpc/RPCHelpers.hpp"
+#include "util/log/Logger.hpp"
 
 #include <boost/asio/post.hpp>
 #include <boost/json/object.hpp>

src/feed/impl/ProposedTransactionFeed.hpp

@@ -19,12 +19,12 @@
 
 #pragma once
 
-#include "feed/Types.h"
-#include "feed/impl/TrackableSignal.h"
-#include "feed/impl/TrackableSignalMap.h"
-#include "feed/impl/Util.h"
-#include "util/log/Logger.h"
-#include "util/prometheus/Gauge.h"
+#include "feed/Types.hpp"
+#include "feed/impl/TrackableSignal.hpp"
+#include "feed/impl/TrackableSignalMap.hpp"
+#include "feed/impl/Util.hpp"
+#include "util/log/Logger.hpp"
+#include "util/prometheus/Gauge.hpp"
 
 #include <boost/asio/io_context.hpp>
 #include <boost/asio/strand.hpp>

src/feed/impl/SingleFeedBase.cpp

@@ -17,12 +17,12 @@
 */
 //==============================================================================
 
-#include "feed/impl/SingleFeedBase.h"
+#include "feed/impl/SingleFeedBase.hpp"
 
-#include "feed/Types.h"
-#include "feed/impl/TrackableSignal.h"
-#include "feed/impl/Util.h"
-#include "util/log/Logger.h"
+#include "feed/Types.hpp"
+#include "feed/impl/TrackableSignal.hpp"
+#include "feed/impl/Util.hpp"
+#include "util/log/Logger.hpp"
 
 #include <boost/asio/io_context.hpp>
 #include <boost/asio/post.hpp>

src/feed/impl/SingleFeedBase.hpp

@@ -19,10 +19,10 @@
 
 #pragma once
 
-#include "feed/Types.h"
-#include "feed/impl/TrackableSignal.h"
-#include "util/log/Logger.h"
-#include "util/prometheus/Gauge.h"
+#include "feed/Types.hpp"
+#include "feed/impl/TrackableSignal.hpp"
+#include "util/log/Logger.hpp"
+#include "util/prometheus/Gauge.hpp"
 
 #include <boost/asio/io_context.hpp>
 #include <boost/asio/strand.hpp>

src/feed/impl/TrackableSignalMap.hpp

@@ -19,7 +19,7 @@
 
 #pragma once
 
-#include "feed/impl/TrackableSignal.h"
+#include "feed/impl/TrackableSignal.hpp"
 
 #include <boost/signals2.hpp>
 

src/feed/impl/TransactionFeed.cpp

@@ -17,14 +17,14 @@
 */
 //==============================================================================
 
-#include "feed/impl/TransactionFeed.h"
+#include "feed/impl/TransactionFeed.hpp"
 
-#include "data/BackendInterface.h"
-#include "data/Types.h"
-#include "feed/Types.h"
-#include "rpc/JS.h"
-#include "rpc/RPCHelpers.h"
-#include "util/log/Logger.h"
+#include "data/BackendInterface.hpp"
+#include "data/Types.hpp"
+#include "feed/Types.hpp"
+#include "rpc/JS.hpp"
+#include "rpc/RPCHelpers.hpp"
+#include "util/log/Logger.hpp"
 
 #include <boost/asio/post.hpp>
 #include <boost/asio/spawn.hpp>

src/feed/impl/TransactionFeed.hpp

@@ -19,14 +19,14 @@
 
 #pragma once
 
-#include "data/BackendInterface.h"
-#include "data/Types.h"
-#include "feed/Types.h"
-#include "feed/impl/TrackableSignal.h"
-#include "feed/impl/TrackableSignalMap.h"
-#include "feed/impl/Util.h"
-#include "util/log/Logger.h"
-#include "util/prometheus/Gauge.h"
+#include "data/BackendInterface.hpp"
+#include "data/Types.hpp"
+#include "feed/Types.hpp"
+#include "feed/impl/TrackableSignal.hpp"
+#include "feed/impl/TrackableSignalMap.hpp"
+#include "feed/impl/Util.hpp"
+#include "util/log/Logger.hpp"
+#include "util/prometheus/Gauge.hpp"
 
 #include <boost/asio/io_context.hpp>
 #include <boost/asio/strand.hpp>

src/feed/impl/Util.hpp

@@ -19,9 +19,9 @@
 
 #pragma once
 
-#include "util/prometheus/Gauge.h"
-#include "util/prometheus/Label.h"
-#include "util/prometheus/Prometheus.h"
+#include "util/prometheus/Gauge.hpp"
+#include "util/prometheus/Label.hpp"
+#include "util/prometheus/Prometheus.hpp"
 
 #include <fmt/core.h>
 

src/main/Main.cpp

@@ -17,24 +17,24 @@
 */
 //==============================================================================
 
-#include "data/BackendFactory.h"
-#include "etl/ETLHelpers.h"
-#include "etl/ETLService.h"
-#include "feed/SubscriptionManager.h"
-#include "main/Build.h"
-#include "rpc/Counters.h"
-#include "rpc/RPCEngine.h"
-#include "rpc/WorkQueue.h"
-#include "rpc/common/impl/HandlerProvider.h"
-#include "util/TerminationHandler.h"
-#include "util/config/Config.h"
-#include "util/log/Logger.h"
-#include "util/prometheus/Prometheus.h"
-#include "web/DOSGuard.h"
-#include "web/IntervalSweepHandler.h"
-#include "web/RPCServerHandler.h"
-#include "web/Server.h"
-#include "web/WhitelistHandler.h"
+#include "data/BackendFactory.hpp"
+#include "etl/ETLHelpers.hpp"
+#include "etl/ETLService.hpp"
+#include "feed/SubscriptionManager.hpp"
+#include "main/Build.hpp"
+#include "rpc/Counters.hpp"
+#include "rpc/RPCEngine.hpp"
+#include "rpc/WorkQueue.hpp"
+#include "rpc/common/impl/HandlerProvider.hpp"
+#include "util/TerminationHandler.hpp"
+#include "util/config/Config.hpp"
+#include "util/log/Logger.hpp"
+#include "util/prometheus/Prometheus.hpp"
+#include "web/DOSGuard.hpp"
+#include "web/IntervalSweepHandler.hpp"
+#include "web/RPCServerHandler.hpp"
+#include "web/Server.hpp"
+#include "web/WhitelistHandler.hpp"
 
 #include <boost/asio/buffer.hpp>
 #include <boost/asio/io_context.hpp>

src/rpc/AMMHelpers.cpp

@@ -17,12 +17,21 @@
 */
 //==============================================================================
 
-#include "rpc/AMMHelpers.h"
+#include "rpc/AMMHelpers.hpp"
 
-#include "data/BackendInterface.h"
-#include "util/log/Logger.h"
+#include "data/BackendInterface.hpp"
+#include "rpc/RPCHelpers.hpp"
 
+#include <boost/asio/spawn.hpp>
 #include <ripple/protocol/AMMCore.h>
+#include <ripple/protocol/AccountID.h>
+#include <ripple/protocol/Issue.h>
+#include <ripple/protocol/SField.h>
+#include <ripple/protocol/STLedgerEntry.h>
+#include <ripple/protocol/UintTypes.h>
+
+#include <cstdint>
+#include <utility>
 
 namespace rpc {
 

src/rpc/AMMHelpers.hpp

@@ -19,8 +19,17 @@
 
 #pragma once
 
-#include "data/BackendInterface.h"
-#include "rpc/RPCHelpers.h"
+#include "data/BackendInterface.hpp"
+
+#include <boost/asio/spawn.hpp>
+#include <ripple/protocol/AccountID.h>
+#include <ripple/protocol/Issue.h>
+#include <ripple/protocol/STAmount.h>
+#include <ripple/protocol/STLedgerEntry.h>
+#include <ripple/protocol/UintTypes.h>
+
+#include <cstdint>
+#include <utility>
 
 namespace rpc {
 

src/rpc/Amendments.hpp

@@ -19,9 +19,12 @@
 
 #pragma once
 
+#include <ripple/basics/Slice.h>
 #include <ripple/basics/base_uint.h>
 #include <ripple/protocol/digest.h>
 
+#include <string_view>
+
 namespace rpc {
 
 #define REGISTER_AMENDMENT(name) inline static const ripple::uint256 name = GetAmendmentId(#name);