Merge branch 'develop' into ximinez/online-delete-gaps

* XRPLF/develop:
  test: add more tests for `ledger_entry` RPC (5858)
  refactor: Rename `rippled.cfg` to `xrpld.cfg` (6098)
  Revert "chore: Pin ruamel.yaml<0.19 in pre-commit-hooks (6166)" (6167)
  chore: Pin ruamel.yaml<0.19 in pre-commit-hooks (6166)
  fix: Remove cryptographic libs from libxrpl Conan package (6163)

.github/actions/generate-version/action.yml

@@ -1,32 +0,0 @@
-name: Generate build version number
-description: "Generate build version number."
-
-outputs:
-  version:
-    description: "The generated build version number."
-    value: ${{ steps.version.outputs.version }}
-
-runs:
-  using: composite
-  steps:
-    - name: Generate version
-      id: version
-      shell: bash
-      env:
-        # Only append the commit hash for the develop branch. For releases the
-        # version is used as-is. We will shorten it to 6 characters below.
-        COMMIT_HASH: ${{ github.ref == 'refs/heads/develop' && github.sha || '' }}
-      run: |
-        echo 'Extracting version from BuildInfo.cpp.'
-        VERSION="$(cat src/libxrpl/protocol/BuildInfo.cpp | grep "versionString =" | awk -F '"' '{print $2}')"
-        if [[ -z "${VERSION}" ]]; then
-          echo 'Unable to extract version from BuildInfo.cpp.'
-          exit 1
-        fi
-
-        if [[ -n "${COMMIT_HASH}" ]]; then
-          echo 'Appending shortened commit hash to version.'
-          VERSION="${VERSION}-${COMMIT_HASH:0:6}"
-        fi
-
-        echo "version=${VERSION}" >> "${GITHUB_OUTPUT}"

.github/actions/upload-recipe/action.yml

@@ -1,46 +0,0 @@
-name: Upload Conan recipe
-description: "Upload recipe to a Conan remote."
-
-inputs:
-  conan_recipe_ref:
-    description: "The Conan recipe reference ('name/version') to upload."
-    required: true
-  conan_remote_name:
-    description: "The name of the Conan remote to use."
-    required: true
-  conan_remote_url:
-    description: "The URL of the Conan endpoint to use."
-    required: true
-  conan_remote_username:
-    description: "The username for logging into the Conan remote."
-    required: true
-  conan_remote_password:
-    description: "The password for logging into the Conan remote."
-    required: true
-
-runs:
-  using: composite
-
-  steps:
-    - name: Set up Conan
-      uses: ./.github/actions/setup-conan
-      with:
-        conan_remote_name: ${{ inputs.conan_remote_name }}
-        conan_remote_url: ${{ inputs.conan_remote_url }}
-
-    - name: Log into Conan remote
-      shell: bash
-      env:
-        CONAN_REMOTE_NAME: ${{ inputs.conan_remote_name }}
-        CONAN_REMOTE_USERNAME: ${{ inputs.conan_remote_username }}
-        CONAN_REMOTE_PASSWORD: ${{ inputs.conan_remote_password }}
-      run: conan remote login "${CONAN_REMOTE_NAME}" "${CONAN_REMOTE_USERNAME}" --password "${CONAN_REMOTE_PASSWORD}"
-
-    - name: Upload Conan recipe
-      shell: bash
-      env:
-        CONAN_RECIPE_REF: ${{ inputs.conan_recipe_ref }}
-        CONAN_REMOTE_NAME: ${{ inputs.conan_remote_name }}
-      run: |
-        conan export .
-        conan upload --confirm --check --remote="${CONAN_REMOTE_NAME}" ${CONAN_RECIPE_REF}

.github/scripts/strategy-matrix/generate.py

@@ -20,8 +20,8 @@ class Config:
 Generate a strategy matrix for GitHub Actions CI.
 
 On each PR commit we will build a selection of Debian, RHEL, Ubuntu, MacOS, and
-Windows configurations, while upon merge into the develop or release branches,
-we will build all configurations, and test most of them.
+Windows configurations, while upon merge into the develop, release, or master
+branches, we will build all configurations, and test most of them.
 
 We will further set additional CMake arguments as follows:
 - All builds will have the `tests`, `werr`, and `xrpld` options.

.github/workflows/on-pr.yml

@@ -53,6 +53,7 @@ jobs:
             .github/scripts/rename/**
             .github/workflows/reusable-check-levelization.yml
             .github/workflows/reusable-check-rename.yml
+            .github/workflows/reusable-notify-clio.yml
             .github/workflows/on-pr.yml
 
             # Keep the paths below in sync with those in `on-trigger.yml`.
@@ -65,7 +66,6 @@ jobs:
             .github/workflows/reusable-build-test.yml
             .github/workflows/reusable-strategy-matrix.yml
             .github/workflows/reusable-test.yml
-            .github/workflows/reusable-upload-recipe.yml
             .codecov.yml
             cmake/**
             conan/**
@@ -121,15 +121,12 @@ jobs:
     secrets:
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
-  upload-recipe:
+  notify-clio:
     needs:
       - should-run
       - build-test
-    # Only run when committing to the release branch in the XRPLF repository. We
-    # also upload the recipe when merging into the develop branch, but that is
-    # handled in the on-trigger.yml workflow.
-    if: ${{ github.repository_owner == 'XRPLF' && needs.should-run.outputs.go == 'true' && startsWith(github.ref, 'refs/heads/release') }}
-    uses: ./.github/workflows/reusable-upload-recipe.yml
+    if: ${{ needs.should-run.outputs.go == 'true' && (startsWith(github.base_ref, 'release') || github.base_ref == 'master') }}
+    uses: ./.github/workflows/reusable-notify-clio.yml
     secrets:
       clio_notify_token: ${{ secrets.CLIO_NOTIFY_TOKEN }}
       conan_remote_username: ${{ secrets.CONAN_REMOTE_USERNAME }}
@@ -140,7 +137,6 @@ jobs:
     needs:
       - build-test
       - check-levelization
-      - upload-recipe
     runs-on: ubuntu-latest
     steps:
       - name: Fail

.github/workflows/on-trigger.yml

@@ -1,8 +1,9 @@
-# This workflow runs all workflows to build and test the code on various Linux
-# flavors, as well as on MacOS and Windows, on a scheduled basis, on merge into
-# the 'develop' or 'release*' branches, or when requested manually. Upon
-# successful completion, it also uploads the built libxrpl package to the Conan
-# remote.
+# This workflow runs all workflows to build the dependencies required for the
+# project on various Linux flavors, as well as on MacOS and Windows, on a
+# scheduled basis, on merge into the 'develop', 'release', or 'master' branches,
+# or manually. The missing commits check is only run when the code is merged
+# into the 'develop' or 'release' branches, and the documentation is built when
+# the code is merged into the 'develop' branch.
 name: Trigger
 
 on:
@@ -10,6 +11,7 @@ on:
     branches:
       - "develop"
       - "release*"
+      - "master"
     paths:
       # These paths are unique to `on-trigger.yml`.
       - ".github/workflows/on-trigger.yml"
@@ -24,7 +26,6 @@ on:
       - ".github/workflows/reusable-build-test.yml"
       - ".github/workflows/reusable-strategy-matrix.yml"
       - ".github/workflows/reusable-test.yml"
-      - ".github/workflows/reusable-upload-recipe.yml"
       - ".codecov.yml"
       - "cmake/**"
       - "conan/**"
@@ -69,22 +70,11 @@ jobs:
     with:
       # Enable ccache only for events targeting the XRPLF repository, since
       # other accounts will not have access to our remote cache storage.
-      # However, we do not enable ccache for events targeting a release branch,
-      # to protect against the rare case that the output produced by ccache is
-      # not identical to a regular compilation.
-      ccache_enabled: ${{ github.repository_owner == 'XRPLF' && !startsWith(github.ref, 'refs/heads/release') }}
+      # However, we do not enable ccache for events targeting the master or a
+      # release branch, to protect against the rare case that the output
+      # produced by ccache is not identical to a regular compilation.
+      ccache_enabled: ${{ github.repository_owner == 'XRPLF' && !(github.base_ref == 'master' || startsWith(github.base_ref, 'release')) }}
       os: ${{ matrix.os }}
       strategy_matrix: ${{ github.event_name == 'schedule' && 'all' || 'minimal' }}
     secrets:
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
-
-  upload-recipe:
-    needs: build-test
-    # Only run when pushing to the develop branch in the XRPLF repository. We
-    # also upload the recipe when committing to a release branch, but that is
-    # handled in the on-pr.yml workflow.
-    if: ${{ github.repository_owner == 'XRPLF' && github.event_name == 'push' && github.ref == 'refs/heads/develop' }}
-    uses: ./.github/workflows/reusable-upload-recipe.yml
-    secrets:
-      conan_remote_username: ${{ secrets.CONAN_REMOTE_USERNAME }}
-      conan_remote_password: ${{ secrets.CONAN_REMOTE_PASSWORD }}

.github/workflows/pre-commit.yml

@@ -3,9 +3,7 @@ name: Run pre-commit hooks
 on:
   pull_request:
   push:
-    branches:
-      - "develop"
-      - "release*"
+    branches: [develop, release, master]
   workflow_dispatch:
 
 jobs:

.github/workflows/reusable-notify-clio.yml

@@ -1,7 +1,7 @@
-# This workflow exports the built libxrpl package to the Conan remote, and for
-# releases also notifies the Clio repository about the new version, so it can
-# check for compatibility.
-name: Upload Conan recipe
+# This workflow exports the built libxrpl package to the Conan remote on a
+# a channel named after the pull request, and notifies the Clio repository about
+# the new version so it can check for compatibility.
+name: Notify Clio
 
 # This workflow can only be triggered by other workflows.
 on:
@@ -20,7 +20,7 @@ on:
     secrets:
       clio_notify_token:
         description: "The GitHub token to notify Clio about new versions."
-        required: false
+        required: true
       conan_remote_username:
         description: "The username for logging into the Conan remote."
         required: true
@@ -29,7 +29,7 @@ on:
         required: true
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}-upload-recipe
+  group: ${{ github.workflow }}-${{ github.ref }}-clio
   cancel-in-progress: true
 
 defaults:
@@ -38,35 +38,46 @@ defaults:
 
 jobs:
   upload:
+    if: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
     runs-on: ubuntu-latest
     container: ghcr.io/xrplf/ci/ubuntu-noble:gcc-13-sha-5dd7158
     steps:
       - name: Checkout repository
         uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-
-      - name: Generate build version number
-        id: version
-        uses: ./.github/actions/generate-version
-
-      - name: Determine recipe reference
-        id: ref
-        run: echo "ref=xrpl/${{ steps.version.outputs.version }}" >> "${GITHUB_OUTPUT}"
-
-      - name: Upload recipe
-        uses: ./.github/actions/upload-recipe
-        id: upload
+      - name: Generate outputs
+        id: generate
+        env:
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        run: |
+          echo 'Generating user and channel.'
+          echo "user=clio" >> "${GITHUB_OUTPUT}"
+          echo "channel=pr_${PR_NUMBER}" >> "${GITHUB_OUTPUT}"
+          echo 'Extracting version.'
+          echo "version=$(cat src/libxrpl/protocol/BuildInfo.cpp | grep "versionString =" | awk -F '"' '{print $2}')" >> "${GITHUB_OUTPUT}"
+      - name: Calculate conan reference
+        id: conan_ref
+        run: |
+          echo "conan_ref=${{ steps.generate.outputs.version }}@${{ steps.generate.outputs.user }}/${{ steps.generate.outputs.channel }}" >> "${GITHUB_OUTPUT}"
+      - name: Set up Conan
+        uses: ./.github/actions/setup-conan
         with:
-          conan_recipe_ref: ${{ steps.version.outputs.version }}
           conan_remote_name: ${{ inputs.conan_remote_name }}
           conan_remote_url: ${{ inputs.conan_remote_url }}
-          conan_remote_username: ${{ secrets.conan_remote_username }}
-          conan_remote_password: ${{ secrets.conan_remote_password }}
+      - name: Log into Conan remote
+        env:
+          CONAN_REMOTE_NAME: ${{ inputs.conan_remote_name }}
+        run: conan remote login "${CONAN_REMOTE_NAME}" "${{ secrets.conan_remote_username }}" --password "${{ secrets.conan_remote_password }}"
+      - name: Upload package
+        env:
+          CONAN_REMOTE_NAME: ${{ inputs.conan_remote_name }}
+        run: |
+          conan export --user=${{ steps.generate.outputs.user }} --channel=${{ steps.generate.outputs.channel }} .
+          conan upload --confirm --check --remote="${CONAN_REMOTE_NAME}" xrpl/${{ steps.conan_ref.outputs.conan_ref }}
     outputs:
-      conan_ref: ${{ steps.ref.outputs.ref }}
+      conan_ref: ${{ steps.conan_ref.outputs.conan_ref }}
 
   notify:
     needs: upload
-    if: ${{ github.event_name == 'pull_request' }}
     runs-on: ubuntu-latest
     steps:
       - name: Notify Clio

cfg/xrpld-example.cfg

@@ -1051,10 +1051,11 @@
 #                           The online delete process checks periodically
 #                           that xrpld is still in sync with the network,
 #                           and that the validated ledger is less than
-#                           'age_threshold_seconds' old. If not, then continue
+#                           'age_threshold_seconds' old, and that all
+#                           recent ledgers are available. If not, then continue
 #                           sleeping for this number of seconds and
 #                           checking until healthy.
-#                           Default is 5.
+#                           Default is 1.
 #
 #   Notes:
 #       The 'node_db' entry configures the primary, persistent storage.

src/test/app/LedgerMaster_test.cpp

@@ -2,6 +2,7 @@
 #include <test/jtx/Env.h>
 
 #include <xrpld/app/ledger/LedgerMaster.h>
+#include <xrpld/app/misc/SHAMapStore.h>
 
 namespace xrpl {
 namespace test {
@@ -100,6 +101,88 @@ class LedgerMaster_test : public beast::unit_test::suite
         }
     }
 
+    void
+    testCompleteLedgerRange(FeatureBitset features)
+    {
+        // Note that this test is intentionally very similar to
+        // SHAMapStore_test::testLedgerGaps, but has a different
+        // focus.
+
+        testcase("Complete Ledger operations");
+
+        using namespace test::jtx;
+
+        auto const deleteInterval = 8;
+
+        Env env{*this, envconfig([](auto cfg) {
+                    return online_delete(std::move(cfg), deleteInterval);
+                })};
+
+        auto const alice = Account("alice");
+        env.fund(XRP(1000), alice);
+        env.close();
+
+        auto& lm = env.app().getLedgerMaster();
+        LedgerIndex minSeq = 2;
+        LedgerIndex maxSeq = env.closed()->header().seq;
+        auto& store = env.app().getSHAMapStore();
+        store.rendezvous();
+        LedgerIndex lastRotated = store.getLastRotated();
+        BEAST_EXPECTS(maxSeq == 3, to_string(maxSeq));
+        BEAST_EXPECTS(
+            lm.getCompleteLedgers() == "2-3", lm.getCompleteLedgers());
+        BEAST_EXPECTS(lastRotated == 3, to_string(lastRotated));
+        BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 0);
+        BEAST_EXPECT(
+            lm.missingFromCompleteLedgerRange(minSeq + 1, maxSeq - 1) == 0);
+        BEAST_EXPECT(
+            lm.missingFromCompleteLedgerRange(minSeq - 1, maxSeq + 1) == 2);
+        BEAST_EXPECT(
+            lm.missingFromCompleteLedgerRange(minSeq - 2, maxSeq - 2) == 2);
+        BEAST_EXPECT(
+            lm.missingFromCompleteLedgerRange(minSeq + 2, maxSeq + 2) == 2);
+
+        // Close enough ledgers to rotate a few times
+        for (int i = 0; i < 24; ++i)
+        {
+            for (int t = 0; t < 3; ++t)
+            {
+                env(noop(alice));
+            }
+            env.close();
+            store.rendezvous();
+
+            ++maxSeq;
+
+            if (maxSeq == lastRotated + deleteInterval)
+            {
+                minSeq = lastRotated;
+                lastRotated = maxSeq;
+            }
+            BEAST_EXPECTS(
+                env.closed()->header().seq == maxSeq,
+                to_string(env.closed()->header().seq));
+            BEAST_EXPECTS(
+                store.getLastRotated() == lastRotated,
+                to_string(store.getLastRotated()));
+            std::stringstream expectedRange;
+            expectedRange << minSeq << "-" << maxSeq;
+            BEAST_EXPECTS(
+                lm.getCompleteLedgers() == expectedRange.str(),
+                lm.getCompleteLedgers());
+            BEAST_EXPECT(
+                lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 0);
+            BEAST_EXPECT(
+                lm.missingFromCompleteLedgerRange(minSeq + 1, maxSeq - 1) == 0);
+            BEAST_EXPECT(
+                lm.missingFromCompleteLedgerRange(minSeq - 1, maxSeq + 1) == 2);
+            BEAST_EXPECT(
+                lm.missingFromCompleteLedgerRange(minSeq - 2, maxSeq - 2) == 2);
+            BEAST_EXPECT(
+                lm.missingFromCompleteLedgerRange(minSeq + 2, maxSeq + 2) == 2);
+        }
+    }
+
 public:
     void
     run() override
@@ -113,6 +196,7 @@ public:
     testWithFeats(FeatureBitset features)
     {
         testTxnIdFromIndex(features);
+        testCompleteLedgerRange(features);
     }
 };
 

src/test/app/SHAMapStore_test.cpp

@@ -1,6 +1,7 @@
 #include <test/jtx.h>
 #include <test/jtx/envconfig.h>
 
+#include <xrpld/app/ledger/LedgerMaster.h>
 #include <xrpld/app/main/Application.h>
 #include <xrpld/app/main/NodeStoreScheduler.h>
 #include <xrpld/app/misc/SHAMapStore.h>
@@ -10,6 +11,8 @@
 #include <xrpl/nodestore/detail/DatabaseRotatingImp.h>
 #include <xrpl/protocol/jss.h>
 
+#include <thread>
+
 namespace xrpl {
 namespace test {
 
@@ -20,10 +23,8 @@ class SHAMapStore_test : public beast::unit_test::suite
     static auto
     onlineDelete(std::unique_ptr<Config> cfg)
     {
-        cfg->LEDGER_HISTORY = deleteInterval;
-        auto& section = cfg->section(ConfigSection::nodeDatabase());
-        section.set("online_delete", std::to_string(deleteInterval));
-        return cfg;
+        using namespace jtx;
+        return online_delete(std::move(cfg), deleteInterval);
     }
 
     static auto
@@ -626,6 +627,184 @@ public:
         BEAST_EXPECT(dbr->getName() == "3");
     }
 
+    void
+    testLedgerGaps()
+    {
+        // Note that this test is intentionally very similar to
+        // LedgerMaster_test::testCompleteLedgerRange, but has a different
+        // focus.
+
+        testcase("Wait for ledger gaps to fill in");
+
+        using namespace test::jtx;
+
+        Env env{*this, envconfig(onlineDelete)};
+
+        std::map<LedgerIndex, uint256> hashes;
+
+        auto failureMessage = [&](char const* label,
+                                  auto expected,
+                                  auto actual) {
+            std::stringstream ss;
+            ss << label << ": Expected: " << expected << ", Got: " << actual;
+            return ss.str();
+        };
+
+        auto const alice = Account("alice");
+        env.fund(XRP(1000), alice);
+        env.close();
+
+        auto& lm = env.app().getLedgerMaster();
+        LedgerIndex minSeq = 2;
+        LedgerIndex maxSeq = env.closed()->header().seq;
+        auto& store = env.app().getSHAMapStore();
+        store.rendezvous();
+        LedgerIndex lastRotated = store.getLastRotated();
+        BEAST_EXPECTS(maxSeq == 3, to_string(maxSeq));
+        BEAST_EXPECTS(
+            lm.getCompleteLedgers() == "2-3", lm.getCompleteLedgers());
+        BEAST_EXPECTS(lastRotated == 3, to_string(lastRotated));
+        BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 0);
+        BEAST_EXPECT(
+            lm.missingFromCompleteLedgerRange(minSeq + 1, maxSeq - 1) == 0);
+        BEAST_EXPECT(
+            lm.missingFromCompleteLedgerRange(minSeq - 1, maxSeq + 1) == 2);
+        BEAST_EXPECT(
+            lm.missingFromCompleteLedgerRange(minSeq - 2, maxSeq - 2) == 2);
+        BEAST_EXPECT(
+            lm.missingFromCompleteLedgerRange(minSeq + 2, maxSeq + 2) == 2);
+
+        // Close enough ledgers to rotate a few times
+        while (maxSeq < 20)
+        {
+            for (int t = 0; t < 3; ++t)
+            {
+                env(noop(alice));
+            }
+            env.close();
+            store.rendezvous();
+
+            ++maxSeq;
+
+            if (maxSeq + 1 == lastRotated + deleteInterval)
+            {
+                using namespace std::chrono_literals;
+
+                // The next ledger will trigger a rotation. Delete the
+                // current ledger from LedgerMaster.
+                std::this_thread::sleep_for(100ms);
+                LedgerIndex const deleteSeq = maxSeq;
+                while (!lm.haveLedger(deleteSeq))
+                {
+                    std::this_thread::sleep_for(100ms);
+                }
+                lm.clearLedger(deleteSeq);
+
+                auto expectedRange =
+                    [](auto minSeq, auto deleteSeq, auto maxSeq) {
+                        std::stringstream expectedRange;
+                        expectedRange << minSeq << "-" << (deleteSeq - 1);
+                        if (deleteSeq + 1 == maxSeq)
+                            expectedRange << "," << maxSeq;
+                        else if (deleteSeq < maxSeq)
+                            expectedRange << "," << (deleteSeq + 1) << "-"
+                                          << maxSeq;
+                        return expectedRange.str();
+                    };
+                BEAST_EXPECTS(
+                    lm.getCompleteLedgers() ==
+                        expectedRange(minSeq, deleteSeq, maxSeq),
+                    failureMessage(
+                        "Complete ledgers",
+                        expectedRange(minSeq, deleteSeq, maxSeq),
+                        lm.getCompleteLedgers()));
+                BEAST_EXPECT(
+                    lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 1);
+
+                // Close another ledger, which will trigger a rotation, but the
+                // rotation will be stuck until the missing ledger is filled in.
+                env.close();
+                // DO NOT CALL rendezvous()! You'll end up with a deadlock.
+                ++maxSeq;
+
+                // Nothing has changed
+                BEAST_EXPECTS(
+                    store.getLastRotated() == lastRotated,
+                    failureMessage(
+                        "lastRotated", lastRotated, store.getLastRotated()));
+                BEAST_EXPECTS(
+                    lm.getCompleteLedgers() ==
+                        expectedRange(minSeq, deleteSeq, maxSeq),
+                    failureMessage(
+                        "Complete ledgers",
+                        expectedRange(minSeq, deleteSeq, maxSeq),
+                        lm.getCompleteLedgers()));
+
+                // Close 5 more ledgers, waiting one second in between to
+                // simulate the ledger making progress while online delete waits
+                // for the missing ledger to be filled in.
+                // This ensures the healthWait check has time to run and
+                // detect the gap.
+                for (int l = 0; l < 5; ++l)
+                {
+                    env.close();
+                    // DO NOT CALL rendezvous()! You'll end up with a deadlock.
+                    ++maxSeq;
+                    // Nothing has changed
+                    BEAST_EXPECTS(
+                        store.getLastRotated() == lastRotated,
+                        failureMessage(
+                            "lastRotated",
+                            lastRotated,
+                            store.getLastRotated()));
+                    BEAST_EXPECTS(
+                        lm.getCompleteLedgers() ==
+                            expectedRange(minSeq, deleteSeq, maxSeq),
+                        failureMessage(
+                            "Complete Ledgers",
+                            expectedRange(minSeq, deleteSeq, maxSeq),
+                            lm.getCompleteLedgers()));
+                    std::this_thread::sleep_for(1s);
+                }
+
+                // Put the missing ledger back in LedgerMaster
+                lm.setLedgerRangePresent(deleteSeq, deleteSeq);
+
+                // Wait for the rotation to finish
+                store.rendezvous();
+
+                minSeq = lastRotated;
+                lastRotated = deleteSeq + 1;
+            }
+            BEAST_EXPECT(maxSeq != lastRotated + deleteInterval);
+            BEAST_EXPECTS(
+                env.closed()->header().seq == maxSeq,
+                failureMessage("maxSeq", maxSeq, env.closed()->header().seq));
+            BEAST_EXPECTS(
+                store.getLastRotated() == lastRotated,
+                failureMessage(
+                    "lastRotated", lastRotated, store.getLastRotated()));
+            std::stringstream expectedRange;
+            expectedRange << minSeq << "-" << maxSeq;
+            BEAST_EXPECTS(
+                lm.getCompleteLedgers() == expectedRange.str(),
+                failureMessage(
+                    "CompleteLedgers",
+                    expectedRange.str(),
+                    lm.getCompleteLedgers()));
+            BEAST_EXPECT(
+                lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 0);
+            BEAST_EXPECT(
+                lm.missingFromCompleteLedgerRange(minSeq + 1, maxSeq - 1) == 0);
+            BEAST_EXPECT(
+                lm.missingFromCompleteLedgerRange(minSeq - 1, maxSeq + 1) == 2);
+            BEAST_EXPECT(
+                lm.missingFromCompleteLedgerRange(minSeq - 2, maxSeq - 2) == 2);
+            BEAST_EXPECT(
+                lm.missingFromCompleteLedgerRange(minSeq + 2, maxSeq + 2) == 2);
+        }
+    }
+
     void
     run() override
     {
@@ -633,6 +812,7 @@ public:
         testAutomatic();
         testCanDelete();
         testRotate();
+        testLedgerGaps();
     }
 };
 

src/test/jtx/envconfig.h

@@ -58,6 +58,17 @@ envconfig(F&& modfunc, Args&&... args)
     return modfunc(envconfig(), std::forward<Args>(args)...);
 }
 
+/// @brief adjust config to enable online_delete
+///
+/// @param cfg config instance to be modified
+///
+/// @param deleteInterval how many new ledgers should be available before
+/// rotating. Defaults to 8, because the standalone minimum is 8.
+///
+/// @return unique_ptr to Config instance
+std::unique_ptr<Config>
+online_delete(std::unique_ptr<Config> cfg, std::uint32_t deleteInterval = 8);
+
 /// @brief adjust config so no admin ports are enabled
 ///
 /// this is intended for use with envconfig, as in

src/test/jtx/impl/envconfig.cpp

@@ -53,6 +53,15 @@ setupConfigForUnitTests(Config& cfg)
 
 namespace jtx {
 
+std::unique_ptr<Config>
+online_delete(std::unique_ptr<Config> cfg, std::uint32_t deleteInterval)
+{
+    cfg->LEDGER_HISTORY = deleteInterval;
+    auto& section = cfg->section(ConfigSection::nodeDatabase());
+    section.set("online_delete", std::to_string(deleteInterval));
+    return cfg;
+}
+
 std::unique_ptr<Config>
 no_admin(std::unique_ptr<Config> cfg)
 {

src/test/rpc/LedgerEntry_test.cpp

@@ -5,8 +5,6 @@
 #include <test/jtx/multisign.h>
 #include <test/jtx/xchain_bridge.h>
 
-#include <xrpld/app/tx/apply.h>
-
 #include <xrpl/beast/unit_test.h>
 #include <xrpl/json/json_value.h>
 #include <xrpl/protocol/AccountID.h>
@@ -32,7 +30,6 @@ enum class FieldType {
     CurrencyField,
     HashField,
     HashOrObjectField,
-    FixedHashField,
     IssueField,
     ObjectField,
     StringField,
@@ -89,7 +86,6 @@ getTypeName(FieldType typeID)
         case FieldType::CurrencyField:
             return "Currency";
         case FieldType::HashField:
-        case FieldType::FixedHashField:
             return "hex string";
         case FieldType::HashOrObjectField:
             return "hex string or object";
@@ -206,7 +202,6 @@ class LedgerEntry_test : public beast::unit_test::suite
         static auto const& badBlobValues = remove({3, 7, 8, 16});
         static auto const& badCurrencyValues = remove({14});
         static auto const& badHashValues = remove({2, 3, 7, 8, 16});
-        static auto const& badFixedHashValues = remove({1, 2, 3, 4, 7, 8, 16});
         static auto const& badIndexValues = remove({12, 16, 18, 19});
         static auto const& badUInt32Values = remove({2, 3});
         static auto const& badUInt64Values = remove({2, 3});
@@ -227,8 +222,6 @@ class LedgerEntry_test : public beast::unit_test::suite
                 return badHashValues;
             case FieldType::HashOrObjectField:
                 return badIndexValues;
-            case FieldType::FixedHashField:
-                return badFixedHashValues;
             case FieldType::IssueField:
                 return badIssueValues;
             case FieldType::UInt32Field:
@@ -724,12 +717,7 @@ class LedgerEntry_test : public beast::unit_test::suite
         }
 
         // negative tests
-        testMalformedField(
-            env,
-            Json::Value{},
-            jss::amendments,
-            FieldType::FixedHashField,
-            "malformedRequest");
+        runLedgerEntryTest(env, jss::amendments);
     }
 
     void
@@ -1550,12 +1538,7 @@ class LedgerEntry_test : public beast::unit_test::suite
         }
 
         // negative tests
-        testMalformedField(
-            env,
-            Json::Value{},
-            jss::fee,
-            FieldType::FixedHashField,
-            "malformedRequest");
+        runLedgerEntryTest(env, jss::fee);
     }
 
     void
@@ -1578,12 +1561,7 @@ class LedgerEntry_test : public beast::unit_test::suite
         }
 
         // negative tests
-        testMalformedField(
-            env,
-            Json::Value{},
-            jss::hashes,
-            FieldType::FixedHashField,
-            "malformedRequest");
+        runLedgerEntryTest(env, jss::hashes);
     }
 
     void
@@ -1708,12 +1686,7 @@ class LedgerEntry_test : public beast::unit_test::suite
         }
 
         // negative tests
-        testMalformedField(
-            env,
-            Json::Value{},
-            jss::nunl,
-            FieldType::FixedHashField,
-            "malformedRequest");
+        runLedgerEntryTest(env, jss::nunl);
     }
 
     void
@@ -2370,438 +2343,6 @@ class LedgerEntry_test : public beast::unit_test::suite
         }
     }
 
-    /// Test the ledger entry types that don't take parameters
-    void
-    testFixed()
-    {
-        using namespace test::jtx;
-
-        Account const alice{"alice"};
-        Account const bob{"bob"};
-
-        Env env{*this, envconfig([](auto cfg) {
-                    cfg->START_UP = Config::FRESH;
-                    return cfg;
-                })};
-
-        env.close();
-
-        /** Verifies that the RPC result has the expected data
-         *
-         * @param good: Indicates that the request should have succeeded
-         *   and returned a ledger object of `expectedType` type.
-         * @param jv: The RPC result Json value
-         * @param expectedType: The type that the ledger object should
-         *   have if "good".
-         * @param expectedError: Optional. The expected error if not
-         *   good. Defaults to "entryNotFound".
-         */
-        auto checkResult =
-            [&](bool good,
-                Json::Value const& jv,
-                Json::StaticString const& expectedType,
-                std::optional<std::string> const& expectedError = {}) {
-                if (good)
-                {
-                    BEAST_EXPECTS(
-                        jv.isObject() && jv.isMember(jss::result) &&
-                            !jv[jss::result].isMember(jss::error) &&
-                            jv[jss::result].isMember(jss::node) &&
-                            jv[jss::result][jss::node].isMember(
-                                sfLedgerEntryType.jsonName) &&
-                            jv[jss::result][jss::node]
-                              [sfLedgerEntryType.jsonName] == expectedType,
-                        to_string(jv));
-                }
-                else
-                {
-                    BEAST_EXPECTS(
-                        jv.isObject() && jv.isMember(jss::result) &&
-                            jv[jss::result].isMember(jss::error) &&
-                            !jv[jss::result].isMember(jss::node) &&
-                            jv[jss::result][jss::error] ==
-                                expectedError.value_or("entryNotFound"),
-                        to_string(jv));
-                }
-            };
-
-        /** Runs a series of tests for a given fixed-position ledger
-         * entry.
-         *
-         * @param field: The Json request field to use.
-         * @param expectedType: The type that the ledger object should
-         *   have if "good".
-         * @param expectedKey: The keylet of the fixed object.
-         * @param good: Indicates whether the object is expected to
-         *   exist.
-         */
-        auto test = [&](Json::StaticString const& field,
-                        Json::StaticString const& expectedType,
-                        Keylet const& expectedKey,
-                        bool good) {
-            testcase << expectedType.c_str() << (good ? "" : " not")
-                     << " found";
-
-            auto const hexKey = strHex(expectedKey.key);
-
-            {
-                // Test bad values
-                // "field":null
-                Json::Value params;
-                params[jss::ledger_index] = jss::validated;
-                params[field] = Json::nullValue;
-                auto const jv =
-                    env.rpc("json", "ledger_entry", to_string(params));
-                checkResult(false, jv, expectedType, "malformedRequest");
-                BEAST_EXPECT(!jv[jss::result].isMember(jss::index));
-            }
-
-            {
-                Json::Value params;
-                // "field":"string"
-                params[jss::ledger_index] = jss::validated;
-                params[field] = "arbitrary string";
-                auto const jv =
-                    env.rpc("json", "ledger_entry", to_string(params));
-                checkResult(false, jv, expectedType, "malformedRequest");
-                BEAST_EXPECT(!jv[jss::result].isMember(jss::index));
-            }
-
-            {
-                Json::Value params;
-                // "field":false
-                params[jss::ledger_index] = jss::validated;
-                params[field] = false;
-                auto const jv =
-                    env.rpc("json", "ledger_entry", to_string(params));
-                checkResult(false, jv, expectedType, "invalidParams");
-                BEAST_EXPECT(!jv[jss::result].isMember(jss::index));
-            }
-
-            {
-                Json::Value params;
-
-                // "field":[incorrect index hash]
-                auto const badKey = strHex(expectedKey.key + uint256{1});
-                params[jss::ledger_index] = jss::validated;
-                params[field] = badKey;
-                auto const jv =
-                    env.rpc("json", "ledger_entry", to_string(params));
-                checkResult(false, jv, expectedType, "entryNotFound");
-                BEAST_EXPECTS(
-                    jv[jss::result][jss::index] == badKey, to_string(jv));
-            }
-
-            {
-                Json::Value params;
-                // "index":"field" using API 2
-                params[jss::ledger_index] = jss::validated;
-                params[jss::index] = field;
-                params[jss::api_version] = 2;
-                auto const jv =
-                    env.rpc("json", "ledger_entry", to_string(params));
-                checkResult(false, jv, expectedType, "malformedRequest");
-                BEAST_EXPECT(!jv[jss::result].isMember(jss::index));
-            }
-
-            std::string const pdIdx = [&]() {
-                {
-                    Json::Value params;
-                    // Test good values
-                    // Use the "field":true notation
-                    params[jss::ledger_index] = jss::validated;
-                    params[field] = true;
-                    auto const jv =
-                        env.rpc("json", "ledger_entry", to_string(params));
-                    // Index will always be returned for valid parameters.
-                    std::string const pdIdx =
-                        jv[jss::result][jss::index].asString();
-                    BEAST_EXPECTS(hexKey == pdIdx, to_string(jv));
-                    checkResult(good, jv, expectedType);
-
-                    return pdIdx;
-                }
-            }();
-
-            {
-                Json::Value params;
-                // "field":"[index hash]"
-                params[jss::ledger_index] = jss::validated;
-                params[field] = hexKey;
-                auto const jv =
-                    env.rpc("json", "ledger_entry", to_string(params));
-                checkResult(good, jv, expectedType);
-                BEAST_EXPECT(jv[jss::result][jss::index].asString() == hexKey);
-            }
-
-            {
-                // Bad value
-                // Use the "index":"field" notation with API 2
-                Json::Value params;
-                params[jss::ledger_index] = jss::validated;
-                params[jss::index] = field;
-                params[jss::api_version] = 2;
-                auto const jv =
-                    env.rpc("json", "ledger_entry", to_string(params));
-                checkResult(false, jv, expectedType, "malformedRequest");
-                BEAST_EXPECT(!jv[jss::result].isMember(jss::index));
-            }
-
-            {
-                Json::Value params;
-                // Use the "index":"field" notation with API 3
-                params[jss::ledger_index] = jss::validated;
-                params[jss::index] = field;
-                params[jss::api_version] = 3;
-                auto const jv =
-                    env.rpc("json", "ledger_entry", to_string(params));
-                // Index is correct either way
-                BEAST_EXPECT(jv[jss::result][jss::index].asString() == hexKey);
-                checkResult(good, jv, expectedType);
-            }
-
-            {
-                Json::Value params;
-                // Use the "index":"[index hash]" notation
-                params[jss::ledger_index] = jss::validated;
-                params[jss::index] = pdIdx;
-                auto const jv =
-                    env.rpc("json", "ledger_entry", to_string(params));
-                // Index is correct either way
-                BEAST_EXPECT(jv[jss::result][jss::index].asString() == hexKey);
-                checkResult(good, jv, expectedType);
-            }
-        };
-
-        test(jss::amendments, jss::Amendments, keylet::amendments(), true);
-        test(jss::fee, jss::FeeSettings, keylet::fees(), true);
-        // There won't be an nunl
-        test(jss::nunl, jss::NegativeUNL, keylet::negativeUNL(), false);
-        // Can only get the short skip list this way
-        test(jss::hashes, jss::LedgerHashes, keylet::skip(), true);
-    }
-
-    void
-    testHashes()
-    {
-        using namespace test::jtx;
-
-        Account const alice{"alice"};
-        Account const bob{"bob"};
-
-        Env env{*this, envconfig([](auto cfg) {
-                    cfg->START_UP = Config::FRESH;
-                    return cfg;
-                })};
-
-        env.close();
-
-        /** Verifies that the RPC result has the expected data
-         *
-         * @param good: Indicates that the request should have succeeded
-         *   and returned a ledger object of `expectedType` type.
-         * @param jv: The RPC result Json value
-         * @param expectedCount: The number of Hashes expected in the
-         *   object if "good".
-         * @param expectedError: Optional. The expected error if not
-         *   good. Defaults to "entryNotFound".
-         */
-        auto checkResult =
-            [&](bool good,
-                Json::Value const& jv,
-                int expectedCount,
-                std::optional<std::string> const& expectedError = {}) {
-                if (good)
-                {
-                    BEAST_EXPECTS(
-                        jv.isObject() && jv.isMember(jss::result) &&
-                            !jv[jss::result].isMember(jss::error) &&
-                            jv[jss::result].isMember(jss::node) &&
-                            jv[jss::result][jss::node].isMember(
-                                sfLedgerEntryType.jsonName) &&
-                            jv[jss::result][jss::node]
-                              [sfLedgerEntryType.jsonName] == jss::LedgerHashes,
-                        to_string(jv));
-                    BEAST_EXPECTS(
-                        jv[jss::result].isMember(jss::node) &&
-                            jv[jss::result][jss::node].isMember("Hashes") &&
-                            jv[jss::result][jss::node]["Hashes"].size() ==
-                                expectedCount,
-                        to_string(jv[jss::result][jss::node]["Hashes"].size()));
-                }
-                else
-                {
-                    BEAST_EXPECTS(
-                        jv.isObject() && jv.isMember(jss::result) &&
-                            jv[jss::result].isMember(jss::error) &&
-                            !jv[jss::result].isMember(jss::node) &&
-                            jv[jss::result][jss::error] ==
-                                expectedError.value_or("entryNotFound"),
-                        to_string(jv));
-                }
-            };
-
-        /** Runs a series of tests for a given ledger index.
-         *
-         * @param ledger: The ledger index value of the "hashes" request
-         *   parameter. May not necessarily be a number.
-         * @param expectedKey: The expected keylet of the object.
-         * @param good: Indicates whether the object is expected to
-         *   exist.
-         * @param expectedCount: The number of Hashes expected in the
-         *   object if "good".
-         */
-        auto test = [&](Json::Value ledger,
-                        Keylet const& expectedKey,
-                        bool good,
-                        int expectedCount = 0) {
-            testcase << "LedgerHashes: seq: " << env.current()->header().seq
-                     << " \"hashes\":" << to_string(ledger)
-                     << (good ? "" : " not") << " found";
-
-            auto const hexKey = strHex(expectedKey.key);
-
-            {
-                // Test bad values
-                // "hashes":null
-                Json::Value params;
-                params[jss::ledger_index] = jss::validated;
-                params[jss::hashes] = Json::nullValue;
-                auto jv = env.rpc("json", "ledger_entry", to_string(params));
-                checkResult(false, jv, 0, "malformedRequest");
-                BEAST_EXPECT(!jv[jss::result].isMember(jss::index));
-            }
-
-            {
-                Json::Value params;
-                // "hashes":"non-uint string"
-                params[jss::ledger_index] = jss::validated;
-                params[jss::hashes] = "arbitrary string";
-                auto const jv =
-                    env.rpc("json", "ledger_entry", to_string(params));
-                checkResult(false, jv, 0, "malformedRequest");
-                BEAST_EXPECT(!jv[jss::result].isMember(jss::index));
-            }
-
-            {
-                Json::Value params;
-                // "hashes":"uint string" is invalid, too
-                params[jss::ledger_index] = jss::validated;
-                params[jss::hashes] = "10";
-                auto const jv =
-                    env.rpc("json", "ledger_entry", to_string(params));
-                checkResult(false, jv, 0, "malformedRequest");
-                BEAST_EXPECT(!jv[jss::result].isMember(jss::index));
-            }
-
-            {
-                Json::Value params;
-                // "hashes":false
-                params[jss::ledger_index] = jss::validated;
-                params[jss::hashes] = false;
-                auto const jv =
-                    env.rpc("json", "ledger_entry", to_string(params));
-                checkResult(false, jv, 0, "invalidParams");
-                BEAST_EXPECT(!jv[jss::result].isMember(jss::index));
-            }
-
-            {
-                Json::Value params;
-                // "hashes":-1
-                params[jss::ledger_index] = jss::validated;
-                params[jss::hashes] = -1;
-                auto const jv =
-                    env.rpc("json", "ledger_entry", to_string(params));
-                checkResult(false, jv, 0, "internal");
-                BEAST_EXPECT(!jv[jss::result].isMember(jss::index));
-            }
-
-            // "hashes":[incorrect index hash]
-            {
-                Json::Value params;
-                auto const badKey = strHex(expectedKey.key + uint256{1});
-                params[jss::ledger_index] = jss::validated;
-                params[jss::hashes] = badKey;
-                auto const jv =
-                    env.rpc("json", "ledger_entry", to_string(params));
-                checkResult(false, jv, 0, "entryNotFound");
-                BEAST_EXPECT(jv[jss::result][jss::index] == badKey);
-            }
-
-            {
-                Json::Value params;
-                // Test good values
-                // Use the "hashes":ledger notation
-                params[jss::ledger_index] = jss::validated;
-                params[jss::hashes] = ledger;
-                auto const jv =
-                    env.rpc("json", "ledger_entry", to_string(params));
-                checkResult(good, jv, expectedCount);
-                // Index will always be returned for valid parameters.
-                std::string const pdIdx =
-                    jv[jss::result][jss::index].asString();
-                BEAST_EXPECTS(hexKey == pdIdx, strHex(pdIdx));
-            }
-
-            {
-                Json::Value params;
-                // "hashes":"[index hash]"
-                params[jss::ledger_index] = jss::validated;
-                params[jss::hashes] = hexKey;
-                auto const jv =
-                    env.rpc("json", "ledger_entry", to_string(params));
-                checkResult(good, jv, expectedCount);
-                // Index is correct either way
-                BEAST_EXPECTS(
-                    hexKey == jv[jss::result][jss::index].asString(),
-                    strHex(jv[jss::result][jss::index].asString()));
-            }
-
-            {
-                Json::Value params;
-                // Use the "index":"[index hash]" notation
-                params[jss::ledger_index] = jss::validated;
-                params[jss::index] = hexKey;
-                auto const jv =
-                    env.rpc("json", "ledger_entry", to_string(params));
-                checkResult(good, jv, expectedCount);
-                // Index is correct either way
-                BEAST_EXPECTS(
-                    hexKey == jv[jss::result][jss::index].asString(),
-                    strHex(jv[jss::result][jss::index].asString()));
-            }
-        };
-
-        // short skip list
-        test(true, keylet::skip(), true, 2);
-        // long skip list at index 0
-        test(1, keylet::skip(1), false);
-        // long skip list at index 1
-        test(1 << 17, keylet::skip(1 << 17), false);
-
-        // Close more ledgers, but stop short of the flag ledger
-        for (auto i = env.current()->seq(); i <= 250; ++i)
-            env.close();
-
-        // short skip list
-        test(true, keylet::skip(), true, 249);
-        // long skip list at index 0
-        test(1, keylet::skip(1), false);
-        // long skip list at index 1
-        test(1 << 17, keylet::skip(1 << 17), false);
-
-        // Close a flag ledger so the first "long" skip list is created
-        for (auto i = env.current()->seq(); i <= 260; ++i)
-            env.close();
-
-        // short skip list
-        test(true, keylet::skip(), true, 256);
-        // long skip list at index 0
-        test(1, keylet::skip(1), true, 1);
-        // long skip list at index 1
-        test(1 << 17, keylet::skip(1 << 17), false);
-    }
-
     void
     testCLI()
     {
@@ -2859,8 +2400,6 @@ public:
         testOracleLedgerEntry();
         testMPT();
         testPermissionedDomain();
-        testFixed();
-        testHashes();
         testCLI();
     }
 };

src/xrpld/app/ledger/LedgerMaster.h

@@ -108,7 +108,10 @@ public:
     failedSave(std::uint32_t seq, uint256 const& hash);
 
     std::string
-    getCompleteLedgers();
+    getCompleteLedgers() const;
+
+    std::size_t
+    missingFromCompleteLedgerRange(LedgerIndex first, LedgerIndex last) const;
 
     /** Apply held transactions to the open ledger
         This is normally called as we close the ledger.
@@ -325,7 +328,7 @@ private:
     // A set of transactions to replay during the next close
     std::unique_ptr<LedgerReplay> replayData;
 
-    std::recursive_mutex mCompleteLock;
+    std::recursive_mutex mutable mCompleteLock;
     RangeSet<std::uint32_t> mCompleteLedgers;
 
     // Publish thread is running.

src/xrpld/app/ledger/detail/LedgerMaster.cpp

@@ -1572,12 +1572,36 @@ LedgerMaster::getPublishedLedger()
 }
 
 std::string
-LedgerMaster::getCompleteLedgers()
+LedgerMaster::getCompleteLedgers() const
 {
     std::lock_guard sl(mCompleteLock);
     return to_string(mCompleteLedgers);
 }
 
+std::size_t
+LedgerMaster::missingFromCompleteLedgerRange(
+    LedgerIndex first,
+    LedgerIndex last) const
+{
+    // Make a copy of the range to avoid holding the lock
+    auto const range = [&] {
+        std::lock_guard sl(mCompleteLock);
+        return mCompleteLedgers;
+    }();
+
+    std::size_t missing = 0;
+
+    for (LedgerIndex idx = first; idx <= last; ++idx)
+    {
+        if (!boost::icl::contains(range, idx))
+        {
+            ++missing;
+        }
+    }
+
+    return missing;
+}
+
 std::optional<NetClock::time_point>
 LedgerMaster::getCloseTimeBySeq(LedgerIndex ledgerIndex)
 {

src/xrpld/app/misc/SHAMapStoreImp.cpp

@@ -289,6 +289,18 @@ SHAMapStoreImp::run()
             validatedSeq >= lastRotated + deleteInterval_ &&
             canDelete_ >= lastRotated - 1 && healthWait() == keepGoing;
 
+        JLOG(journal_.debug())
+            << "run: Setting lastGoodValidatedLedger_ to " << validatedSeq;
+
+        {
+            // Note that this is set after the healthWait() check, so that we
+            // don't start the rotation until the validated ledger is fully
+            // processed. It is not guaranteed to be done at this point. It also
+            // allows the testLedgerGaps unit test to work.
+            std::unique_lock<std::mutex> lock(mutex_);
+            lastGoodValidatedLedger_ = validatedSeq;
+        }
+
         // will delete up to (not including) lastRotated
         if (readyToRotate)
         {
@@ -297,7 +309,9 @@ SHAMapStoreImp::run()
                 << lastRotated << " deleteInterval " << deleteInterval_
                 << " canDelete_ " << canDelete_ << " state "
                 << app_.getOPs().strOperatingMode(false) << " age "
-                << ledgerMaster_->getValidatedLedgerAge().count() << 's';
+                << ledgerMaster_->getValidatedLedgerAge().count()
+                << "s. Complete ledgers: "
+                << ledgerMaster_->getCompleteLedgers();
 
             clearPrior(lastRotated);
             if (healthWait() == stopping)
@@ -360,7 +374,10 @@ SHAMapStoreImp::run()
                     clearCaches(validatedSeq);
                 });
 
-            JLOG(journal_.warn()) << "finished rotation " << validatedSeq;
+            JLOG(journal_.warn())
+                << "finished rotation. validatedSeq: " << validatedSeq
+                << ", lastRotated: " << lastRotated << ". Complete ledgers: "
+                << ledgerMaster_->getCompleteLedgers();
         }
     }
 }
@@ -615,22 +632,47 @@ SHAMapStoreImp::clearPrior(LedgerIndex lastRotated)
 SHAMapStoreImp::HealthResult
 SHAMapStoreImp::healthWait()
 {
+    auto index = ledgerMaster_->getValidLedgerIndex();
     auto age = ledgerMaster_->getValidatedLedgerAge();
     OperatingMode mode = netOPs_->getOperatingMode();
     std::unique_lock lock(mutex_);
-    while (!stop_ && (mode != OperatingMode::FULL || age > ageThreshold_))
+
+    auto numMissing = ledgerMaster_->missingFromCompleteLedgerRange(
+        lastGoodValidatedLedger_, index);
+    while (
+        !stop_ &&
+        (mode != OperatingMode::FULL || age > ageThreshold_ || numMissing > 0))
     {
+        // this value shouldn't change, so grab it while we have the
+        // lock
+        auto const lastGood = lastGoodValidatedLedger_;
+
         lock.unlock();
-        JLOG(journal_.warn()) << "Waiting " << recoveryWaitTime_.count()
-                              << "s for node to stabilize. state: "
-                              << app_.getOPs().strOperatingMode(mode, false)
-                              << ". age " << age.count() << 's';
+
+        auto const stream = mode != OperatingMode::FULL || age > ageThreshold_
+            ? journal_.warn()
+            : journal_.info();
+        JLOG(stream) << "Waiting " << recoveryWaitTime_.count()
+                     << "s for node to stabilize. state: "
+                     << app_.getOPs().strOperatingMode(mode, false) << ". age "
+                     << age.count() << "s. Missing ledgers: " << numMissing
+                     << ". Expect: " << lastGood << "-" << index
+                     << ". Complete ledgers: "
+                     << ledgerMaster_->getCompleteLedgers();
         std::this_thread::sleep_for(recoveryWaitTime_);
+        index = ledgerMaster_->getValidLedgerIndex();
         age = ledgerMaster_->getValidatedLedgerAge();
         mode = netOPs_->getOperatingMode();
+        numMissing =
+            ledgerMaster_->missingFromCompleteLedgerRange(lastGood, index);
+
         lock.lock();
     }
 
+    JLOG(journal_.debug()) << "healthWait: Setting lastGoodValidatedLedger_ to "
+                           << index;
+    lastGoodValidatedLedger_ = index;
+
     return stop_ ? stopping : keepGoing;
 }
 

src/xrpld/app/misc/SHAMapStoreImp.h

@@ -71,6 +71,11 @@ private:
     std::thread thread_;
     bool stop_ = false;
     bool healthy_ = true;
+    // Used to prevent ledger gaps from forming during online deletion. Keeps
+    // track of the last validated ledger that was processed without gaps. There
+    // are no guarantees about gaps while online delete is not running. For
+    // that, use advisory_delete and check for gaps externally.
+    LedgerIndex lastGoodValidatedLedger_ = 0;
     mutable std::condition_variable cond_;
     mutable std::condition_variable rendezvous_;
     mutable std::mutex mutex_;
@@ -84,11 +89,11 @@ private:
     std::uint32_t deleteBatch_ = 100;
     std::chrono::milliseconds backOff_{100};
     std::chrono::seconds ageThreshold_{60};
-    /// If  the node is out of sync during an online_delete healthWait()
-    /// call, sleep the thread for this time, and continue checking until
-    /// recovery.
+    /// If  the node is out of sync, or any recent ledgers are not
+    /// available during an online_delete healthWait() call, sleep
+    /// the thread for this time, and continue checking until recovery.
     /// See also: "recovery_wait_seconds" in xrpld-example.cfg
-    std::chrono::seconds recoveryWaitTime_{5};
+    std::chrono::seconds recoveryWaitTime_{1};
 
     // these do not exist upon SHAMapStore creation, but do exist
     // as of run() or before
@@ -212,6 +217,8 @@ private:
     enum HealthResult { stopping, keepGoing };
     [[nodiscard]] HealthResult
     healthWait();
+    bool
+    hasCompleteRange(LedgerIndex first, LedgerIndex last);
 
 public:
     void

src/xrpld/rpc/handlers/LedgerEntry.cpp

@@ -18,32 +18,6 @@
 
 namespace xrpl {
 
-using FunctionType = std::function<Expected<uint256, Json::Value>(
-    Json::Value const&,
-    Json::StaticString const,
-    unsigned const apiVersion)>;
-
-static Expected<uint256, Json::Value>
-parseFixed(
-    Keylet const& keylet,
-    Json::Value const& params,
-    Json::StaticString const& fieldName,
-    unsigned const apiVersion);
-
-// Helper function to return FunctionType for objects that have a fixed
-// location. That is, they don't take parameters to compute the index.
-// e.g. amendments, fees, negative UNL, etc.
-static FunctionType
-fixed(Keylet const& keylet)
-{
-    return [keylet](
-               Json::Value const& params,
-               Json::StaticString const fieldName,
-               unsigned const apiVersion) -> Expected<uint256, Json::Value> {
-        return parseFixed(keylet, params, fieldName, apiVersion);
-    };
-}
-
 static Expected<uint256, Json::Value>
 parseObjectID(
     Json::Value const& params,
@@ -59,33 +33,13 @@ parseObjectID(
 }
 
 static Expected<uint256, Json::Value>
-parseIndex(
-    Json::Value const& params,
-    Json::StaticString const fieldName,
-    unsigned const apiVersion)
+parseIndex(Json::Value const& params, Json::StaticString const fieldName)
 {
-    if (apiVersion > 2u && params.isString())
-    {
-        std::string const index = params.asString();
-        if (index == jss::amendments.c_str())
-            return keylet::amendments().key;
-        if (index == jss::fee.c_str())
-            return keylet::fees().key;
-        if (index == jss::nunl)
-            return keylet::negativeUNL().key;
-        if (index == jss::hashes)
-            // Note this only finds the "short" skip list. Use "hashes":index to
-            // get the long list.
-            return keylet::skip().key;
-    }
     return parseObjectID(params, fieldName, "hex string");
 }
 
 static Expected<uint256, Json::Value>
-parseAccountRoot(
-    Json::Value const& params,
-    Json::StaticString const fieldName,
-    [[maybe_unused]] unsigned const apiVersion)
+parseAccountRoot(Json::Value const& params, Json::StaticString const fieldName)
 {
     if (auto const account = LedgerEntryHelpers::parse<AccountID>(params))
     {
@@ -96,13 +50,14 @@ parseAccountRoot(
         "malformedAddress", fieldName, "AccountID");
 }
 
-auto const parseAmendments = fixed(keylet::amendments());
+static Expected<uint256, Json::Value>
+parseAmendments(Json::Value const& params, Json::StaticString const fieldName)
+{
+    return parseObjectID(params, fieldName, "hex string");
+}
 
 static Expected<uint256, Json::Value>
-parseAMM(
-    Json::Value const& params,
-    Json::StaticString const fieldName,
-    [[maybe_unused]] unsigned const apiVersion)
+parseAMM(Json::Value const& params, Json::StaticString const fieldName)
 {
     if (!params.isObject())
     {
@@ -130,10 +85,7 @@ parseAMM(
 }
 
 static Expected<uint256, Json::Value>
-parseBridge(
-    Json::Value const& params,
-    Json::StaticString const fieldName,
-    [[maybe_unused]] unsigned const apiVersion)
+parseBridge(Json::Value const& params, Json::StaticString const fieldName)
 {
     if (!params.isMember(jss::bridge))
     {
@@ -164,19 +116,13 @@ parseBridge(
 }
 
 static Expected<uint256, Json::Value>
-parseCheck(
-    Json::Value const& params,
-    Json::StaticString const fieldName,
-    [[maybe_unused]] unsigned const apiVersion)
+parseCheck(Json::Value const& params, Json::StaticString const fieldName)
 {
     return parseObjectID(params, fieldName, "hex string");
 }
 
 static Expected<uint256, Json::Value>
-parseCredential(
-    Json::Value const& cred,
-    Json::StaticString const fieldName,
-    [[maybe_unused]] unsigned const apiVersion)
+parseCredential(Json::Value const& cred, Json::StaticString const fieldName)
 {
     if (!cred.isObject())
     {
@@ -207,10 +153,7 @@ parseCredential(
 }
 
 static Expected<uint256, Json::Value>
-parseDelegate(
-    Json::Value const& params,
-    Json::StaticString const fieldName,
-    [[maybe_unused]] unsigned const apiVersion)
+parseDelegate(Json::Value const& params, Json::StaticString const fieldName)
 {
     if (!params.isObject())
     {
@@ -301,10 +244,7 @@ parseAuthorizeCredentials(Json::Value const& jv)
 }
 
 static Expected<uint256, Json::Value>
-parseDepositPreauth(
-    Json::Value const& dp,
-    Json::StaticString const fieldName,
-    [[maybe_unused]] unsigned const apiVersion)
+parseDepositPreauth(Json::Value const& dp, Json::StaticString const fieldName)
 {
     if (!dp.isObject())
     {
@@ -357,10 +297,7 @@ parseDepositPreauth(
 }
 
 static Expected<uint256, Json::Value>
-parseDID(
-    Json::Value const& params,
-    Json::StaticString const fieldName,
-    [[maybe_unused]] unsigned const apiVersion)
+parseDID(Json::Value const& params, Json::StaticString const fieldName)
 {
     auto const account = LedgerEntryHelpers::parse<AccountID>(params);
     if (!account)
@@ -375,8 +312,7 @@ parseDID(
 static Expected<uint256, Json::Value>
 parseDirectoryNode(
     Json::Value const& params,
-    Json::StaticString const fieldName,
-    [[maybe_unused]] unsigned const apiVersion)
+    Json::StaticString const fieldName)
 {
     if (!params.isObject())
     {
@@ -429,10 +365,7 @@ parseDirectoryNode(
 }
 
 static Expected<uint256, Json::Value>
-parseEscrow(
-    Json::Value const& params,
-    Json::StaticString const fieldName,
-    [[maybe_unused]] unsigned const apiVersion)
+parseEscrow(Json::Value const& params, Json::StaticString const fieldName)
 {
     if (!params.isObject())
     {
@@ -451,53 +384,20 @@ parseEscrow(
     return keylet::escrow(*id, *seq).key;
 }
 
-auto const parseFeeSettings = fixed(keylet::fees());
-
 static Expected<uint256, Json::Value>
-parseFixed(
-    Keylet const& keylet,
-    Json::Value const& params,
-    Json::StaticString const& fieldName,
-    [[maybe_unused]] unsigned const apiVersion)
+parseFeeSettings(Json::Value const& params, Json::StaticString const fieldName)
 {
-    if (!params.isBool())
-    {
-        return parseObjectID(params, fieldName, "hex string");
-    }
-    if (!params.asBool())
-    {
-        return LedgerEntryHelpers::invalidFieldError(
-            "invalidParams", fieldName, "true");
-    }
-
-    return keylet.key;
+    return parseObjectID(params, fieldName, "hex string");
 }
 
 static Expected<uint256, Json::Value>
-parseLedgerHashes(
-    Json::Value const& params,
-    Json::StaticString const fieldName,
-    unsigned const apiVersion)
+parseLedgerHashes(Json::Value const& params, Json::StaticString const fieldName)
 {
-    if (params.isUInt() || params.isInt())
-    {
-        // If the index doesn't parse as a UInt, throw
-        auto const index = params.asUInt();
-
-        // Return the "long" skip list for the given ledger index.
-        auto const keylet = keylet::skip(index);
-        return keylet.key;
-    }
-    // Return the key in `params` or the "short" skip list, which contains
-    // hashes since the last flag ledger.
-    return parseFixed(keylet::skip(), params, fieldName, apiVersion);
+    return parseObjectID(params, fieldName, "hex string");
 }
 
 static Expected<uint256, Json::Value>
-parseLoanBroker(
-    Json::Value const& params,
-    Json::StaticString const fieldName,
-    [[maybe_unused]] unsigned const apiVersion)
+parseLoanBroker(Json::Value const& params, Json::StaticString const fieldName)
 {
     if (!params.isObject())
     {
@@ -517,10 +417,7 @@ parseLoanBroker(
 }
 
 static Expected<uint256, Json::Value>
-parseLoan(
-    Json::Value const& params,
-    Json::StaticString const fieldName,
-    [[maybe_unused]] unsigned const apiVersion)
+parseLoan(Json::Value const& params, Json::StaticString const fieldName)
 {
     if (!params.isObject())
     {
@@ -540,10 +437,7 @@ parseLoan(
 }
 
 static Expected<uint256, Json::Value>
-parseMPToken(
-    Json::Value const& params,
-    Json::StaticString const fieldName,
-    [[maybe_unused]] unsigned const apiVersion)
+parseMPToken(Json::Value const& params, Json::StaticString const fieldName)
 {
     if (!params.isObject())
     {
@@ -566,8 +460,7 @@ parseMPToken(
 static Expected<uint256, Json::Value>
 parseMPTokenIssuance(
     Json::Value const& params,
-    Json::StaticString const fieldName,
-    [[maybe_unused]] unsigned const apiVersion)
+    Json::StaticString const fieldName)
 {
     auto const mptIssuanceID = LedgerEntryHelpers::parse<uint192>(params);
     if (!mptIssuanceID)
@@ -578,30 +471,25 @@ parseMPTokenIssuance(
 }
 
 static Expected<uint256, Json::Value>
-parseNFTokenOffer(
-    Json::Value const& params,
-    Json::StaticString const fieldName,
-    [[maybe_unused]] unsigned const apiVersion)
+parseNFTokenOffer(Json::Value const& params, Json::StaticString const fieldName)
 {
     return parseObjectID(params, fieldName, "hex string");
 }
 
 static Expected<uint256, Json::Value>
-parseNFTokenPage(
-    Json::Value const& params,
-    Json::StaticString const fieldName,
-    [[maybe_unused]] unsigned const apiVersion)
+parseNFTokenPage(Json::Value const& params, Json::StaticString const fieldName)
 {
     return parseObjectID(params, fieldName, "hex string");
 }
 
-auto const parseNegativeUNL = fixed(keylet::negativeUNL());
+static Expected<uint256, Json::Value>
+parseNegativeUNL(Json::Value const& params, Json::StaticString const fieldName)
+{
+    return parseObjectID(params, fieldName, "hex string");
+}
 
 static Expected<uint256, Json::Value>
-parseOffer(
-    Json::Value const& params,
-    Json::StaticString const fieldName,
-    [[maybe_unused]] unsigned const apiVersion)
+parseOffer(Json::Value const& params, Json::StaticString const fieldName)
 {
     if (!params.isObject())
     {
@@ -622,10 +510,7 @@ parseOffer(
 }
 
 static Expected<uint256, Json::Value>
-parseOracle(
-    Json::Value const& params,
-    Json::StaticString const fieldName,
-    [[maybe_unused]] unsigned const apiVersion)
+parseOracle(Json::Value const& params, Json::StaticString const fieldName)
 {
     if (!params.isObject())
     {
@@ -646,10 +531,7 @@ parseOracle(
 }
 
 static Expected<uint256, Json::Value>
-parsePayChannel(
-    Json::Value const& params,
-    Json::StaticString const fieldName,
-    [[maybe_unused]] unsigned const apiVersion)
+parsePayChannel(Json::Value const& params, Json::StaticString const fieldName)
 {
     return parseObjectID(params, fieldName, "hex string");
 }
@@ -657,8 +539,7 @@ parsePayChannel(
 static Expected<uint256, Json::Value>
 parsePermissionedDomain(
     Json::Value const& pd,
-    Json::StaticString const fieldName,
-    [[maybe_unused]] unsigned const apiVersion)
+    Json::StaticString const fieldName)
 {
     if (pd.isString())
     {
@@ -687,8 +568,7 @@ parsePermissionedDomain(
 static Expected<uint256, Json::Value>
 parseRippleState(
     Json::Value const& jvRippleState,
-    Json::StaticString const fieldName,
-    [[maybe_unused]] unsigned const apiVersion)
+    Json::StaticString const fieldName)
 {
     Currency uCurrency;
 
@@ -738,19 +618,13 @@ parseRippleState(
 }
 
 static Expected<uint256, Json::Value>
-parseSignerList(
-    Json::Value const& params,
-    Json::StaticString const fieldName,
-    [[maybe_unused]] unsigned const apiVersion)
+parseSignerList(Json::Value const& params, Json::StaticString const fieldName)
 {
     return parseObjectID(params, fieldName, "hex string");
 }
 
 static Expected<uint256, Json::Value>
-parseTicket(
-    Json::Value const& params,
-    Json::StaticString const fieldName,
-    [[maybe_unused]] unsigned const apiVersion)
+parseTicket(Json::Value const& params, Json::StaticString const fieldName)
 {
     if (!params.isObject())
     {
@@ -771,10 +645,7 @@ parseTicket(
 }
 
 static Expected<uint256, Json::Value>
-parseVault(
-    Json::Value const& params,
-    Json::StaticString const fieldName,
-    [[maybe_unused]] unsigned const apiVersion)
+parseVault(Json::Value const& params, Json::StaticString const fieldName)
 {
     if (!params.isObject())
     {
@@ -797,8 +668,7 @@ parseVault(
 static Expected<uint256, Json::Value>
 parseXChainOwnedClaimID(
     Json::Value const& claim_id,
-    Json::StaticString const fieldName,
-    [[maybe_unused]] unsigned const apiVersion)
+    Json::StaticString const fieldName)
 {
     if (!claim_id.isObject())
     {
@@ -823,8 +693,7 @@ parseXChainOwnedClaimID(
 static Expected<uint256, Json::Value>
 parseXChainOwnedCreateAccountClaimID(
     Json::Value const& claim_id,
-    Json::StaticString const fieldName,
-    [[maybe_unused]] unsigned const apiVersion)
+    Json::StaticString const fieldName)
 {
     if (!claim_id.isObject())
     {
@@ -848,6 +717,10 @@ parseXChainOwnedCreateAccountClaimID(
     return keylet.key;
 }
 
+using FunctionType = Expected<uint256, Json::Value> (*)(
+    Json::Value const&,
+    Json::StaticString const);
+
 struct LedgerEntry
 {
     Json::StaticString fieldName;
@@ -880,7 +753,7 @@ doLedgerEntry(RPC::JsonContext& context)
         {jss::ripple_state, parseRippleState, ltRIPPLE_STATE},
     });
 
-    auto const hasMoreThanOneMember = [&]() {
+    auto hasMoreThanOneMember = [&]() {
         int count = 0;
 
         for (auto const& ledgerEntry : ledgerEntryParsers)
@@ -924,8 +797,8 @@ doLedgerEntry(RPC::JsonContext& context)
                 Json::Value const& params = ledgerEntry.fieldName == jss::bridge
                     ? context.params
                     : context.params[ledgerEntry.fieldName];
-                auto const result = ledgerEntry.parseFunction(
-                    params, ledgerEntry.fieldName, context.apiVersion);
+                auto const result =
+                    ledgerEntry.parseFunction(params, ledgerEntry.fieldName);
                 if (!result)
                     return result.error();
 
@@ -956,13 +829,9 @@ doLedgerEntry(RPC::JsonContext& context)
             throw;
     }
 
-    // Return the computed index regardless of whether the node exists.
-    jvResult[jss::index] = to_string(uNodeIndex);
-
     if (uNodeIndex.isZero())
     {
-        RPC::inject_error(rpcENTRY_NOT_FOUND, jvResult);
-        return jvResult;
+        return RPC::make_error(rpcENTRY_NOT_FOUND);
     }
 
     auto const sleNode = lpLedger->read(keylet::unchecked(uNodeIndex));
@@ -974,14 +843,12 @@ doLedgerEntry(RPC::JsonContext& context)
     if (!sleNode)
     {
         // Not found.
-        RPC::inject_error(rpcENTRY_NOT_FOUND, jvResult);
-        return jvResult;
+        return RPC::make_error(rpcENTRY_NOT_FOUND);
     }
 
     if ((expectedType != ltANY) && (expectedType != sleNode->getType()))
     {
-        RPC::inject_error(rpcUNEXPECTED_LEDGER_TYPE, jvResult);
-        return jvResult;
+        return RPC::make_error(rpcUNEXPECTED_LEDGER_TYPE);
     }
 
     if (bNodeBinary)
@@ -991,10 +858,12 @@ doLedgerEntry(RPC::JsonContext& context)
         sleNode->add(s);
 
         jvResult[jss::node_binary] = strHex(s.peekData());
+        jvResult[jss::index] = to_string(uNodeIndex);
     }
     else
     {
         jvResult[jss::node] = sleNode->getJson(JsonOptions::none);
+        jvResult[jss::index] = to_string(uNodeIndex);
     }
 
     return jvResult;