diff --git a/.github/scripts/conan/generate_matrix.py b/.github/scripts/conan/generate_matrix.py index a00a6c2b..699b9d29 100755 --- a/.github/scripts/conan/generate_matrix.py +++ b/.github/scripts/conan/generate_matrix.py @@ -3,7 +3,9 @@ import itertools import json LINUX_OS = ["heavy", "heavy-arm64"] -LINUX_CONTAINERS = ['{ "image": "ghcr.io/xrplf/clio-ci:3c1811233a2d3222cbb70ce17bae92c448756bfe" }'] +LINUX_CONTAINERS = [ + '{ "image": "ghcr.io/xrplf/clio-ci:213752862ca95ecadeb59a6176c3db91a7864b3e" }' +] LINUX_COMPILERS = ["gcc", "clang"] MACOS_OS = ["macos15"] diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6bbcb6b4..604c965c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -45,7 +45,7 @@ jobs: build_type: [Release, Debug] container: [ - '{ "image": "ghcr.io/xrplf/clio-ci:3c1811233a2d3222cbb70ce17bae92c448756bfe" }', + '{ "image": "ghcr.io/xrplf/clio-ci:213752862ca95ecadeb59a6176c3db91a7864b3e" }', ] static: [true] @@ -75,7 +75,7 @@ jobs: uses: ./.github/workflows/reusable-build.yml with: runs_on: heavy - container: '{ "image": "ghcr.io/xrplf/clio-ci:3c1811233a2d3222cbb70ce17bae92c448756bfe" }' + container: '{ "image": "ghcr.io/xrplf/clio-ci:213752862ca95ecadeb59a6176c3db91a7864b3e" }' conan_profile: gcc build_type: Debug download_ccache: true @@ -94,7 +94,7 @@ jobs: uses: ./.github/workflows/reusable-build.yml with: runs_on: heavy - container: '{ "image": "ghcr.io/xrplf/clio-ci:3c1811233a2d3222cbb70ce17bae92c448756bfe" }' + container: '{ "image": "ghcr.io/xrplf/clio-ci:213752862ca95ecadeb59a6176c3db91a7864b3e" }' conan_profile: gcc build_type: Release download_ccache: true @@ -111,7 +111,7 @@ jobs: needs: build-and-test runs-on: heavy container: - image: ghcr.io/xrplf/clio-ci:3c1811233a2d3222cbb70ce17bae92c448756bfe + image: ghcr.io/xrplf/clio-ci:213752862ca95ecadeb59a6176c3db91a7864b3e steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/check-libxrpl.yml b/.github/workflows/check-libxrpl.yml index 3c287e60..fe5c5b6b 100644 --- a/.github/workflows/check-libxrpl.yml +++ b/.github/workflows/check-libxrpl.yml @@ -17,7 +17,7 @@ jobs: name: Build Clio / `libXRPL ${{ github.event.client_payload.version }}` runs-on: heavy container: - image: ghcr.io/xrplf/clio-ci:3c1811233a2d3222cbb70ce17bae92c448756bfe + image: ghcr.io/xrplf/clio-ci:213752862ca95ecadeb59a6176c3db91a7864b3e steps: - uses: actions/checkout@v4 @@ -67,7 +67,7 @@ jobs: needs: build runs-on: heavy container: - image: ghcr.io/xrplf/clio-ci:3c1811233a2d3222cbb70ce17bae92c448756bfe + image: ghcr.io/xrplf/clio-ci:213752862ca95ecadeb59a6176c3db91a7864b3e steps: - uses: actions/download-artifact@v5 diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml index 98fac405..fb336703 100644 --- a/.github/workflows/clang-tidy.yml +++ b/.github/workflows/clang-tidy.yml @@ -27,7 +27,7 @@ jobs: if: github.event_name != 'push' || contains(github.event.head_commit.message, 'clang-tidy auto fixes') runs-on: heavy container: - image: ghcr.io/xrplf/clio-ci:3c1811233a2d3222cbb70ce17bae92c448756bfe + image: ghcr.io/xrplf/clio-ci:213752862ca95ecadeb59a6176c3db91a7864b3e permissions: contents: write diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 05f2e641..9d2bf4d1 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -14,7 +14,7 @@ jobs: build: runs-on: ubuntu-latest container: - image: ghcr.io/xrplf/clio-ci:3c1811233a2d3222cbb70ce17bae92c448756bfe + image: ghcr.io/xrplf/clio-ci:213752862ca95ecadeb59a6176c3db91a7864b3e steps: - name: Checkout diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 09dd00a6..fb2beceb 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -39,17 +39,17 @@ jobs: conan_profile: gcc build_type: Release static: true - container: '{ "image": "ghcr.io/xrplf/clio-ci:3c1811233a2d3222cbb70ce17bae92c448756bfe" }' + container: '{ "image": "ghcr.io/xrplf/clio-ci:213752862ca95ecadeb59a6176c3db91a7864b3e" }' - os: heavy conan_profile: gcc build_type: Debug static: true - container: '{ "image": "ghcr.io/xrplf/clio-ci:3c1811233a2d3222cbb70ce17bae92c448756bfe" }' + container: '{ "image": "ghcr.io/xrplf/clio-ci:213752862ca95ecadeb59a6176c3db91a7864b3e" }' - os: heavy conan_profile: gcc.ubsan build_type: Release static: false - container: '{ "image": "ghcr.io/xrplf/clio-ci:3c1811233a2d3222cbb70ce17bae92c448756bfe" }' + container: '{ "image": "ghcr.io/xrplf/clio-ci:213752862ca95ecadeb59a6176c3db91a7864b3e" }' uses: ./.github/workflows/reusable-build-test.yml with: @@ -73,7 +73,7 @@ jobs: include: - os: heavy conan_profile: clang - container: '{ "image": "ghcr.io/xrplf/clio-ci:3c1811233a2d3222cbb70ce17bae92c448756bfe" }' + container: '{ "image": "ghcr.io/xrplf/clio-ci:213752862ca95ecadeb59a6176c3db91a7864b3e" }' static: true - os: macos15 conan_profile: apple-clang diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index c6443fd7..69cfeefe 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -11,4 +11,4 @@ jobs: uses: XRPLF/actions/.github/workflows/pre-commit.yml@afbcbdafbe0ce5439492fb87eda6441371086386 with: runs_on: heavy - container: '{ "image": "ghcr.io/xrplf/clio-pre-commit:3c1811233a2d3222cbb70ce17bae92c448756bfe" }' + container: '{ "image": "ghcr.io/xrplf/clio-pre-commit:213752862ca95ecadeb59a6176c3db91a7864b3e" }' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bb6208b1..9cb099ca 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -29,7 +29,7 @@ jobs: conan_profile: gcc build_type: Release static: true - container: '{ "image": "ghcr.io/xrplf/clio-ci:3c1811233a2d3222cbb70ce17bae92c448756bfe" }' + container: '{ "image": "ghcr.io/xrplf/clio-ci:213752862ca95ecadeb59a6176c3db91a7864b3e" }' uses: ./.github/workflows/reusable-build-test.yml with: diff --git a/.github/workflows/reusable-release.yml b/.github/workflows/reusable-release.yml index 0e4d4470..e4cd3da8 100644 --- a/.github/workflows/reusable-release.yml +++ b/.github/workflows/reusable-release.yml @@ -42,7 +42,7 @@ jobs: release: runs-on: heavy container: - image: ghcr.io/xrplf/clio-ci:3c1811233a2d3222cbb70ce17bae92c448756bfe + image: ghcr.io/xrplf/clio-ci:213752862ca95ecadeb59a6176c3db91a7864b3e env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/sanitizers.yml b/.github/workflows/sanitizers.yml index 0eb55807..068c971e 100644 --- a/.github/workflows/sanitizers.yml +++ b/.github/workflows/sanitizers.yml @@ -44,7 +44,7 @@ jobs: uses: ./.github/workflows/reusable-build-test.yml with: runs_on: heavy - container: '{ "image": "ghcr.io/xrplf/clio-ci:3c1811233a2d3222cbb70ce17bae92c448756bfe" }' + container: '{ "image": "ghcr.io/xrplf/clio-ci:213752862ca95ecadeb59a6176c3db91a7864b3e" }' download_ccache: false upload_ccache: false conan_profile: ${{ matrix.compiler }}${{ matrix.sanitizer_ext }} diff --git a/docker/ci/README.md b/docker/ci/README.md index 8cb6a92d..f41a9a96 100644 --- a/docker/ci/README.md +++ b/docker/ci/README.md @@ -9,7 +9,7 @@ The image is based on Ubuntu 20.04 and contains: - Clang 19 - ClangBuildAnalyzer 1.6.0 - Conan 2.20.1 -- Doxygen 1.12 +- Doxygen 1.14 - GCC 15.2.0 - GDB 16.3 - gh 2.74 diff --git a/docker/develop/compose.yaml b/docker/develop/compose.yaml index ed9429dd..0c02af1c 100644 --- a/docker/develop/compose.yaml +++ b/docker/develop/compose.yaml @@ -1,6 +1,6 @@ services: clio_develop: - image: ghcr.io/xrplf/clio-ci:3c1811233a2d3222cbb70ce17bae92c448756bfe + image: ghcr.io/xrplf/clio-ci:213752862ca95ecadeb59a6176c3db91a7864b3e volumes: - clio_develop_conan_data:/root/.conan2/p - clio_develop_ccache:/root/.ccache diff --git a/docs/Doxyfile b/docs/Doxyfile index 0e987938..65e3b6dc 100644 --- a/docs/Doxyfile +++ b/docs/Doxyfile @@ -15,6 +15,7 @@ EXTRACT_ANON_NSPACES = NO SORT_MEMBERS_CTORS_1ST = YES INPUT = ${SOURCE}/src +USE_MDFILE_AS_MAINPAGE = ${SOURCE}/src/README.md EXCLUDE_SYMBOLS = ${EXCLUDES} RECURSIVE = YES HAVE_DOT = ${USE_DOT} diff --git a/docs/build-clio.md b/docs/build-clio.md index 6a750d22..bb5ebbed 100644 --- a/docs/build-clio.md +++ b/docs/build-clio.md @@ -177,7 +177,7 @@ There are several CMake options you can use to customize the build: ### Generating API docs for Clio -The API documentation for Clio is generated by [Doxygen](https://www.doxygen.nl/index.html). If you want to generate the API documentation when building Clio, make sure to install Doxygen 1.12.0 on your system. +The API documentation for Clio is generated by [Doxygen](https://www.doxygen.nl/index.html). If you want to generate the API documentation when building Clio, make sure to install Doxygen 1.14.0 on your system. To generate the API docs, please use CMake option `-Ddocs=ON` as described above and build the `docs` target. @@ -191,7 +191,7 @@ Open the `index.html` file in your browser to see the documentation pages. It is also possible to build Clio using [Docker](https://www.docker.com/) if you don't want to install all the dependencies on your machine. ```sh -docker run -it ghcr.io/xrplf/clio-ci:3c1811233a2d3222cbb70ce17bae92c448756bfe +docker run -it ghcr.io/xrplf/clio-ci:213752862ca95ecadeb59a6176c3db91a7864b3e git clone https://github.com/XRPLF/clio cd clio ``` diff --git a/pre-commit-hooks/check-doxygen-docs.sh b/pre-commit-hooks/check-doxygen-docs.sh index dce0b93d..cc548792 100755 --- a/pre-commit-hooks/check-doxygen-docs.sh +++ b/pre-commit-hooks/check-doxygen-docs.sh @@ -36,19 +36,19 @@ EOF exit 0 fi -# Check version of doxygen is at least 1.12 +# Check version of doxygen is at least 1.14 version=$($DOXYGEN --version | grep -o '[0-9\.]*') -if [[ "1.12.0" > "$version" ]]; then +if [[ "1.14.0" > "$version" ]]; then # No hard error if doxygen version is not the one we want - let CI deal with it cat < fetchLatestLedgerSequence(boost::asio::yield_context yield) const override { @@ -262,9 +250,6 @@ public: return std::nullopt; } - /** - * @copydoc BackendInterface::fetchLedgerBySequence - */ std::optional fetchLedgerBySequence(std::uint32_t const sequence, boost::asio::yield_context yield) const override { @@ -292,9 +277,6 @@ public: return std::nullopt; } - /** - * @copydoc BackendInterface::fetchLedgerByHash - */ std::optional fetchLedgerByHash(ripple::uint256 const& hash, boost::asio::yield_context yield) const override { @@ -315,9 +297,6 @@ public: return std::nullopt; } - /** - * @copydoc BackendInterface::hardFetchLedgerRange(boost::asio::yield_context) const - */ std::optional hardFetchLedgerRange(boost::asio::yield_context yield) const override { @@ -356,9 +335,6 @@ public: return std::nullopt; } - /** - * @copydoc BackendInterface::fetchAllTransactionsInLedger - */ std::vector fetchAllTransactionsInLedger(std::uint32_t const ledgerSequence, boost::asio::yield_context yield) const override { @@ -366,9 +342,6 @@ public: return fetchTransactions(hashes, yield); } - /** - * @copydoc BackendInterface::fetchAllTransactionHashesInLedger - */ std::vector fetchAllTransactionHashesInLedger( std::uint32_t const ledgerSequence, @@ -402,9 +375,6 @@ public: return hashes; } - /** - * @copydoc BackendInterface::fetchNFT - */ std::optional fetchNFT( ripple::uint256 const& tokenID, @@ -444,9 +414,6 @@ public: return std::nullopt; } - /** - * @copydoc BackendInterface::fetchNFTTransactions - */ TransactionsAndCursor fetchNFTTransactions( ripple::uint256 const& tokenID, @@ -518,9 +485,6 @@ public: return {txns, {}}; } - /** - * @copydoc BackendInterface::fetchMPTHolders - */ MPTHoldersAndCursor fetchMPTHolders( ripple::uint192 const& mptID, @@ -560,9 +524,6 @@ public: return {mptObjects, {}}; } - /** - * @copydoc BackendInterface::doFetchLedgerObject - */ std::optional doFetchLedgerObject( ripple::uint256 const& key, @@ -585,9 +546,6 @@ public: return std::nullopt; } - /** - * @copydoc BackendInterface::doFetchLedgerObjectSeq - */ std::optional doFetchLedgerObjectSeq( ripple::uint256 const& key, @@ -609,9 +567,6 @@ public: return std::nullopt; } - /** - * @copydoc BackendInterface::fetchTransaction - */ std::optional fetchTransaction(ripple::uint256 const& hash, boost::asio::yield_context yield) const override { @@ -629,9 +584,6 @@ public: return std::nullopt; } - /** - * @copydoc BackendInterface::doFetchSuccessorKey - */ std::optional doFetchSuccessorKey( ripple::uint256 key, @@ -654,9 +606,6 @@ public: return std::nullopt; } - /** - * @copydoc BackendInterface::fetchTransactions - */ std::vector fetchTransactions(std::vector const& hashes, boost::asio::yield_context yield) const override { @@ -698,9 +647,6 @@ public: return results; } - /** - * @copydoc BackendInterface::doFetchLedgerObjects - */ std::vector doFetchLedgerObjects( std::vector const& keys, @@ -741,9 +687,6 @@ public: return results; } - /** - * @copydoc BackendInterface::fetchLedgerDiff - */ std::vector fetchLedgerDiff(std::uint32_t const ledgerSequence, boost::asio::yield_context yield) const override { @@ -789,9 +732,6 @@ public: return results; } - /** - * @copydoc BackendInterface::fetchMigratorStatus - */ std::optional fetchMigratorStatus(std::string const& migratorName, boost::asio::yield_context yield) const override { @@ -812,9 +752,6 @@ public: return {}; } - /** - * @copydoc BackendInterface::fetchClioNodesData - */ std::expected>, std::string> fetchClioNodesData(boost::asio::yield_context yield) const override { @@ -831,9 +768,6 @@ public: return result; } - /** - * @copydoc BackendInterface::doWriteLedgerObject - */ void doWriteLedgerObject(std::string&& key, std::uint32_t const seq, std::string&& blob) override { @@ -845,9 +779,6 @@ public: executor_.write(schema_->insertObject, std::move(key), seq, std::move(blob)); } - /** - * @copydoc BackendInterface::writeSuccessor - */ void writeSuccessor(std::string&& key, std::uint32_t const seq, std::string&& successor) override { @@ -859,9 +790,6 @@ public: executor_.write(schema_->insertSuccessor, std::move(key), seq, std::move(successor)); } - /** - * @copydoc BackendInterface::writeAccountTransactions - */ void writeAccountTransactions(std::vector data) override { @@ -881,9 +809,6 @@ public: executor_.write(std::move(statements)); } - /** - * @copydoc BackendInterface::writeAccountTransaction - */ void writeAccountTransaction(AccountTransactionsData record) override { @@ -901,9 +826,6 @@ public: executor_.write(std::move(statements)); } - /** - * @copydoc BackendInterface::writeNFTTransactions - */ void writeNFTTransactions(std::vector const& data) override { @@ -919,9 +841,6 @@ public: executor_.write(std::move(statements)); } - /** - * @copydoc BackendInterface::writeTransaction - */ void writeTransaction( std::string&& hash, @@ -939,9 +858,6 @@ public: ); } - /** - * @copydoc BackendInterface::writeNFTs - */ void writeNFTs(std::vector const& data) override { @@ -980,9 +896,6 @@ public: executor_.writeEach(std::move(statements)); } - /** - * @copydoc BackendInterface::writeNFTs - */ void writeMPTHolders(std::vector const& data) override { @@ -994,9 +907,6 @@ public: executor_.write(std::move(statements)); } - /** - * @copydoc BackendInterface::startWrites - */ void startWrites() const override { @@ -1004,9 +914,6 @@ public: // probably was used in PG to start a transaction or smth. } - /** - * @copydoc BackendInterface::writeMigratorStatus - */ void writeMigratorStatus(std::string const& migratorName, std::string const& status) override { @@ -1015,27 +922,18 @@ public: ); } - /** - * @copydoc BackendInterface::writeNodeMessage - */ void writeNodeMessage(boost::uuids::uuid const& uuid, std::string message) override { executor_.writeSync(schema_->updateClioNodeMessage, data::cassandra::Text{std::move(message)}, uuid); } - /** - * @copydoc BackendInterface::isTooBusy - */ bool isTooBusy() const override { return executor_.isTooBusy(); } - /** - * @copydoc BackendInterface::stats - */ boost::json::object stats() const override { diff --git a/src/etl/README.md b/src/etl/README.md index 39e5575d..4b8cbf11 100644 --- a/src/etl/README.md +++ b/src/etl/README.md @@ -1,5 +1,7 @@ # ETL subsystem +@page "etl" ETL subsystem + A single Clio node has one or more ETL sources specified in the config file. Clio subscribes to the `ledgers` stream of each of the ETL sources. The stream sends a message whenever a new ledger is validated. Upon receiving a message on the stream, Clio fetches the data associated with the newly validated ledger from one of the ETL sources. The fetch is performed via a gRPC request called `GetLedger`. This request returns the ledger header, transactions and metadata blobs, and every ledger object added/modified/deleted as part of this ledger. The ETL subsystem then writes all of this data to the databases, and moves on to the next ledger. diff --git a/src/main/Mainpage.hpp b/src/main/Mainpage.hpp deleted file mode 100644 index 079d82b3..00000000 --- a/src/main/Mainpage.hpp +++ /dev/null @@ -1,42 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of clio: https://github.com/XRPLF/clio - Copyright (c) 2023, the clio developers. - - Permission to use, copy, modify, and distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -/** - * @mainpage Clio API server - * - * @section intro Introduction - * - * Clio is an XRP Ledger API server optimized for RPC calls over WebSocket or JSON-RPC. - * - * It stores validated historical ledger and transaction data in a more space efficient format, and uses up to 4 times - * less space than rippled. - * - * Clio can be configured to store data in Apache Cassandra or - * ScyllaDB, enabling scalable read throughput. Multiple Clio nodes can share - * access to the same dataset, which allows for a highly available cluster of Clio nodes without the need for redundant - * data storage or computation. - * - * @section Develop - * - * As you prepare to develop code for Clio, please be sure you are aware of our current - * Contribution guidelines. - * - * Read [rpc/README.md](../rpc/README.md) carefully to know more about writing your own handlers for - * Clio. - */ diff --git a/src/migration/README.md b/src/migration/README.md index 9f0d5455..93ace419 100644 --- a/src/migration/README.md +++ b/src/migration/README.md @@ -1,5 +1,7 @@ # Clio Migration +@page "migration" Clio Migration + Clio maintains the off-chain data of XRPL and multiple indexes tables to powering complex queries. To simplify the creation of index tables, this migration framework handles the process of database change and facilitates the migration of historical data seamlessly. ## Command Line Usage diff --git a/src/rpc/README.md b/src/rpc/README.md index bc1a3eb2..717a173e 100644 --- a/src/rpc/README.md +++ b/src/rpc/README.md @@ -1,4 +1,6 @@ -# RPC subsystem +# RPC subsystem + +@page "rpc" RPC subsystem The RPC subsystem is where the common framework for handling incoming JSON requests is implemented. diff --git a/src/util/async/README.md b/src/util/async/README.md index 13c86fdd..c01a2869 100644 --- a/src/util/async/README.md +++ b/src/util/async/README.md @@ -1,5 +1,7 @@ # Async framework +@page "async" Async framework + ## Introduction Clio uses threads intensively. Multiple parts of Clio were/are implemented by running a `std::thread` with some sort of loop inside. Every time this pattern is reimplemented in a slightly different way. State is managed using asynchronous queues, atomic flags, mutexes and other low level primitives. diff --git a/src/web/README.md b/src/web/README.md index 76b0ce41..094c1a5e 100644 --- a/src/web/README.md +++ b/src/web/README.md @@ -1,5 +1,7 @@ # Web server subsystem +@page "web" Web server subsystem + This folder contains all of the classes for running the web server. The web server subsystem: