ci: Clio docker image (#1509)

Fixes #1051.

.github/actions/build_docker_image/action.yml

@@ -0,0 +1,66 @@
+name: Build and push Docker image
+description: Build and push Docker image to DockerHub and GitHub Container Registry
+inputs:
+  image_name:
+    description: Name of the image to build
+    required: true
+  push_image:
+    description: Whether to push the image to the registry (true/false)
+    required: true
+  directory:
+    description: The directory containing the Dockerfile
+    required: true
+  tags:
+    description: Comma separated tags to apply to the image
+    required: true
+  platforms:
+    description: Platforms to build the image for (e.g. linux/amd64,linux/arm64)
+    required: true
+  description:
+    description: Short description of the image
+    required: true
+runs:
+  using: composite
+  steps:
+      - name: Login to DockerHub
+        if: ${{ inputs.push_image == 'true' }}
+        uses: docker/login-action@v3
+        with:
+          username: ${{ env.DOCKERHUB_USER }}
+          password: ${{ env.DOCKERHUB_PW }}
+
+      - name: Login to GitHub Container Registry
+        if: ${{ inputs.push_image == 'true' }}
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ env.GITHUB_TOKEN }}
+
+      - uses: docker/setup-qemu-action@v3
+      - uses: docker/setup-buildx-action@v3
+
+      - uses: docker/metadata-action@v5
+        id: meta
+        with:
+          images: ${{ inputs.image_name }}
+          tags: ${{ steps.generate_tags.outputs.tags }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          context: ${{ inputs.directory }}
+          platforms: ${{ inputs.platforms }}
+          push: ${{ inputs.push_image == 'true' }}
+          tags: ${{ steps.meta.outputs.tags }}
+
+      - name: Update DockerHub description
+        if: ${{ inputs.push_image == 'true' }}
+        uses: peter-evans/dockerhub-description@v4
+        with:
+          username: ${{ env.DOCKERHUB_USERNAME }}
+          password: ${{ env.DOCKERHUB_PW }}
+          repository: ${{ inputs.image_name }}
+          short-description: ${{ inputs.description }}
+          readme-filepath: ${{ inputs.directory }}/README.md
+

.github/workflows/build.yml

@@ -52,28 +52,28 @@ jobs:
             build_type: Release
             conan_profile: gcc
             code_coverage: false
-            static: true
+            static: false
           - os: heavy
             container:
               image: rippleci/clio_ci:latest
             build_type: Debug
             conan_profile: gcc
             code_coverage: true
-            static: true
+            static: false
           - os: heavy
             container:
               image: rippleci/clio_ci:latest
             build_type: Release
             conan_profile: clang
             code_coverage: false
-            static: true
+            static: false
           - os: heavy
             container:
               image: rippleci/clio_ci:latest
             build_type: Debug
             conan_profile: clang
             code_coverage: false
-            static: true
+            static: false
           - os: macos14
             build_type: Release
             code_coverage: false

.github/workflows/build_clio_docker_image.yml

@@ -0,0 +1,86 @@
+name: Build and publish Clio docker image
+on:
+  workflow_call:
+    inputs:
+      tags:
+        required: true
+        type: string
+        description: Comma separated tags for docker image
+      artifact_name:
+        type: string
+        description: Name of Github artifact to put into docker image
+      strip_binary:
+        type: boolean
+        description: Whether to strip clio binary
+        default: true
+
+  workflow_dispatch:
+    inputs:
+      tags:
+        required: true
+        type: string
+        description: Comma separated tags for docker image
+      clio_server_binary_url:
+        required: true
+        type: string
+        description: Url to download clio_server binary from
+      binary_sha256:
+        required: true
+        type: string
+        description: sha256 hash of the binary
+      strip_binary:
+        type: boolean
+        description: Whether to strip clio binary
+        default: true
+
+jobs:
+  build_and_publish_image:
+    name: Build and publish image
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Download Clio binary from artifact
+        if: ${{ inputs.artifact_name != null }}
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ inputs.artifact_name }}
+          path: ./docker/clio/artifact/
+
+      - name: Download Clio binary from url
+        if: ${{ inputs.clio_server_binary_url != null }}
+        shell: bash
+        run: |
+          wget ${{inputs.clio_server_binary_url}} -P ./docker/clio/artifact/
+          if [ "$(sha256sum ./docker/clio/clio_server | awk '{print $1}')" != "${{inputs.binary_sha256}}" ]; then
+            echo "Binary sha256 sum doesn't match"
+            exit 1
+          fi
+      - name: Unpack binary
+        shell: bash
+        run: |
+          cd docker/clio/artifact
+          artifact=$(find . -type file)
+          tar -xvf $artifact
+          mv clio_server ../clio_server
+          cd ../
+          rm -rf ./artifact
+
+      - name: Strip binary
+        if: ${{ inputs.strip_binary }}
+        shell: bash
+        run: strip ./docker/clio/clio_server
+
+      - name: Build Docker image
+        uses: ./.github/actions/build_docker_image
+        env:
+          DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+          DOCKERHUB_PW: ${{ secrets.DOCKERHUB_PW }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          image_name: rippleci/clio
+          push_image: true
+          directory: docker/clio
+          tags: ${{ inputs.tags }}
+          platforms: linux/amd64
+          description: Clio is an XRP Ledger API server.

.github/workflows/nightly.yml

@@ -172,6 +172,16 @@ jobs:
             --target $GITHUB_SHA --notes-file "${RUNNER_TEMP}/nightly_notes.md" \
             ./nightly_release/clio_server*
 
+  build_and_publish_docker_image:
+    uses: ./.github/workflows/build_clio_docker_image.yml
+    secrets: inherit
+    with:
+      tags: |
+          type=raw,value=nightly
+          type=ref,value=${{ env.GITHUB_SHA }}
+      artifact_name: clio_server_Linux_Release
+      strip_binary: true
+
   create_issue_on_failure:
     needs: [build, run_tests, nightly_release]
     if: ${{ always() && contains(needs.*.result, 'failure') }}

.github/workflows/update_docker_ci.yml

@@ -18,30 +18,19 @@ jobs:
     name: Build and push docker image
     runs-on: [self-hosted, heavy]
     steps:
-      - name: Login to DockerHub
-        if: ${{ github.event_name != 'pull_request' }}
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKERHUB_USER }}
-          password: ${{ secrets.DOCKERHUB_PW }}
-
       - uses: actions/checkout@v4
-      - uses: docker/setup-qemu-action@v3
-      - uses: docker/setup-buildx-action@v3
-      - uses: docker/metadata-action@v5
-        id: meta
+      - uses: ./.github/actions/build_docker_image
+        env:
+          DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+          DOCKERHUB_PW: ${{ secrets.DOCKERHUB_PW }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
-          images: rippleci/clio_ci
+          image_name: rippleci/clio_ci
+          push_image: ${{ github.event_name != 'pull_request' }}
+          directory: docker/ci
           tags: |
             type=raw,value=latest
             type=raw,value=gcc_12_clang_16
-            type=raw,value=${{ env.GITHUB_SHA }}
-
-      - name: Build and push
-        uses: docker/build-push-action@v5
-        with:
-          context: ${{ github.workspace }}/docker/ci
+            type=ref,value=${{ env.GITHUB_SHA }}
           platforms: linux/amd64,linux/arm64
-          push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ steps.meta.outputs.tags }}
-
+          description: CI image for XRPLF/clio.

docker/ci/README.md

@@ -0,0 +1,15 @@
+# CI image for XRPLF/clio
+
+This image contains an environment to build [Clio](https://github.com/XRPLF/clio), check code and documentation.
+It is used in [Clio Github Actions](https://github.com/XRPLF/clio/actions) but can also be used to compile Clio locally.
+
+The image is based on Ubuntu 20.04 and contains:
+- clang 16
+- gcc 12.3
+- doxygen 1.10
+- gh 2.40
+- ccache 4.8.3
+- conan
+- and some other useful tools
+
+Conan is set up to build Clio without any additional steps. There are two preset conan profiles: `clang` and `gcc` to use corresponding compiler.

docker/clio/README.md

@@ -0,0 +1,23 @@
+# Clio official docker image
+
+[Clio](https://github.com/XRPLF/clio) is an XRP Ledger API server optimized for RPC calls over WebSocket or JSON-RPC.
+It stores validated historical ledger and transaction data in a space efficient format.
+
+This image contains `clio_server` binary allowing users to run Clio easily.
+
+## Clio configuration file
+
+Please note that while Clio requires a configuration file, this image doesn't include any default config.
+Your configuration file should be mounted under the path `/opt/clio/etc/config.json`.
+Clio repository provides an [example](https://github.com/XRPLF/clio/blob/develop/docs/examples/config/example-config.json) of the configuration file.
+
+Config file recommendations:
+- Set `log_to_console` to `false` if you want to avoid logs being written to `stdout`.
+- Set `log_directory` to `/opt/clio/log` to store logs in a volume.
+
+## Usage
+
+The following command can be used to run Clio in docker (assuming server's port is `51233` in your config):
+```bash
+docker run -d -v <path to your config.json>:/opt/clio/etc/config.json -v <path to store logs>:/opt/clio/log -p 51233:51233 rippleci/clio
+```

docker/clio/dockerfile

@@ -0,0 +1,16 @@
+FROM ubuntu:22.04
+
+COPY ./clio_server /opt/clio/bin/clio_server
+
+RUN ln -s /opt/clio/bin/clio_server /usr/local/bin/clio_server && \
+    mkdir -p /opt/clio/etc/ && \
+    mkdir -p /opt/clio/log/ && \
+    groupadd -g 10001 clio && \
+    useradd -u 10000 -g 10001 -s /bin/bash clio && \
+    chown clio:clio /opt/clio/log && \
+    apt update && \
+    apt install -y libatomic1
+
+USER clio
+ENTRYPOINT ["/opt/clio/bin/clio_server"]
+CMD ["--conf", "/opt/clio/etc/config.json"]

docker/develop/compose.yaml

@@ -1,4 +1,3 @@
-version: '3.7'
 services:
   clio_develop:
     image: rippleci/clio_ci:latest

src/etl/impl/GrpcSource.cpp

@@ -24,6 +24,7 @@
 #include "util/Assert.hpp"
 #include "util/log/Logger.hpp"
 
+#include <boost/asio/io_context.hpp>
 #include <boost/asio/ip/address.hpp>
 #include <boost/asio/ip/tcp.hpp>
 #include <fmt/core.h>
@@ -39,6 +40,7 @@
 #include <exception>
 #include <memory>
 #include <sstream>
+#include <stdexcept>
 #include <string>
 #include <utility>
 #include <vector>
@@ -49,9 +51,14 @@ GrpcSource::GrpcSource(std::string const& ip, std::string const& grpcPort, std::
     : log_(fmt::format("ETL_Grpc[{}:{}]", ip, grpcPort)), backend_(std::move(backend))
 {
     try {
-        boost::asio::ip::tcp::endpoint const endpoint{boost::asio::ip::make_address(ip), std::stoi(grpcPort)};
+        boost::asio::io_context ctx;
+        boost::asio::ip::tcp::resolver resolver{ctx};
+        auto const resolverResult = resolver.resolve(ip, grpcPort);
+        if (resolverResult.empty()) {
+            throw std::runtime_error("Failed to resolve " + ip + ":" + grpcPort);
+        }
         std::stringstream ss;
-        ss << endpoint;
+        ss << resolverResult.begin()->endpoint();
         grpc::ChannelArguments chArgs;
         chArgs.SetMaxReceiveMessageSize(-1);
         stub_ = org::xrpl::rpc::v1::XRPLedgerAPIService::NewStub(

src/main/Main.cpp

@@ -27,6 +27,7 @@
 #include "rpc/RPCEngine.hpp"
 #include "rpc/WorkQueue.hpp"
 #include "rpc/common/impl/HandlerProvider.hpp"
+#include "util/SignalsHandler.hpp"
 #include "util/TerminationHandler.hpp"
 #include "util/config/Config.hpp"
 #include "util/log/Logger.hpp"
@@ -168,12 +169,14 @@ int
 main(int argc, char* argv[])
 try {
     util::setTerminationHandler();
+
     auto const configPath = parseCli(argc, argv);
     auto const config = ConfigReader::open(configPath);
     if (!config) {
         std::cerr << "Couldnt parse config '" << configPath << "'." << std::endl;
         return EXIT_FAILURE;
     }
+    util::SignalsHandler signalsHandler{config};
 
     LogService::init(config);
     LOG(LogService::info()) << "Clio version: " << Build::getClioFullVersionString();

tests/unit/etl/GrpcSourceTests.cpp

@@ -44,7 +44,7 @@ struct GrpcSourceTests : NoLoggerFixture, util::prometheus::WithPrometheus, test
     GrpcSourceTests()
         : WithMockXrpLedgerAPIService("localhost:0")
         , mockBackend_(std::make_shared<testing::StrictMock<MockBackend>>(util::Config{}))
-        , grpcSource_("127.0.0.1", std::to_string(getXRPLMockPort()), mockBackend_)
+        , grpcSource_("localhost", std::to_string(getXRPLMockPort()), mockBackend_)
     {
     }