Build Clio with CentOS 7

2025-11-04 11:55:51 +00:00 · 2022-08-18 17:49:00 -07:00
parent 0ebe92de68
commit 983aa29271
17 changed files with 320 additions and 29 deletions
--- a/.dockerignore
+++ b/.dockerignore
@@ -0,0 +1 @@
+build/
--- a/.github/actions/lint/action.yml
+++ b/.github/actions/lint/action.yml
@@ -0,0 +1,13 @@
+runs:
+  using: composite
+  steps:
+    # Github's ubuntu-20.04 image already has clang-format-11 installed
+    - run: |
+        find src unittests -type f \( -name '*.cpp' -o -name '*.h' -o -name '*.ipp' \) -print0 | xargs -0 clang-format-11 -i
+      shell: bash
+
+    - name: Check for differences
+      id: assert
+      shell: bash
+      run: |
+        git diff --color --exit-code | tee "clang-format.patch"
--- a/.github/actions/sign/action.yml
+++ b/.github/actions/sign/action.yml
@@ -0,0 +1,21 @@
+name: 'Sign packages'
+runs:
+  using: "composite"
+
+  steps:
+    - name: Sign
+      shell: bash
+      run: |
+        set -ex -o pipefail
+        echo "$GPG_KEY_B64"| base64 -d | gpg --batch --no-tty --allow-secret-key-import --import -
+        unset GPG_KEY_B64
+        export GPG_PASSPHRASE=$(echo $GPG_KEY_PASS_B64 | base64 -di)
+        unset GPG_KEY_PASS_B64
+        export GPG_KEYID=$(gpg --with-colon --list-secret-keys | head -n1 | cut -d : -f 5)
+        for PKG in $(ls *.deb); do
+          dpkg-sig \
+            -g "--no-tty --digest-algo 'sha512' --passphrase '${GPG_PASSPHRASE}' --pinentry-mode=loopback" \
+                  -k "${GPG_KEYID}" \
+                  --sign builder \
+                  $PKG
+        done
--- a/.github/actions/test/Dockerfile
+++ b/.github/actions/test/Dockerfile
@@ -0,0 +1,6 @@
+FROM cassandra:4.0.4
+
+RUN apt-get update && apt-get install -y postgresql
+COPY entrypoint.sh /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]
--- a/.github/actions/test/entrypoint.sh
+++ b/.github/actions/test/entrypoint.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+pg_ctlcluster 12 main start
+su postgres -c"psql -c\"alter user postgres with password 'postgres'\""
+su cassandra -c "/opt/cassandra/bin/cassandra -R"
+sleep 90
+chmod +x ./clio_tests
+./clio_tests
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -11,42 +11,61 @@ jobs:
    name: Lint
    runs-on: ubuntu-20.04
    steps:
-      - name: Get source
-        uses: actions/checkout@v3
-
+      - uses: actions/checkout@v3
      - name: Run clang-format
-        uses: XRPLF/clio-gha/lint@main
+        uses: ./.github/actions/lint

  build_clio:
    name: Build Clio
    runs-on: [self-hosted, Linux]
+    needs: lint
+    strategy:
+      fail-fast: false
+      matrix:
+        type:
+          - suffix: deb
+            image: rippleci/clio-dpkg-builder:2022-09-17
+            script: dpkg
+          - suffix: rpm
+            image: rippleci/clio-rpm-builder:2022-09-17
+            script: rpm
+    container:
+      image: ${{ matrix.type.image }}
+
    steps:
+      - uses: actions/checkout@v3
+        with:
+          path: clio

-      - name: Clone Clio repo
+      - name: Clone Clio packaging repo
        uses: actions/checkout@v3
        with:
-          path: clio_src
-
-      - name: Clone Clio CI repo
-        uses: actions/checkout@v3
-        with:
-          path: clio_ci
-          repository: 'XRPLF/clio-ci'
+          path: clio-packages
+          repository: XRPLF/clio-packages

      - name: Build
-        uses: XRPLF/clio-gha/build@main
+        shell: bash
+        run: |
+          export CLIO_ROOT=$(realpath clio)
+          if [ ${{ matrix.type.suffix }} == "rpm" ]; then
+            source /opt/rh/devtoolset-11/enable
+          fi
+          cmake -S clio-packages -B clio-packages/build -DCLIO_ROOT=$CLIO_ROOT
+          cmake --build clio-packages/build --parallel $(nproc)
+          cp ./clio-packages/build/clio-prefix/src/clio-build/clio_tests .
+          mv ./clio-packages/build/*.${{ matrix.type.suffix }} .

      - name: Artifact packages
        uses: actions/upload-artifact@v3
        with:
-          name: clio_packages
-          path: ${{ github.workspace }}/*.deb
+          name: clio_${{ matrix.type.suffix }}_packages
+          path: ${{ github.workspace }}/*.${{ matrix.type.suffix }}

      - name: Artifact clio_tests
        uses: actions/upload-artifact@v3
        with:
-          name: clio_tests
-          path: clio_tests
+          name: clio_tests-${{ matrix.type.suffix }}
+          path: ${{ github.workspace }}/clio_tests

  sign:
    name: Sign packages
@@ -56,22 +75,34 @@ jobs:
    env:
      GPG_KEY_B64: ${{ secrets.GPG_KEY_B64 }}
      GPG_KEY_PASS_B64: ${{ secrets.GPG_KEY_PASS_B64 }}
-
+    strategy:
+      fail-fast: false
+      matrix:
+        type:
+          - suffix: deb
+            image: ubuntu:20.04
+            script: dpkg
+          # - suffix: rpm
+          #   image: centos:7
+          #   script: rpm
+    container:
+      image: ${{ matrix.type.image }}
    steps:
+      - uses: actions/checkout@v3
+      - name: Install dpkg-sig
+        run: |
+          apt-get update && apt-get install -y dpkg-sig gnupg
      - name: Get package artifact
        uses: actions/download-artifact@v3
        with:
-          name: clio_packages
+          name: clio_${{ matrix.type.suffix }}_packages

      - name: find packages
-        run: find . -name "*.deb"
+        run: find . -name "*.${{ matrix.type.suffix }}"

-      - name: Install dpkg-sig
-        run: |
-          sudo apt-get update && sudo apt-get install -y dpkg-sig
+      - name: Sign packages
+        uses: ./.github/actions/sign

-      - name: Sign Debian packages
-        uses: XRPLF/clio-gha/sign@main

      - name: Verify the signature
        run: |
@@ -84,22 +115,28 @@ jobs:
        id: shortsha
        run: echo "::set-output name=sha8::$(echo ${GITHUB_SHA} | cut -c1-8)"

-      - name: Artifact Debian package
+      - name: Artifact signed packages
        uses: actions/upload-artifact@v2
        with:
-          name: clio-deb-packages-${{ steps.shortsha.outputs.sha8 }}
+          name: signed-clio-deb-packages-${{ steps.shortsha.outputs.sha8 }}
          path: ${{ github.workspace }}/*.deb

  test_clio:
    name: Test Clio
    runs-on: [self-hosted, Linux]
    needs: build_clio
+    strategy:
+      fail-fast: false
+      matrix:
+        suffix: [rpm, deb]
    steps:
+      - uses: actions/checkout@v3
+
      - name: Get clio_tests artifact
        uses: actions/download-artifact@v3
        with:
-          name: clio_tests
+          name: clio_tests-${{ matrix.suffix }}

      - name: Run tests
        timeout-minutes: 10
-        uses: XRPLF/clio-gha/test@main
+        uses: ./.github/actions/test
--- a/docker/centos/Dockerfile
+++ b/docker/centos/Dockerfile
@@ -0,0 +1,49 @@
+# FROM centos:7 as deps
+FROM centos:7 as build
+
+ENV CLIO_DIR=/opt/clio/
+# ENV OPENSSL_DIR=/opt/openssl
+
+RUN yum -y install git epel-release centos-release-scl perl-IPC-Cmd openssl
+RUN yum install -y devtoolset-11
+ENV version=3.16
+ENV build=3
+# RUN curl -OJL https://cmake.org/files/v$version/cmake-$version.$build.tar.gz
+COPY docker/shared/install_cmake.sh /install_cmake.sh
+RUN /install_cmake.sh 3.16.3 /usr/local
+RUN source /opt/rh/devtoolset-11/enable
+WORKDIR /tmp
+# RUN mkdir $OPENSSL_DIR && cd $OPENSSL_DIR
+COPY docker/centos/build_git_centos7.sh build_git_centos7.sh
+
+RUN ./build_git_centos7.sh
+RUN git clone https://github.com/openssl/openssl
+WORKDIR /tmp/openssl
+RUN git checkout OpenSSL_1_1_1q
+#--prefix=/usr --openssldir=/etc/ssl --libdir=lib no-shared zlib-dynamic
+RUN SSLDIR=$(openssl version -d | cut -d: -f2 | tr -d [:space:]\") && ./config -fPIC --prefix=/usr --openssldir=${SSLDIR} zlib shared && \
+    make -j $(nproc) && \
+    make install_sw
+WORKDIR /tmp
+# FROM centos:7 as build
+
+RUN git clone https://github.com/xrplf/clio.git
+COPY docker/shared/build_boost.sh build_boost.sh
+ENV OPENSSL_ROOT=/opt/local/openssl
+ENV BOOST_ROOT=/boost
+RUN source scl_source enable devtoolset-11 && /tmp/build_boost.sh 1.75.0
+RUN yum install -y bison flex
+RUN yum install -y rpmdevtools rpmlint
+RUN source /opt/rh/devtoolset-11/enable && cd /tmp/clio && \
+    cmake -B build -DBUILD_TESTS=1 && \
+    cmake --build build --parallel $(nproc)
+RUN mkdir output
+RUN strip clio/build/clio_server && strip clio/build/clio_tests
+RUN cp clio/build/clio_tests output/ && cp clio/build/clio_server output/
+RUN cp clio/example-config.json output/example-config.json
+
+FROM centos:7
+COPY --from=build /tmp/output /clio
+RUN mkdir -p /opt/clio/etc && mv /clio/example-config.json /opt/clio/etc/config.json
+
+CMD ["/clio/clio_server", "/opt/clio/etc/config.json"]
--- a/docker/centos/build_git_centos7.sh
+++ b/docker/centos/build_git_centos7.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -ex
+GIT_VERSION="2.37.1"
+curl -OJL https://github.com/git/git/archive/refs/tags/v${GIT_VERSION}.tar.gz
+tar zxvf git-${GIT_VERSION}.tar.gz
+cd git-${GIT_VERSION}
+
+yum install -y centos-release-scl epel-release
+yum update -y
+yum install -y devtoolset-11 autoconf gnu-getopt gettext zlib-devel  libcurl-devel
+
+source /opt/rh/devtoolset-11/enable
+make configure
+./configure
+make git -j$(nproc)
+make install git
+git  --version | cut -d ' ' -f3
--- a/docker/centos/install_cmake.sh
+++ b/docker/centos/install_cmake.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -eo pipefail
+
+CMAKE_VERSION=${1:-"3.16.3"}
+cd /tmp
+URL="https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz"
+curl -OJLs $URL
+tar xzvf cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz
+mv cmake-${CMAKE_VERSION}-Linux-x86_64 /opt/
+ln -s /opt/cmake-${CMAKE_VERSION}-Linux-x86_64/bin/cmake /usr/local/bin/cmake
--- a/docker/clio_docker/centos/build_boost.sh
+++ b/docker/clio_docker/centos/build_boost.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -exu
+
+#yum install wget lz4 lz4-devel git llvm13-static.x86_64 llvm13-devel.x86_64 devtoolset-11-binutils zlib-static
+# it's either those or link=static that halves the failures. probably link=static
+BOOST_VERSION=$1
+BOOST_VERSION_=$(echo ${BOOST_VERSION} | tr . _)
+echo "BOOST_VERSION: ${BOOST_VERSION}"
+echo "BOOST_VERSION_: ${BOOST_VERSION_}"
+curl -OJLs "https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_}.tar.gz"
+tar zxf "boost_${BOOST_VERSION_}.tar.gz"
+cd boost_${BOOST_VERSION_} && ./bootstrap.sh && ./b2 --without-python link=static -j$(nproc)
+mkdir -p /boost && mv boost /boost && mv stage /boost
--- a/docker/clio_docker/centos/build_git_centos7.sh
+++ b/docker/clio_docker/centos/build_git_centos7.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -ex
+GIT_VERSION="2.37.1"
+curl -OJL https://github.com/git/git/archive/refs/tags/v${GIT_VERSION}.tar.gz
+tar zxvf git-${GIT_VERSION}.tar.gz
+cd git-${GIT_VERSION}
+
+yum install -y centos-release-scl epel-release
+yum update -y
+yum install -y devtoolset-11 autoconf gnu-getopt gettext zlib-devel  libcurl-devel
+
+source /opt/rh/devtoolset-11/enable
+make configure
+./configure
+make git -j$(nproc)
+make install git
+git  --version | cut -d ' ' -f3
--- a/docker/clio_docker/centos/dockerfile
+++ b/docker/clio_docker/centos/dockerfile
@@ -0,0 +1,34 @@
+FROM centos:7
+
+ENV CLIO_DIR=/opt/clio/
+# ENV OPENSSL_DIR=/opt/openssl
+
+RUN yum -y install git epel-release centos-release-scl perl-IPC-Cmd openssl
+RUN yum install -y devtoolset-11
+ENV version=3.16
+ENV build=3
+# RUN curl -OJL https://cmake.org/files/v$version/cmake-$version.$build.tar.gz
+COPY install_cmake.sh /install_cmake.sh
+RUN /install_cmake.sh 3.16.3 /usr/local
+RUN source /opt/rh/devtoolset-11/enable
+WORKDIR /tmp
+# RUN mkdir $OPENSSL_DIR && cd $OPENSSL_DIR
+COPY build_git_centos7.sh build_git_centos7.sh
+
+RUN ./build_git_centos7.sh
+RUN git clone https://github.com/openssl/openssl
+WORKDIR /tmp/openssl
+RUN git checkout OpenSSL_1_1_1q
+#--prefix=/usr --openssldir=/etc/ssl --libdir=lib no-shared zlib-dynamic
+RUN SSLDIR=$(openssl version -d | cut -d: -f2 | tr -d [:space:]\") && ./config -fPIC --prefix=/usr --openssldir=${SSLDIR} zlib shared && \
+    make -j $(nproc) && \
+    make install_sw
+WORKDIR /tmp
+RUN git clone https://github.com/xrplf/clio.git
+COPY build_boost.sh build_boost.sh
+ENV OPENSSL_ROOT=/opt/local/openssl
+ENV BOOST_ROOT=/boost
+RUN source scl_source enable devtoolset-11 && /tmp/build_boost.sh 1.75.0
+RUN yum install -y bison flex
+RUN source /opt/rh/devtoolset-11/enable && \
+    cd /tmp/clio && cmake -B build -Dtests=0 -Dlocal_libarchive=1 -Dunity=0 -DBUILD_TESTS=0 && cmake --build build --parallel $(nproc)
--- a/docker/clio_docker/centos/install_cmake.sh
+++ b/docker/clio_docker/centos/install_cmake.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -eo pipefail
+
+CMAKE_VERSION=${1:-"3.16.3"}
+cd /tmp
+URL="https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz"
+curl -OJLs $URL
+tar xzvf cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz
+mv cmake-${CMAKE_VERSION}-Linux-x86_64 /opt/
+ln -s /opt/cmake-${CMAKE_VERSION}-Linux-x86_64/bin/cmake /usr/local/bin/cmake
--- a/docker/shared/build_boost.sh
+++ b/docker/shared/build_boost.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -exu
+
+#yum install wget lz4 lz4-devel git llvm13-static.x86_64 llvm13-devel.x86_64 devtoolset-11-binutils zlib-static
+# it's either those or link=static that halves the failures. probably link=static
+BOOST_VERSION=$1
+BOOST_VERSION_=$(echo ${BOOST_VERSION} | tr . _)
+echo "BOOST_VERSION: ${BOOST_VERSION}"
+echo "BOOST_VERSION_: ${BOOST_VERSION_}"
+curl -OJLs "https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_}.tar.gz"
+tar zxf "boost_${BOOST_VERSION_}.tar.gz"
+cd boost_${BOOST_VERSION_} && ./bootstrap.sh && ./b2 --without-python link=static -j$(nproc)
+mkdir -p /boost && mv boost /boost && mv stage /boost
--- a/docker/shared/install_cmake.sh
+++ b/docker/shared/install_cmake.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -eo pipefail
+
+CMAKE_VERSION=${1:-"3.16.3"}
+cd /tmp
+URL="https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz"
+curl -OJLs $URL
+tar xzvf cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz
+mv cmake-${CMAKE_VERSION}-Linux-x86_64 /opt/
+ln -s /opt/cmake-${CMAKE_VERSION}-Linux-x86_64/bin/cmake /usr/local/bin/cmake
--- a/docker/shared/install_openssl.sh
+++ b/docker/shared/install_openssl.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+set -e
--- a/docker/ubuntu/Dockerfile
+++ b/docker/ubuntu/Dockerfile
@@ -0,0 +1,24 @@
+FROM ubuntu:20.04 AS boost
+
+RUN apt-get update && apt-get install -y build-essential
+ARG BOOST_VERSION_=1_75_0
+ARG BOOST_VERSION=1.75.0
+COPY docker/shared/build_boost.sh .
+RUN apt install -y curl
+RUN ./build_boost.sh ${BOOST_VERSION}
+ENV BOOST_ROOT=/boost
+
+FROM ubuntu:20.04 AS build
+ENV BOOST_ROOT=/boost
+COPY --from=boost /boost /boost
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install --no-install-recommends -y build-essential software-properties-common pkg-config libssl-dev wget curl gpg git zlib1g-dev bison flex autoconf lsb-release
+RUN apt install -y gpg-agent
+RUN wget https://apt.llvm.org/llvm.sh
+RUN chmod +x llvm.sh && ./llvm.sh 14 && ./llvm.sh 15
+# COPY . /clio
+## Install cmake
+ARG CMAKE_VERSION=3.16.3
+COPY docker/shared/install_cmake.sh .
+RUN ./install_cmake.sh ${CMAKE_VERSION}
+ENV PATH="/opt/local/cmake/bin:$PATH"