From 983aa29271bd87ddf90214bbcdc823ea5888d3f6 Mon Sep 17 00:00:00 2001 From: Michael Legleux Date: Thu, 18 Aug 2022 17:49:00 -0700 Subject: [PATCH] Build Clio with CentOS 7 --- .dockerignore | 1 + .github/actions/lint/action.yml | 13 +++ .github/actions/sign/action.yml | 21 ++++ .github/actions/test/Dockerfile | 6 ++ .github/actions/test/entrypoint.sh | 8 ++ .github/workflows/build.yml | 95 +++++++++++++------ docker/centos/Dockerfile | 49 ++++++++++ docker/centos/build_git_centos7.sh | 18 ++++ docker/centos/install_cmake.sh | 11 +++ docker/clio_docker/centos/build_boost.sh | 13 +++ .../clio_docker/centos/build_git_centos7.sh | 18 ++++ docker/clio_docker/centos/dockerfile | 34 +++++++ docker/clio_docker/centos/install_cmake.sh | 11 +++ docker/shared/build_boost.sh | 13 +++ docker/shared/install_cmake.sh | 11 +++ docker/shared/install_openssl.sh | 3 + docker/ubuntu/Dockerfile | 24 +++++ 17 files changed, 320 insertions(+), 29 deletions(-) create mode 100644 .dockerignore create mode 100644 .github/actions/lint/action.yml create mode 100644 .github/actions/sign/action.yml create mode 100644 .github/actions/test/Dockerfile create mode 100755 .github/actions/test/entrypoint.sh create mode 100644 docker/centos/Dockerfile create mode 100755 docker/centos/build_git_centos7.sh create mode 100755 docker/centos/install_cmake.sh create mode 100755 docker/clio_docker/centos/build_boost.sh create mode 100755 docker/clio_docker/centos/build_git_centos7.sh create mode 100644 docker/clio_docker/centos/dockerfile create mode 100755 docker/clio_docker/centos/install_cmake.sh create mode 100755 docker/shared/build_boost.sh create mode 100755 docker/shared/install_cmake.sh create mode 100755 docker/shared/install_openssl.sh create mode 100644 docker/ubuntu/Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..567609b1 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +build/ diff --git a/.github/actions/lint/action.yml b/.github/actions/lint/action.yml new file mode 100644 index 00000000..87cec73f --- /dev/null +++ b/.github/actions/lint/action.yml @@ -0,0 +1,13 @@ +runs: + using: composite + steps: + # Github's ubuntu-20.04 image already has clang-format-11 installed + - run: | + find src unittests -type f \( -name '*.cpp' -o -name '*.h' -o -name '*.ipp' \) -print0 | xargs -0 clang-format-11 -i + shell: bash + + - name: Check for differences + id: assert + shell: bash + run: | + git diff --color --exit-code | tee "clang-format.patch" diff --git a/.github/actions/sign/action.yml b/.github/actions/sign/action.yml new file mode 100644 index 00000000..99064b9c --- /dev/null +++ b/.github/actions/sign/action.yml @@ -0,0 +1,21 @@ +name: 'Sign packages' +runs: + using: "composite" + + steps: + - name: Sign + shell: bash + run: | + set -ex -o pipefail + echo "$GPG_KEY_B64"| base64 -d | gpg --batch --no-tty --allow-secret-key-import --import - + unset GPG_KEY_B64 + export GPG_PASSPHRASE=$(echo $GPG_KEY_PASS_B64 | base64 -di) + unset GPG_KEY_PASS_B64 + export GPG_KEYID=$(gpg --with-colon --list-secret-keys | head -n1 | cut -d : -f 5) + for PKG in $(ls *.deb); do + dpkg-sig \ + -g "--no-tty --digest-algo 'sha512' --passphrase '${GPG_PASSPHRASE}' --pinentry-mode=loopback" \ + -k "${GPG_KEYID}" \ + --sign builder \ + $PKG + done diff --git a/.github/actions/test/Dockerfile b/.github/actions/test/Dockerfile new file mode 100644 index 00000000..a2c10da4 --- /dev/null +++ b/.github/actions/test/Dockerfile @@ -0,0 +1,6 @@ +FROM cassandra:4.0.4 + +RUN apt-get update && apt-get install -y postgresql +COPY entrypoint.sh /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/.github/actions/test/entrypoint.sh b/.github/actions/test/entrypoint.sh new file mode 100755 index 00000000..86d5fadd --- /dev/null +++ b/.github/actions/test/entrypoint.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +pg_ctlcluster 12 main start +su postgres -c"psql -c\"alter user postgres with password 'postgres'\"" +su cassandra -c "/opt/cassandra/bin/cassandra -R" +sleep 90 +chmod +x ./clio_tests +./clio_tests diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8f0d864e..eff66577 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,42 +11,61 @@ jobs: name: Lint runs-on: ubuntu-20.04 steps: - - name: Get source - uses: actions/checkout@v3 - + - uses: actions/checkout@v3 - name: Run clang-format - uses: XRPLF/clio-gha/lint@main + uses: ./.github/actions/lint build_clio: name: Build Clio runs-on: [self-hosted, Linux] + needs: lint + strategy: + fail-fast: false + matrix: + type: + - suffix: deb + image: rippleci/clio-dpkg-builder:2022-09-17 + script: dpkg + - suffix: rpm + image: rippleci/clio-rpm-builder:2022-09-17 + script: rpm + container: + image: ${{ matrix.type.image }} + steps: + - uses: actions/checkout@v3 + with: + path: clio - - name: Clone Clio repo + - name: Clone Clio packaging repo uses: actions/checkout@v3 with: - path: clio_src - - - name: Clone Clio CI repo - uses: actions/checkout@v3 - with: - path: clio_ci - repository: 'XRPLF/clio-ci' + path: clio-packages + repository: XRPLF/clio-packages - name: Build - uses: XRPLF/clio-gha/build@main + shell: bash + run: | + export CLIO_ROOT=$(realpath clio) + if [ ${{ matrix.type.suffix }} == "rpm" ]; then + source /opt/rh/devtoolset-11/enable + fi + cmake -S clio-packages -B clio-packages/build -DCLIO_ROOT=$CLIO_ROOT + cmake --build clio-packages/build --parallel $(nproc) + cp ./clio-packages/build/clio-prefix/src/clio-build/clio_tests . + mv ./clio-packages/build/*.${{ matrix.type.suffix }} . - name: Artifact packages uses: actions/upload-artifact@v3 with: - name: clio_packages - path: ${{ github.workspace }}/*.deb + name: clio_${{ matrix.type.suffix }}_packages + path: ${{ github.workspace }}/*.${{ matrix.type.suffix }} - name: Artifact clio_tests uses: actions/upload-artifact@v3 with: - name: clio_tests - path: clio_tests + name: clio_tests-${{ matrix.type.suffix }} + path: ${{ github.workspace }}/clio_tests sign: name: Sign packages @@ -56,22 +75,34 @@ jobs: env: GPG_KEY_B64: ${{ secrets.GPG_KEY_B64 }} GPG_KEY_PASS_B64: ${{ secrets.GPG_KEY_PASS_B64 }} - + strategy: + fail-fast: false + matrix: + type: + - suffix: deb + image: ubuntu:20.04 + script: dpkg + # - suffix: rpm + # image: centos:7 + # script: rpm + container: + image: ${{ matrix.type.image }} steps: + - uses: actions/checkout@v3 + - name: Install dpkg-sig + run: | + apt-get update && apt-get install -y dpkg-sig gnupg - name: Get package artifact uses: actions/download-artifact@v3 with: - name: clio_packages + name: clio_${{ matrix.type.suffix }}_packages - name: find packages - run: find . -name "*.deb" + run: find . -name "*.${{ matrix.type.suffix }}" - - name: Install dpkg-sig - run: | - sudo apt-get update && sudo apt-get install -y dpkg-sig + - name: Sign packages + uses: ./.github/actions/sign - - name: Sign Debian packages - uses: XRPLF/clio-gha/sign@main - name: Verify the signature run: | @@ -84,22 +115,28 @@ jobs: id: shortsha run: echo "::set-output name=sha8::$(echo ${GITHUB_SHA} | cut -c1-8)" - - name: Artifact Debian package + - name: Artifact signed packages uses: actions/upload-artifact@v2 with: - name: clio-deb-packages-${{ steps.shortsha.outputs.sha8 }} + name: signed-clio-deb-packages-${{ steps.shortsha.outputs.sha8 }} path: ${{ github.workspace }}/*.deb test_clio: name: Test Clio runs-on: [self-hosted, Linux] needs: build_clio + strategy: + fail-fast: false + matrix: + suffix: [rpm, deb] steps: + - uses: actions/checkout@v3 + - name: Get clio_tests artifact uses: actions/download-artifact@v3 with: - name: clio_tests + name: clio_tests-${{ matrix.suffix }} - name: Run tests timeout-minutes: 10 - uses: XRPLF/clio-gha/test@main + uses: ./.github/actions/test diff --git a/docker/centos/Dockerfile b/docker/centos/Dockerfile new file mode 100644 index 00000000..763a336e --- /dev/null +++ b/docker/centos/Dockerfile @@ -0,0 +1,49 @@ +# FROM centos:7 as deps +FROM centos:7 as build + +ENV CLIO_DIR=/opt/clio/ +# ENV OPENSSL_DIR=/opt/openssl + +RUN yum -y install git epel-release centos-release-scl perl-IPC-Cmd openssl +RUN yum install -y devtoolset-11 +ENV version=3.16 +ENV build=3 +# RUN curl -OJL https://cmake.org/files/v$version/cmake-$version.$build.tar.gz +COPY docker/shared/install_cmake.sh /install_cmake.sh +RUN /install_cmake.sh 3.16.3 /usr/local +RUN source /opt/rh/devtoolset-11/enable +WORKDIR /tmp +# RUN mkdir $OPENSSL_DIR && cd $OPENSSL_DIR +COPY docker/centos/build_git_centos7.sh build_git_centos7.sh + +RUN ./build_git_centos7.sh +RUN git clone https://github.com/openssl/openssl +WORKDIR /tmp/openssl +RUN git checkout OpenSSL_1_1_1q +#--prefix=/usr --openssldir=/etc/ssl --libdir=lib no-shared zlib-dynamic +RUN SSLDIR=$(openssl version -d | cut -d: -f2 | tr -d [:space:]\") && ./config -fPIC --prefix=/usr --openssldir=${SSLDIR} zlib shared && \ + make -j $(nproc) && \ + make install_sw +WORKDIR /tmp +# FROM centos:7 as build + +RUN git clone https://github.com/xrplf/clio.git +COPY docker/shared/build_boost.sh build_boost.sh +ENV OPENSSL_ROOT=/opt/local/openssl +ENV BOOST_ROOT=/boost +RUN source scl_source enable devtoolset-11 && /tmp/build_boost.sh 1.75.0 +RUN yum install -y bison flex +RUN yum install -y rpmdevtools rpmlint +RUN source /opt/rh/devtoolset-11/enable && cd /tmp/clio && \ + cmake -B build -DBUILD_TESTS=1 && \ + cmake --build build --parallel $(nproc) +RUN mkdir output +RUN strip clio/build/clio_server && strip clio/build/clio_tests +RUN cp clio/build/clio_tests output/ && cp clio/build/clio_server output/ +RUN cp clio/example-config.json output/example-config.json + +FROM centos:7 +COPY --from=build /tmp/output /clio +RUN mkdir -p /opt/clio/etc && mv /clio/example-config.json /opt/clio/etc/config.json + +CMD ["/clio/clio_server", "/opt/clio/etc/config.json"] diff --git a/docker/centos/build_git_centos7.sh b/docker/centos/build_git_centos7.sh new file mode 100755 index 00000000..0db195f3 --- /dev/null +++ b/docker/centos/build_git_centos7.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -ex +GIT_VERSION="2.37.1" +curl -OJL https://github.com/git/git/archive/refs/tags/v${GIT_VERSION}.tar.gz +tar zxvf git-${GIT_VERSION}.tar.gz +cd git-${GIT_VERSION} + +yum install -y centos-release-scl epel-release +yum update -y +yum install -y devtoolset-11 autoconf gnu-getopt gettext zlib-devel libcurl-devel + +source /opt/rh/devtoolset-11/enable +make configure +./configure +make git -j$(nproc) +make install git +git --version | cut -d ' ' -f3 diff --git a/docker/centos/install_cmake.sh b/docker/centos/install_cmake.sh new file mode 100755 index 00000000..6a3cba4f --- /dev/null +++ b/docker/centos/install_cmake.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -eo pipefail + +CMAKE_VERSION=${1:-"3.16.3"} +cd /tmp +URL="https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz" +curl -OJLs $URL +tar xzvf cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz +mv cmake-${CMAKE_VERSION}-Linux-x86_64 /opt/ +ln -s /opt/cmake-${CMAKE_VERSION}-Linux-x86_64/bin/cmake /usr/local/bin/cmake diff --git a/docker/clio_docker/centos/build_boost.sh b/docker/clio_docker/centos/build_boost.sh new file mode 100755 index 00000000..a7253128 --- /dev/null +++ b/docker/clio_docker/centos/build_boost.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -exu + +#yum install wget lz4 lz4-devel git llvm13-static.x86_64 llvm13-devel.x86_64 devtoolset-11-binutils zlib-static +# it's either those or link=static that halves the failures. probably link=static +BOOST_VERSION=$1 +BOOST_VERSION_=$(echo ${BOOST_VERSION} | tr . _) +echo "BOOST_VERSION: ${BOOST_VERSION}" +echo "BOOST_VERSION_: ${BOOST_VERSION_}" +curl -OJLs "https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_}.tar.gz" +tar zxf "boost_${BOOST_VERSION_}.tar.gz" +cd boost_${BOOST_VERSION_} && ./bootstrap.sh && ./b2 --without-python link=static -j$(nproc) +mkdir -p /boost && mv boost /boost && mv stage /boost diff --git a/docker/clio_docker/centos/build_git_centos7.sh b/docker/clio_docker/centos/build_git_centos7.sh new file mode 100755 index 00000000..0db195f3 --- /dev/null +++ b/docker/clio_docker/centos/build_git_centos7.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -ex +GIT_VERSION="2.37.1" +curl -OJL https://github.com/git/git/archive/refs/tags/v${GIT_VERSION}.tar.gz +tar zxvf git-${GIT_VERSION}.tar.gz +cd git-${GIT_VERSION} + +yum install -y centos-release-scl epel-release +yum update -y +yum install -y devtoolset-11 autoconf gnu-getopt gettext zlib-devel libcurl-devel + +source /opt/rh/devtoolset-11/enable +make configure +./configure +make git -j$(nproc) +make install git +git --version | cut -d ' ' -f3 diff --git a/docker/clio_docker/centos/dockerfile b/docker/clio_docker/centos/dockerfile new file mode 100644 index 00000000..012ce09e --- /dev/null +++ b/docker/clio_docker/centos/dockerfile @@ -0,0 +1,34 @@ +FROM centos:7 + +ENV CLIO_DIR=/opt/clio/ +# ENV OPENSSL_DIR=/opt/openssl + +RUN yum -y install git epel-release centos-release-scl perl-IPC-Cmd openssl +RUN yum install -y devtoolset-11 +ENV version=3.16 +ENV build=3 +# RUN curl -OJL https://cmake.org/files/v$version/cmake-$version.$build.tar.gz +COPY install_cmake.sh /install_cmake.sh +RUN /install_cmake.sh 3.16.3 /usr/local +RUN source /opt/rh/devtoolset-11/enable +WORKDIR /tmp +# RUN mkdir $OPENSSL_DIR && cd $OPENSSL_DIR +COPY build_git_centos7.sh build_git_centos7.sh + +RUN ./build_git_centos7.sh +RUN git clone https://github.com/openssl/openssl +WORKDIR /tmp/openssl +RUN git checkout OpenSSL_1_1_1q +#--prefix=/usr --openssldir=/etc/ssl --libdir=lib no-shared zlib-dynamic +RUN SSLDIR=$(openssl version -d | cut -d: -f2 | tr -d [:space:]\") && ./config -fPIC --prefix=/usr --openssldir=${SSLDIR} zlib shared && \ + make -j $(nproc) && \ + make install_sw +WORKDIR /tmp +RUN git clone https://github.com/xrplf/clio.git +COPY build_boost.sh build_boost.sh +ENV OPENSSL_ROOT=/opt/local/openssl +ENV BOOST_ROOT=/boost +RUN source scl_source enable devtoolset-11 && /tmp/build_boost.sh 1.75.0 +RUN yum install -y bison flex +RUN source /opt/rh/devtoolset-11/enable && \ + cd /tmp/clio && cmake -B build -Dtests=0 -Dlocal_libarchive=1 -Dunity=0 -DBUILD_TESTS=0 && cmake --build build --parallel $(nproc) diff --git a/docker/clio_docker/centos/install_cmake.sh b/docker/clio_docker/centos/install_cmake.sh new file mode 100755 index 00000000..6a3cba4f --- /dev/null +++ b/docker/clio_docker/centos/install_cmake.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -eo pipefail + +CMAKE_VERSION=${1:-"3.16.3"} +cd /tmp +URL="https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz" +curl -OJLs $URL +tar xzvf cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz +mv cmake-${CMAKE_VERSION}-Linux-x86_64 /opt/ +ln -s /opt/cmake-${CMAKE_VERSION}-Linux-x86_64/bin/cmake /usr/local/bin/cmake diff --git a/docker/shared/build_boost.sh b/docker/shared/build_boost.sh new file mode 100755 index 00000000..a7253128 --- /dev/null +++ b/docker/shared/build_boost.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -exu + +#yum install wget lz4 lz4-devel git llvm13-static.x86_64 llvm13-devel.x86_64 devtoolset-11-binutils zlib-static +# it's either those or link=static that halves the failures. probably link=static +BOOST_VERSION=$1 +BOOST_VERSION_=$(echo ${BOOST_VERSION} | tr . _) +echo "BOOST_VERSION: ${BOOST_VERSION}" +echo "BOOST_VERSION_: ${BOOST_VERSION_}" +curl -OJLs "https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_}.tar.gz" +tar zxf "boost_${BOOST_VERSION_}.tar.gz" +cd boost_${BOOST_VERSION_} && ./bootstrap.sh && ./b2 --without-python link=static -j$(nproc) +mkdir -p /boost && mv boost /boost && mv stage /boost diff --git a/docker/shared/install_cmake.sh b/docker/shared/install_cmake.sh new file mode 100755 index 00000000..6a3cba4f --- /dev/null +++ b/docker/shared/install_cmake.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -eo pipefail + +CMAKE_VERSION=${1:-"3.16.3"} +cd /tmp +URL="https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz" +curl -OJLs $URL +tar xzvf cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz +mv cmake-${CMAKE_VERSION}-Linux-x86_64 /opt/ +ln -s /opt/cmake-${CMAKE_VERSION}-Linux-x86_64/bin/cmake /usr/local/bin/cmake diff --git a/docker/shared/install_openssl.sh b/docker/shared/install_openssl.sh new file mode 100755 index 00000000..d9628ae7 --- /dev/null +++ b/docker/shared/install_openssl.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +set -e diff --git a/docker/ubuntu/Dockerfile b/docker/ubuntu/Dockerfile new file mode 100644 index 00000000..9078429f --- /dev/null +++ b/docker/ubuntu/Dockerfile @@ -0,0 +1,24 @@ +FROM ubuntu:20.04 AS boost + +RUN apt-get update && apt-get install -y build-essential +ARG BOOST_VERSION_=1_75_0 +ARG BOOST_VERSION=1.75.0 +COPY docker/shared/build_boost.sh . +RUN apt install -y curl +RUN ./build_boost.sh ${BOOST_VERSION} +ENV BOOST_ROOT=/boost + +FROM ubuntu:20.04 AS build +ENV BOOST_ROOT=/boost +COPY --from=boost /boost /boost +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install --no-install-recommends -y build-essential software-properties-common pkg-config libssl-dev wget curl gpg git zlib1g-dev bison flex autoconf lsb-release +RUN apt install -y gpg-agent +RUN wget https://apt.llvm.org/llvm.sh +RUN chmod +x llvm.sh && ./llvm.sh 14 && ./llvm.sh 15 +# COPY . /clio +## Install cmake +ARG CMAKE_VERSION=3.16.3 +COPY docker/shared/install_cmake.sh . +RUN ./install_cmake.sh ${CMAKE_VERSION} +ENV PATH="/opt/local/cmake/bin:$PATH"