From 43cb25533747eb28c307b10840ac96a3fbab5a07 Mon Sep 17 00:00:00 2001 From: Denis Angell Date: Thu, 9 Nov 2023 19:01:50 +0100 Subject: [PATCH] Update Workflow (#193) --- .dockerignore | 7 +- .github/workflows/build-in-docker.yml | 26 +++-- Dockerfile | 123 ------------------------ build-core.sh | 10 +- build-full.sh | 7 ++ release.sh | 17 ---- src/test/rpc/ServerDefinitions_test.cpp | 1 - ssl_script.sh | 47 --------- xahaud.binary.dockerfile | 58 ----------- 9 files changed, 34 insertions(+), 262 deletions(-) delete mode 100644 Dockerfile delete mode 100755 release.sh delete mode 100755 ssl_script.sh delete mode 100644 xahaud.binary.dockerfile diff --git a/.dockerignore b/.dockerignore index 139597f9c..b47809846 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,7 @@ +# Ignore everything +* - +# Allow files and directories +!/build-core.sh +!/build-full.sh +!/release-builder.sh \ No newline at end of file diff --git a/.github/workflows/build-in-docker.yml b/.github/workflows/build-in-docker.yml index 4b6b9d395..83f7938d5 100644 --- a/.github/workflows/build-in-docker.yml +++ b/.github/workflows/build-in-docker.yml @@ -11,21 +11,19 @@ concurrency: cancel-in-progress: true jobs: - hbb: - strategy: - fail-fast: true + builder: runs-on: [self-hosted, vanity] steps: - - name: checkout - uses: actions/checkout@v3 - - name: Set branch name - run: echo "GITHUB_BRANCH=${{ github.head_ref || github.ref }}" >> $GITHUB_ENV + - uses: actions/checkout@v3 + with: + clean: false - name: Build using Docker - run: docker build -f xahaud.binary.dockerfile --build-arg GITHUB_RUN_NUMBER=${{ github.run_number }} --build-arg GITHUB_BRANCH=${{ env.GITHUB_BRANCH }} -t transia/xahaud-binary:${{ github.sha }} . - - name: Release using Docker - run: /bin/bash release.sh ${{ env.GITHUB_BRANCH }} ${{ github.run_number }} ${{ github.sha }} - - name: Test using Docker - run: docker run --rm -i transia/xahaud-binary:${{ github.sha }} sh -c '/io/release-build/xahaud -u' - - name: Clean up Docker - run: docker rmi transia/xahaud-binary:${{ github.sha }} + run: /bin/bash release-builder.sh + + unittests: + needs: builder + runs-on: [self-hosted, vanity] + steps: + - name: Unit tests + run: /bin/bash docker-unit-tests.sh diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 3fcdb31ea..000000000 --- a/Dockerfile +++ /dev/null @@ -1,123 +0,0 @@ -# docker build -t transia/xahaud-hbb-deps . -# Use the Holy Build Box image as the base image -FROM ghcr.io/foobarwidget/holy-build-box-x64 - -# Set the user to root and the group to the current user's group ID -# Note: You will need to replace `$(id -g)` with the actual group ID or remove it if not needed -# USER root:$(id -g) - -# Install wget and other necessary tools -RUN /hbb_exe/activate-exec bash -c "yum update -y && \ - yum install -y wget lz4 lz4-devel git llvm13-static.x86_64 llvm13-devel.x86_64 devtoolset-10-binutils zlib-static ncurses-static \ - devtoolset-7-gcc-c++ \ - devtoolset-9-gcc-c++ \ - devtoolset-10-gcc-c++ \ - snappy snappy-devel \ - zlib zlib-devel \ - lz4-devel \ - libasan && \ - yum clean all && \ - rm -rf /var/cache/yum" - -COPY . /io/ - -# Create and set the working directory to /io -WORKDIR /io - -# Set file permissions mask -RUN /hbb_exe/activate-exec bash -c "umask 0000" - -ENV ZSTD_VERSION="1.1.3" \ - BOOST_ROOT="/usr/local/src/boost_1_75_0" \ - Boost_LIBRARY_DIRS="/usr/local/lib" \ - BOOST_INCLUDEDIR="/usr/local/src/boost_1_75_0" \ - LLVM_DIR="/usr/lib64/llvm13/lib/cmake/llvm/" \ - LLVM_LIBRARY_DIR="/usr/lib64/llvm13/lib/" - -# Install ZStd 1.1.3 -RUN /hbb_exe/activate-exec bash -c "source /opt/rh/devtoolset-7/enable && echo '-- Install ZStd 1.1.3 --' && \ - yum install epel-release -y && \ - wget -nc -q -O zstd-${ZSTD_VERSION}.tar.gz https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz && \ - tar xzvf zstd-${ZSTD_VERSION}.tar.gz && \ - cd zstd-${ZSTD_VERSION} && \ - make -j$(nproc) install && \ - cd .. && \ - rm -rf zstd-${ZSTD_VERSION} zstd-${ZSTD_VERSION}.tar.gz" - -# Install Cmake 3.23.1 -RUN /hbb_exe/activate-exec bash -c "source /opt/rh/devtoolset-7/enable && echo '-- Install Cmake 3.23.1 --' && \ - wget -nc -q https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1-linux-x86_64.tar.gz && \ - tar -xzf cmake-3.23.1-linux-x86_64.tar.gz -C /hbb/ && \ - rm cmake-3.23.1-linux-x86_64.tar.gz" - -# Install Boost 1.75.0 -RUN /hbb_exe/activate-exec bash -c "source /opt/rh/devtoolset-7/enable && echo '-- Install Boost 1.75.0 --' && \ - wget -nc -q https://boostorg.jfrog.io/artifactory/main/release/1.75.0/source/boost_1_75_0.tar.gz && \ - tar -xzf boost_1_75_0.tar.gz && \ - cd boost_1_75_0 && \ - ./bootstrap.sh && \ - ./b2 link=static -j$(nproc) && \ - ./b2 install && \ - cd ../ && \ - rm -rf boost_1_75_0 boost_1_75_0.tar.gz" - -# Install Protobuf 3.20.0 -RUN /hbb_exe/activate-exec bash -c "source /opt/rh/devtoolset-7/enable && echo '-- Install Protobuf 3.20.0 --' && \ - wget -nc -q https://github.com/protocolbuffers/protobuf/releases/download/v3.20.0/protobuf-all-3.20.0.tar.gz && \ - tar -xzf protobuf-all-3.20.0.tar.gz && \ - cd protobuf-3.20.0/ && \ - ./autogen.sh && \ - ./configure --prefix=/usr --disable-shared link=static && \ - make -j$(nproc) && \ - make install && \ - cd .. && \ - rm -rf protobuf-3.20.0 protobuf-all-3.20.0.tar.gz" - -# Build LLD -RUN /hbb_exe/activate-exec bash -c "source /opt/rh/devtoolset-7/enable && echo '-- Build LLD --' && \ - ln /usr/bin/llvm-config-13 /usr/bin/llvm-config && \ - mv /opt/rh/devtoolset-9/root/usr/bin/ar /opt/rh/devtoolset-9/root/usr/bin/ar-9 && \ - ln /opt/rh/devtoolset-10/root/usr/bin/ar /opt/rh/devtoolset-9/root/usr/bin/ar && \ - wget -nc -q https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.1/lld-13.0.1.src.tar.xz && \ - wget -nc -q https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.1/libunwind-13.0.1.src.tar.xz && \ - tar -xf lld-13.0.1.src.tar.xz && \ - tar -xf libunwind-13.0.1.src.tar.xz && \ - cp -r libunwind-13.0.1.src/include libunwind-13.0.1.src/src lld-13.0.1.src/ && \ - cd lld-13.0.1.src && \ - rm -rf build CMakeCache.txt && \ - mkdir build && \ - cd build && \ - cmake .. -DLLVM_LIBRARY_DIR=/usr/lib64/llvm13/lib/ -DCMAKE_INSTALL_PREFIX=/usr/lib64/llvm13/ -DCMAKE_BUILD_TYPE=Release && \ - make -j$(nproc) install && \ - ln -s /usr/lib64/llvm13/lib/include/lld /usr/include/lld && \ - cp /usr/lib64/llvm13/lib/liblld*.a /usr/local/lib/ && \ - cd ../../ && \ - rm -rf lld-13.0.1.src libunwind-13.0.1.src lld-13.0.1.src.tar.xz libunwind-13.0.1.src.tar.xz" - -# Build WasmEdge -RUN /hbb_exe/activate-exec bash -c "source /opt/rh/devtoolset-9/enable && echo '-- Build WasmEdge --' && \ - wget -nc -q https://github.com/WasmEdge/WasmEdge/archive/refs/tags/0.11.2.zip && \ - unzip -o 0.11.2.zip && \ - cd WasmEdge-0.11.2 && \ - mkdir build && \ - cd build && \ - cmake .. \ - -DCMAKE_BUILD_TYPE=Release \ - -DWASMEDGE_BUILD_SHARED_LIB=OFF \ - -DWASMEDGE_BUILD_STATIC_LIB=ON \ - -DWASMEDGE_BUILD_AOT_RUNTIME=ON \ - -DWASMEDGE_FORCE_DISABLE_LTO=ON \ - -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ - -DWASMEDGE_LINK_LLVM_STATIC=ON \ - -DWASMEDGE_BUILD_PLUGINS=OFF \ - -DWASMEDGE_LINK_TOOLS_STATIC=ON \ - -DBoost_NO_BOOST_CMAKE=ON -DLLVM_DIR=/usr/lib64/llvm13/lib/cmake/llvm/ -DLLVM_LIBRARY_DIR=/usr/lib64/llvm13/lib/ && \ - make -j$(nproc) install && \ - cp -r include/api/wasmedge /usr/include/ && \ - cd ../../ && \ - rm -rf WasmEdge-0.11.2 0.11.2.zip" - -# Set the entrypoint to the activate-exec script with bash -# This will be the default command that runs when the container starts -ENTRYPOINT ["/hbb_exe/activate-exec"] -CMD ["bash"] \ No newline at end of file diff --git a/build-core.sh b/build-core.sh index cf1abbcd5..6d95ccea1 100755 --- a/build-core.sh +++ b/build-core.sh @@ -12,7 +12,14 @@ umask 0000; cd /io/ && echo "Importing env... Lines:" && cat .env|wc -l && -source .env && +source .env + +echo $? +if [[ "$?" -ne "0" ]]; then + echo "ERR no .env found/sourced" + exit 127 +fi + perl -i -pe "s/^(\\s*)-DBUILD_SHARED_LIBS=OFF/\\1-DBUILD_SHARED_LIBS=OFF\\n\\1-DROCKSDB_BUILD_SHARED=OFF/g" Builds/CMake/deps/Rocksdb.cmake && mv Builds/CMake/deps/WasmEdge.cmake Builds/CMake/deps/WasmEdge.old && echo "find_package(LLVM REQUIRED CONFIG) @@ -53,6 +60,7 @@ else fi cd ..; + mv src/ripple/net/impl/RegisterSSLCerts.cpp.old src/ripple/net/impl/RegisterSSLCerts.cpp; mv Builds/CMake/deps/Rocksdb.cmake.old Builds/CMake/deps/Rocksdb.cmake; mv Builds/CMake/deps/WasmEdge.old Builds/CMake/deps/WasmEdge.cmake; diff --git a/build-full.sh b/build-full.sh index cad1c760c..b9d7b00ed 100755 --- a/build-full.sh +++ b/build-full.sh @@ -152,6 +152,13 @@ echo "Persisting ENV:" cat .env ./build-core.sh "$1" "$2" "$3" "$4" + +echo $? +if [[ "$?" -ne "0" ]]; then + echo "ERR build-core.sh non 0 exit code" + exit 127 +fi + echo "END [ build-core.sh ]" echo "END INSIDE CONTAINER - FULL" diff --git a/release.sh b/release.sh deleted file mode 100755 index c0cf160a2..000000000 --- a/release.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -set -e -umask 0000; - -# Create a temporary container -container_id=$(docker create transia/xahaud-binary:$3) - -# Copy the xahaud and release.info files from the container to the host -docker cp $container_id:/io/release-build/xahaud /data/builds/$(date +%Y).$(date +%-m).$(date +%-d)-$1+$2 -docker cp $container_id:/io/release-build/release.info /data/builds/$(date +%Y).$(date +%-m).$(date +%-d)-$1+$2.releaseinfo - -# Remove the temporary container -docker rm $container_id - -# Print the published build -echo "Published build to: http://build.xahau.tech/" -echo $(date +%Y).$(date +%-m).$(date +%-d)-$1+$2 \ No newline at end of file diff --git a/src/test/rpc/ServerDefinitions_test.cpp b/src/test/rpc/ServerDefinitions_test.cpp index da0b5f9d4..166681c90 100644 --- a/src/test/rpc/ServerDefinitions_test.cpp +++ b/src/test/rpc/ServerDefinitions_test.cpp @@ -302,7 +302,6 @@ public: "temDST_IS_SRC" : -279, "temDST_NEEDED" : -278, "temHOOK_DATA_TOO_LARGE" : -253, - "temHOOK_REJECTED" : -252, "temINVALID" : -277, "temINVALID_ACCOUNT_ID" : -268, "temINVALID_COUNT" : -266, diff --git a/ssl_script.sh b/ssl_script.sh deleted file mode 100755 index 2b86ca014..000000000 --- a/ssl_script.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -if [ "`grep certbundle.h src/ripple/net/impl/RegisterSSLCerts.cpp | wc -l`" -eq "0" ] -then - cp src/ripple/net/impl/RegisterSSLCerts.cpp src/ripple/net/impl/RegisterSSLCerts.cpp.old - perl -i -pe "s/^{/{ - #ifdef EMBEDDED_CA_BUNDLE - BIO *cbio = BIO_new_mem_buf(ca_bundle.data(), ca_bundle.size()); - X509_STORE *cts = SSL_CTX_get_cert_store(ctx.native_handle()); - if(!cts || !cbio) - JLOG(j.warn()) - << \"Failed to create cts\/cbio when loading embedded certs.\"; - else - { - X509_INFO *itmp; - int i, count = 0, type = X509_FILETYPE_PEM; - STACK_OF(X509_INFO) *inf = PEM_X509_INFO_read_bio(cbio, NULL, NULL, NULL); - - if (!inf) - { - BIO_free(cbio); - JLOG(j.warn()) - << \"Failed to read cbio when loading embedded certs.\"; - } - else - { - for (i = 0; i < sk_X509_INFO_num(inf); i++) - { - itmp = sk_X509_INFO_value(inf, i); - if (itmp->x509) - { - X509_STORE_add_cert(cts, itmp->x509); - count++; - } - if (itmp->crl) - { - X509_STORE_add_crl(cts, itmp->crl); - count++; - } - } - sk_X509_INFO_pop_free(inf, X509_INFO_free); - BIO_free(cbio); - } - } - #endif/g" src/ripple/net/impl/RegisterSSLCerts.cpp && - sed -i "s/#include /\0\n#include /g" src/ripple/net/impl/RegisterSSLCerts.cpp -fi \ No newline at end of file diff --git a/xahaud.binary.dockerfile b/xahaud.binary.dockerfile deleted file mode 100644 index d84ed16dd..000000000 --- a/xahaud.binary.dockerfile +++ /dev/null @@ -1,58 +0,0 @@ -# docker build -f xahaud.binary.dockerfile -t transia/xahaud-binary . -# Use a base image that includes the necessary build tools and libraries -FROM transia/xahaud-hbb-deps - -ARG GITHUB_BRANCH -ARG GITHUB_RUN_NUMBER - -# Copy the project source code into the container -COPY . /io - -# Set the working directory -WORKDIR /io - -# Create directory for certificates -RUN mkdir -p src/certs - -# Download the certificate bundle -RUN curl --silent -k https://raw.githubusercontent.com/RichardAH/rippled-release-builder/main/ca-bundle/certbundle.h -o src/certs/certbundle.h - -# Check if certbundle.h needs to be included in RegisterSSLCerts.cpp -RUN ./ssl_script.sh - -# Set file permissions -RUN /hbb_exe/activate-exec bash -c "umask 0000" - -# Modify Rocksdb.cmake -# RUN cp Builds/CMake/deps/Rocksdb.cmake Builds/CMake/deps/Rocksdb.cmake.old -RUN perl -i -pe "s/^(\\s*)-DBUILD_SHARED_LIBS=OFF/\\1-DBUILD_SHARED_LIBS=OFF\\n\\1-DROCKSDB_BUILD_SHARED=OFF/g" Builds/CMake/deps/Rocksdb.cmake - -# Replace WasmEdge.cmake with a new configuration -RUN /hbb_exe/activate-exec bash -c "echo -e 'find_package(LLVM REQUIRED CONFIG)\nmessage(STATUS \"Found LLVM ${LLVM_PACKAGE_VERSION}\")\nmessage(STATUS \"Using LLVMConfig.cmake in: \${LLVM_DIR}\")\nadd_library (wasmedge STATIC IMPORTED GLOBAL)\nset_target_properties(wasmedge PROPERTIES IMPORTED_LOCATION \${WasmEdge_LIB})\ntarget_link_libraries (ripple_libs INTERFACE wasmedge)\nadd_library (NIH::WasmEdge ALIAS wasmedge)\nmessage(\"WasmEdge DONE\")' > Builds/CMake/deps/WasmEdge.cmake" - -# Update BuildInfo.cpp with the current date and Git information -RUN sed -i s/\"0.0.0\"/\"$(date +%Y).$(date +%-m).$(date +%-d)-${GITHUB_BRANCH}+${GITHUB_RUN_NUMBER}\"/g src/ripple/protocol/impl/BuildInfo.cpp - -# Create build directory -RUN /hbb_exe/activate-exec bash -c "mkdir -p release-build" - -# Configure the build with CMake -RUN /hbb_exe/activate-exec bash -c "cd release-build && source /opt/rh/devtoolset-10/enable && cmake .. -DCMAKE_BUILD_TYPE=Release -DBoost_NO_BOOST_CMAKE=ON -DLLVM_DIR=/usr/lib64/llvm13/lib/cmake/llvm/ -DLLVM_LIBRARY_DIR=/usr/lib64/llvm13/lib/ -DWasmEdge_LIB=/usr/local/lib64/libwasmedge.a && make -j$(nproc) VERBOSE=1" - -# Strip the binary and rename it -RUN /hbb_exe/activate-exec bash -c "cd release-build && strip -s rippled && mv rippled xahaud" - -# Create release information -RUN cd release-build && \ - echo "Build host: $(hostname)" > release.info && \ - echo "Build date: $(date)" >> release.info && \ - echo "Build md5: $(md5sum xahaud)" >> release.info && \ - echo "Git remotes:" >> release.info && \ - git remote -v >> release.info && \ - echo "Git status:" >> release.info && \ - git status -v >> release.info && \ - echo "Git log [last 20]:" >> release.info && \ - git log -n 20 >> release.info - -ENTRYPOINT ["/hbb_exe/activate-exec"] -CMD ["bash"] \ No newline at end of file