From 99b00a6c1b5f14c6a06dae9968b665ca588fd9f1 Mon Sep 17 00:00:00 2001 From: Wietse Wind Date: Tue, 31 Jan 2023 01:15:33 +0100 Subject: [PATCH 1/8] Add self hosted build runner --- .github/workflows/docker-image.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .github/workflows/docker-image.yml diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml new file mode 100644 index 000000000..98b9dfb0f --- /dev/null +++ b/.github/workflows/docker-image.yml @@ -0,0 +1,15 @@ +name: Build using Docker + +on: + push: + branches: [ "dev" ] + pull_request: + branches: [ "dev" ] + +jobs: + build: + runs-on: self-hosted + steps: + - uses: actions/checkout@v3 + - name: Build using Docker + run: /bin/bash release-builder.sh From 2f42ddb2851ba7420d1aa768a960851134e2c5e9 Mon Sep 17 00:00:00 2001 From: Wietse Wind Date: Tue, 31 Jan 2023 01:20:16 +0100 Subject: [PATCH 2/8] Silent downloads, no TTY for Docker, current user for Docker --- release-builder.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/release-builder.sh b/release-builder.sh index f1d0cfb3c..0d5c0e530 100755 --- a/release-builder.sh +++ b/release-builder.sh @@ -12,10 +12,10 @@ then echo 'Run this inside the rippled directory. (.git dir not found).' exit 1 fi -docker run -t -i --rm -v `pwd`:/io --network host ghcr.io/foobarwidget/holy-build-box-x64 /hbb_exe/activate-exec bash -x -c ' +docker run --user $(id -u):$(id -g) --rm -v `pwd`:/io --network host ghcr.io/foobarwidget/holy-build-box-x64 /hbb_exe/activate-exec bash -x -c ' cd /io; mkdir src/certs; -curl -k https://raw.githubusercontent.com/RichardAH/rippled-release-builder/main/ca-bundle/certbundle.h -o src/certs/certbundle.h; +curl --silent -k https://raw.githubusercontent.com/RichardAH/rippled-release-builder/main/ca-bundle/certbundle.h -o src/certs/certbundle.h; if [ "`grep certbundle.h src/ripple/net/impl/RegisterSSLCerts.cpp | wc -l`" -eq "0" ] then cp src/ripple/net/impl/RegisterSSLCerts.cpp src/ripple/net/impl/RegisterSSLCerts.cpp.old @@ -78,24 +78,24 @@ export PATH=`echo $PATH | sed -E "s/devtoolset-9/devtoolset-7/g"` && echo "-- Install ZStd 1.1.3 --" && yum install epel-release -y && ZSTD_VERSION="1.1.3" && -( wget -nc -O zstd-${ZSTD_VERSION}.tar.gz https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz; echo "" ) && +( wget -ncq -O zstd-${ZSTD_VERSION}.tar.gz https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz; echo "" ) && tar xzvf zstd-${ZSTD_VERSION}.tar.gz && cd zstd-${ZSTD_VERSION} && make -j8 install && cd .. && echo "-- Install Cmake 3.23.1 --" && pwd && -( wget -nc https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1-linux-x86_64.tar.gz; echo "" ) && +( wget -ncq https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1-linux-x86_64.tar.gz; echo "" ) && tar -xzf cmake-3.23.1-linux-x86_64.tar.gz -C /hbb/ && echo "-- Install Boost 1.75.0 --" && pwd && -( wget -nc https://boostorg.jfrog.io/artifactory/main/release/1.75.0/source/boost_1_75_0.tar.gz; echo "" ) && +( wget -ncq https://boostorg.jfrog.io/artifactory/main/release/1.75.0/source/boost_1_75_0.tar.gz; echo "" ) && tar -xzf boost_1_75_0.tar.gz && cd boost_1_75_0 && ./bootstrap.sh && ./b2 link=static -j8 && ./b2 install && cd ../ && echo "-- Install Protobuf 3.20.0 --" && pwd && -( wget -nc https://github.com/protocolbuffers/protobuf/releases/download/v3.20.0/protobuf-all-3.20.0.tar.gz; echo "" ) && +( wget -ncq https://github.com/protocolbuffers/protobuf/releases/download/v3.20.0/protobuf-all-3.20.0.tar.gz; echo "" ) && tar -xzf protobuf-all-3.20.0.tar.gz && cd protobuf-3.20.0/ && ./autogen.sh && ./configure --prefix=/usr --disable-shared link=static && make -j8 && make install && @@ -105,8 +105,8 @@ pwd && ln /usr/bin/llvm-config-13 /usr/bin/llvm-config && mv /opt/rh/devtoolset-9/root/usr/bin/ar /opt/rh/devtoolset-9/root/usr/bin/ar-9 && ln /opt/rh/devtoolset-10/root/usr/bin/ar /opt/rh/devtoolset-9/root/usr/bin/ar && -( wget -nc https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.1/lld-13.0.1.src.tar.xz; echo "" ) && -( wget -nc https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.1/libunwind-13.0.1.src.tar.xz; echo "" ) && +( wget -ncq https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.1/lld-13.0.1.src.tar.xz; echo "" ) && +( wget -ncq https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.1/libunwind-13.0.1.src.tar.xz; echo "" ) && tar -xf lld-13.0.1.src.tar.xz && tar -xf libunwind-13.0.1.src.tar.xz && cp -r libunwind-13.0.1.src/include libunwind-13.0.1.src/src lld-13.0.1.src/ && @@ -120,7 +120,7 @@ ln -s /usr/lib64/llvm13/lib/include/lld /usr/include/lld && cp /usr/lib64/llvm13/lib/liblld*.a /usr/local/lib/ && cd ../../ && echo "-- Build WasmEdge --" && -( wget -nc https://github.com/WasmEdge/WasmEdge/archive/refs/tags/0.11.2.zip; unzip -o 0.11.2.zip; ) && +( wget -ncq https://github.com/WasmEdge/WasmEdge/archive/refs/tags/0.11.2.zip; unzip -o 0.11.2.zip; ) && cd WasmEdge-0.11.2 && ( mkdir build; echo "" ) && cd build && From 5c80fb3669c921e27f0e312f9ef3bad32b1da412 Mon Sep 17 00:00:00 2001 From: Wietse Wind Date: Tue, 31 Jan 2023 01:27:55 +0100 Subject: [PATCH 3/8] Support needing root perms to build & runner group --- release-builder.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/release-builder.sh b/release-builder.sh index 0d5c0e530..64a37bde9 100755 --- a/release-builder.sh +++ b/release-builder.sh @@ -12,7 +12,8 @@ then echo 'Run this inside the rippled directory. (.git dir not found).' exit 1 fi -docker run --user $(id -u):$(id -g) --rm -v `pwd`:/io --network host ghcr.io/foobarwidget/holy-build-box-x64 /hbb_exe/activate-exec bash -x -c ' +docker run --user 0:$(id -g) --rm -v `pwd`:/io --network host ghcr.io/foobarwidget/holy-build-box-x64 /hbb_exe/activate-exec bash -x -c ' +umask 0000; cd /io; mkdir src/certs; curl --silent -k https://raw.githubusercontent.com/RichardAH/rippled-release-builder/main/ca-bundle/certbundle.h -o src/certs/certbundle.h; From ecea635db53d10d672ab9307978d9a5c028c364e Mon Sep 17 00:00:00 2001 From: Wietse Wind Date: Tue, 31 Jan 2023 01:30:32 +0100 Subject: [PATCH 4/8] Quiet wget --- release-builder.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/release-builder.sh b/release-builder.sh index 64a37bde9..847951ea0 100755 --- a/release-builder.sh +++ b/release-builder.sh @@ -79,24 +79,24 @@ export PATH=`echo $PATH | sed -E "s/devtoolset-9/devtoolset-7/g"` && echo "-- Install ZStd 1.1.3 --" && yum install epel-release -y && ZSTD_VERSION="1.1.3" && -( wget -ncq -O zstd-${ZSTD_VERSION}.tar.gz https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz; echo "" ) && +( wget -nc -q -O zstd-${ZSTD_VERSION}.tar.gz https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz; echo "" ) && tar xzvf zstd-${ZSTD_VERSION}.tar.gz && cd zstd-${ZSTD_VERSION} && make -j8 install && cd .. && echo "-- Install Cmake 3.23.1 --" && pwd && -( wget -ncq https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1-linux-x86_64.tar.gz; echo "" ) && +( wget -nc -q https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1-linux-x86_64.tar.gz; echo "" ) && tar -xzf cmake-3.23.1-linux-x86_64.tar.gz -C /hbb/ && echo "-- Install Boost 1.75.0 --" && pwd && -( wget -ncq https://boostorg.jfrog.io/artifactory/main/release/1.75.0/source/boost_1_75_0.tar.gz; echo "" ) && +( wget -nc -q https://boostorg.jfrog.io/artifactory/main/release/1.75.0/source/boost_1_75_0.tar.gz; echo "" ) && tar -xzf boost_1_75_0.tar.gz && cd boost_1_75_0 && ./bootstrap.sh && ./b2 link=static -j8 && ./b2 install && cd ../ && echo "-- Install Protobuf 3.20.0 --" && pwd && -( wget -ncq https://github.com/protocolbuffers/protobuf/releases/download/v3.20.0/protobuf-all-3.20.0.tar.gz; echo "" ) && +( wget -nc -q https://github.com/protocolbuffers/protobuf/releases/download/v3.20.0/protobuf-all-3.20.0.tar.gz; echo "" ) && tar -xzf protobuf-all-3.20.0.tar.gz && cd protobuf-3.20.0/ && ./autogen.sh && ./configure --prefix=/usr --disable-shared link=static && make -j8 && make install && @@ -106,8 +106,8 @@ pwd && ln /usr/bin/llvm-config-13 /usr/bin/llvm-config && mv /opt/rh/devtoolset-9/root/usr/bin/ar /opt/rh/devtoolset-9/root/usr/bin/ar-9 && ln /opt/rh/devtoolset-10/root/usr/bin/ar /opt/rh/devtoolset-9/root/usr/bin/ar && -( wget -ncq https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.1/lld-13.0.1.src.tar.xz; echo "" ) && -( wget -ncq https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.1/libunwind-13.0.1.src.tar.xz; echo "" ) && +( wget -nc -q https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.1/lld-13.0.1.src.tar.xz; echo "" ) && +( wget -nc -q https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.1/libunwind-13.0.1.src.tar.xz; echo "" ) && tar -xf lld-13.0.1.src.tar.xz && tar -xf libunwind-13.0.1.src.tar.xz && cp -r libunwind-13.0.1.src/include libunwind-13.0.1.src/src lld-13.0.1.src/ && @@ -121,7 +121,7 @@ ln -s /usr/lib64/llvm13/lib/include/lld /usr/include/lld && cp /usr/lib64/llvm13/lib/liblld*.a /usr/local/lib/ && cd ../../ && echo "-- Build WasmEdge --" && -( wget -ncq https://github.com/WasmEdge/WasmEdge/archive/refs/tags/0.11.2.zip; unzip -o 0.11.2.zip; ) && +( wget -nc -q https://github.com/WasmEdge/WasmEdge/archive/refs/tags/0.11.2.zip; unzip -o 0.11.2.zip; ) && cd WasmEdge-0.11.2 && ( mkdir build; echo "" ) && cd build && From 8aa07383ec1db060fb5a8868bcf61aec70df903b Mon Sep 17 00:00:00 2001 From: Wietse Wind Date: Tue, 31 Jan 2023 01:41:12 +0100 Subject: [PATCH 5/8] Sep. build script: host vs container --- .dockerignore | 7 ++ build-inner.sh | 168 ++++++++++++++++++++++++++++++++++++++++++++ release-builder.sh | 169 ++------------------------------------------- 3 files changed, 181 insertions(+), 163 deletions(-) create mode 100644 .dockerignore create mode 100755 build-inner.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..2f1492e3c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +# Ignore everything +* + +# Allow files and directories +!/build-inner.sh +!/release-builder.sh + diff --git a/build-inner.sh b/build-inner.sh new file mode 100755 index 000000000..87917711e --- /dev/null +++ b/build-inner.sh @@ -0,0 +1,168 @@ +#!/bin/bash + +echo "START INSIDE CONTAINER" + +umask 0000; +cd /io; +mkdir src/certs; +curl --silent -k https://raw.githubusercontent.com/RichardAH/rippled-release-builder/main/ca-bundle/certbundle.h -o src/certs/certbundle.h; +if [ "`grep certbundle.h src/ripple/net/impl/RegisterSSLCerts.cpp | wc -l`" -eq "0" ] +then + cp src/ripple/net/impl/RegisterSSLCerts.cpp src/ripple/net/impl/RegisterSSLCerts.cpp.old + perl -i -pe "s/^{/{ + #ifdef EMBEDDED_CA_BUNDLE + BIO *cbio = BIO_new_mem_buf(ca_bundle.data(), ca_bundle.size()); + X509_STORE *cts = SSL_CTX_get_cert_store(ctx.native_handle()); + if(!cts || !cbio) + JLOG(j.warn()) + << \"Failed to create cts\/cbio when loading embedded certs.\"; + else + { + X509_INFO *itmp; + int i, count = 0, type = X509_FILETYPE_PEM; + STACK_OF(X509_INFO) *inf = PEM_X509_INFO_read_bio(cbio, NULL, NULL, NULL); + + if (!inf) + { + BIO_free(cbio); + JLOG(j.warn()) + << \"Failed to read cbio when loading embedded certs.\"; + } + else + { + for (i = 0; i < sk_X509_INFO_num(inf); i++) + { + itmp = sk_X509_INFO_value(inf, i); + if (itmp->x509) + { + X509_STORE_add_cert(cts, itmp->x509); + count++; + } + if (itmp->crl) + { + X509_STORE_add_crl(cts, itmp->crl); + count++; + } + } + sk_X509_INFO_pop_free(inf, X509_INFO_free); + BIO_free(cbio); + } + } + #endif/g" src/ripple/net/impl/RegisterSSLCerts.cpp && + sed -i "s/#include /\0\n#include /g" src/ripple/net/impl/RegisterSSLCerts.cpp +fi +rm -rf release-build; +mkdir release-build; +mkdir .nih_c; +mkdir .nih_toolchain; +cd .nih_toolchain && +yum install -y wget lz4 lz4-devel git llvm13-static.x86_64 llvm13-devel.x86_64 devtoolset-10-binutils zlib-static ncurses-static -y \ + devtoolset-7-gcc-c++ \ + devtoolset-9-gcc-c++ \ + devtoolset-10-gcc-c++ \ + snappy snappy-devel \ + zlib zlib-devel \ + lz4-devel \ + libasan && +export PATH=`echo $PATH | sed -E "s/devtoolset-9/devtoolset-7/g"` && +echo "-- Install ZStd 1.1.3 --" && +yum install epel-release -y && +ZSTD_VERSION="1.1.3" && +( wget -nc -q -O zstd-${ZSTD_VERSION}.tar.gz https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz; echo "" ) && +tar xzvf zstd-${ZSTD_VERSION}.tar.gz && +cd zstd-${ZSTD_VERSION} && +make -j8 install && +cd .. && +echo "-- Install Cmake 3.23.1 --" && +pwd && +( wget -nc -q https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1-linux-x86_64.tar.gz; echo "" ) && +tar -xzf cmake-3.23.1-linux-x86_64.tar.gz -C /hbb/ && +echo "-- Install Boost 1.75.0 --" && +pwd && +( wget -nc -q https://boostorg.jfrog.io/artifactory/main/release/1.75.0/source/boost_1_75_0.tar.gz; echo "" ) && +tar -xzf boost_1_75_0.tar.gz && +cd boost_1_75_0 && ./bootstrap.sh && ./b2 link=static -j8 && ./b2 install && +cd ../ && +echo "-- Install Protobuf 3.20.0 --" && +pwd && +( wget -nc -q https://github.com/protocolbuffers/protobuf/releases/download/v3.20.0/protobuf-all-3.20.0.tar.gz; echo "" ) && +tar -xzf protobuf-all-3.20.0.tar.gz && +cd protobuf-3.20.0/ && +./autogen.sh && ./configure --prefix=/usr --disable-shared link=static && make -j8 && make install && +cd .. && +echo "-- Build LLD --" && +pwd && +ln /usr/bin/llvm-config-13 /usr/bin/llvm-config && +mv /opt/rh/devtoolset-9/root/usr/bin/ar /opt/rh/devtoolset-9/root/usr/bin/ar-9 && +ln /opt/rh/devtoolset-10/root/usr/bin/ar /opt/rh/devtoolset-9/root/usr/bin/ar && +( wget -nc -q https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.1/lld-13.0.1.src.tar.xz; echo "" ) && +( wget -nc -q https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.1/libunwind-13.0.1.src.tar.xz; echo "" ) && +tar -xf lld-13.0.1.src.tar.xz && +tar -xf libunwind-13.0.1.src.tar.xz && +cp -r libunwind-13.0.1.src/include libunwind-13.0.1.src/src lld-13.0.1.src/ && +cd lld-13.0.1.src && +rm -rf build CMakeCache.txt && +mkdir build && +cd build && +cmake .. -DLLVM_LIBRARY_DIR=/usr/lib64/llvm13/lib/ -DCMAKE_INSTALL_PREFIX=/usr/lib64/llvm13/ -DCMAKE_BUILD_TYPE=Release && +make -j8 install && +ln -s /usr/lib64/llvm13/lib/include/lld /usr/include/lld && +cp /usr/lib64/llvm13/lib/liblld*.a /usr/local/lib/ && +cd ../../ && +echo "-- Build WasmEdge --" && +( wget -nc -q https://github.com/WasmEdge/WasmEdge/archive/refs/tags/0.11.2.zip; unzip -o 0.11.2.zip; ) && +cd WasmEdge-0.11.2 && +( mkdir build; echo "" ) && +cd build && +export BOOST_ROOT="/usr/local/src/boost_1_75_0" && +export Boost_LIBRARY_DIRS="/usr/local/lib" && +export BOOST_INCLUDEDIR="/usr/local/src/boost_1_75_0" && +export PATH=`echo $PATH | sed -E "s/devtoolset-7/devtoolset-9/g"` && +cmake .. \ + -DWASMEDGE_BUILD_SHARED_LIB=OFF \ + -DWASMEDGE_BUILD_STATIC_LIB=ON \ + -DWASMEDGE_BUILD_AOT_RUNTIME=ON \ + -DWASMEDGE_FORCE_DISABLE_LTO=ON \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DWASMEDGE_LINK_LLVM_STATIC=ON \ + -DWASMEDGE_BUILD_PLUGINS=OFF \ + -DWASMEDGE_LINK_TOOLS_STATIC=ON \ + -DBoost_NO_BOOST_CMAKE=ON -DLLVM_DIR=/usr/lib64/llvm13/lib/cmake/llvm/ -DLLVM_LIBRARY_DIR=/usr/lib64/llvm13/lib/ && +make -j8 install && +export PATH=`echo $PATH | sed -E "s/devtoolset-9/devtoolset-10/g"` && +cp -r include/api/wasmedge /usr/include/ && +cd /io/ && +echo "-- Build Rippled --" && +pwd && +cp Builds/CMake/deps/Rocksdb.cmake Builds/CMake/deps/Rocksdb.cmake.old && +perl -i -pe "s/^(\\s*)-DBUILD_SHARED_LIBS=OFF/\\1-DBUILD_SHARED_LIBS=OFF\\n\\1-DROCKSDB_BUILD_SHARED=OFF/g" Builds/CMake/deps/Rocksdb.cmake && +mv Builds/CMake/deps/WasmEdge.cmake Builds/CMake/deps/WasmEdge.old && +echo "find_package(LLVM REQUIRED CONFIG) +message(STATUS \"Found LLVM ${LLVM_PACKAGE_VERSION}\") +message(STATUS \"Using LLVMConfig.cmake in: \${LLVM_DIR}\") +add_library (wasmedge STATIC IMPORTED GLOBAL) +set_target_properties(wasmedge PROPERTIES IMPORTED_LOCATION \${WasmEdge_LIB}) +target_link_libraries (ripple_libs INTERFACE wasmedge) +add_library (NIH::WasmEdge ALIAS wasmedge) +message(\"WasmEdge DONE\") +" > Builds/CMake/deps/WasmEdge.cmake && +cd release-build && +cmake .. -DBoost_NO_BOOST_CMAKE=ON -DLLVM_DIR=/usr/lib64/llvm13/lib/cmake/llvm/ -DLLVM_LIBRARY_DIR=/usr/lib64/llvm13/lib/ -DWasmEdge_LIB=/usr/local/lib64/libwasmedge.a && +make -j8 VERBOSE=1 && +strip -s rippled && +mv rippled xahaud && +echo "Build host: `hostname`" > release.info && +echo "Build date: `date`" >> release.info && +echo "Build md5: `md5sum xahaud`" >> release.info && +echo "Git remotes:" >> release.info && +git remote -v >> release.info +echo "Git status:" >> release.info && +git status -v >> release.info && +echo "Git log [last 20]:" >> release.info && +git log -n 20 >> release.info; +cd ..; +mv src/ripple/net/impl/RegisterSSLCerts.cpp.old src/ripple/net/impl/RegisterSSLCerts.cpp; +mv Builds/CMake/deps/Rocksdb.cmake.old Builds/CMake/deps/Rocksdb.cmake; +mv Builds/CMake/deps/WasmEdge.old Builds/CMake/deps/WasmEdge.cmake; + +echo "END INSIDE CONTAINER" diff --git a/release-builder.sh b/release-builder.sh index 847951ea0..35aa01511 100755 --- a/release-builder.sh +++ b/release-builder.sh @@ -1,4 +1,7 @@ #!/bin/bash + +echo "START BUILDING (HOST)" + which docker 2> /dev/null 2> /dev/null if [ "$?" -eq "1" ] then @@ -9,169 +12,9 @@ fi stat .git 2> /dev/null 2> /dev/null if [ "$?" -eq "1" ] then - echo 'Run this inside the rippled directory. (.git dir not found).' + echo 'Run this inside the source directory. (.git dir not found).' exit 1 fi -docker run --user 0:$(id -g) --rm -v `pwd`:/io --network host ghcr.io/foobarwidget/holy-build-box-x64 /hbb_exe/activate-exec bash -x -c ' -umask 0000; -cd /io; -mkdir src/certs; -curl --silent -k https://raw.githubusercontent.com/RichardAH/rippled-release-builder/main/ca-bundle/certbundle.h -o src/certs/certbundle.h; -if [ "`grep certbundle.h src/ripple/net/impl/RegisterSSLCerts.cpp | wc -l`" -eq "0" ] -then - cp src/ripple/net/impl/RegisterSSLCerts.cpp src/ripple/net/impl/RegisterSSLCerts.cpp.old - perl -i -pe "s/^{/{ - #ifdef EMBEDDED_CA_BUNDLE - BIO *cbio = BIO_new_mem_buf(ca_bundle.data(), ca_bundle.size()); - X509_STORE *cts = SSL_CTX_get_cert_store(ctx.native_handle()); - if(!cts || !cbio) - JLOG(j.warn()) - << \"Failed to create cts\/cbio when loading embedded certs.\"; - else - { - X509_INFO *itmp; - int i, count = 0, type = X509_FILETYPE_PEM; - STACK_OF(X509_INFO) *inf = PEM_X509_INFO_read_bio(cbio, NULL, NULL, NULL); +docker run -i --user 0:$(id -g) --rm -v `pwd`:/io --network host ghcr.io/foobarwidget/holy-build-box-x64 /hbb_exe/activate-exec bash -x /io/build-inner.sh - if (!inf) - { - BIO_free(cbio); - JLOG(j.warn()) - << \"Failed to read cbio when loading embedded certs.\"; - } - else - { - for (i = 0; i < sk_X509_INFO_num(inf); i++) - { - itmp = sk_X509_INFO_value(inf, i); - if (itmp->x509) - { - X509_STORE_add_cert(cts, itmp->x509); - count++; - } - if (itmp->crl) - { - X509_STORE_add_crl(cts, itmp->crl); - count++; - } - } - sk_X509_INFO_pop_free(inf, X509_INFO_free); - BIO_free(cbio); - } - } - #endif/g" src/ripple/net/impl/RegisterSSLCerts.cpp && - sed -i "s/#include /\0\n#include /g" src/ripple/net/impl/RegisterSSLCerts.cpp -fi -rm -rf release-build; -mkdir release-build; -mkdir .nih_c; -mkdir .nih_toolchain; -cd .nih_toolchain && -yum install -y wget lz4 lz4-devel git llvm13-static.x86_64 llvm13-devel.x86_64 devtoolset-10-binutils zlib-static ncurses-static -y \ - devtoolset-7-gcc-c++ \ - devtoolset-9-gcc-c++ \ - devtoolset-10-gcc-c++ \ - snappy snappy-devel \ - zlib zlib-devel \ - lz4-devel \ - libasan && -export PATH=`echo $PATH | sed -E "s/devtoolset-9/devtoolset-7/g"` && -echo "-- Install ZStd 1.1.3 --" && -yum install epel-release -y && -ZSTD_VERSION="1.1.3" && -( wget -nc -q -O zstd-${ZSTD_VERSION}.tar.gz https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz; echo "" ) && -tar xzvf zstd-${ZSTD_VERSION}.tar.gz && -cd zstd-${ZSTD_VERSION} && -make -j8 install && -cd .. && -echo "-- Install Cmake 3.23.1 --" && -pwd && -( wget -nc -q https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1-linux-x86_64.tar.gz; echo "" ) && -tar -xzf cmake-3.23.1-linux-x86_64.tar.gz -C /hbb/ && -echo "-- Install Boost 1.75.0 --" && -pwd && -( wget -nc -q https://boostorg.jfrog.io/artifactory/main/release/1.75.0/source/boost_1_75_0.tar.gz; echo "" ) && -tar -xzf boost_1_75_0.tar.gz && -cd boost_1_75_0 && ./bootstrap.sh && ./b2 link=static -j8 && ./b2 install && -cd ../ && -echo "-- Install Protobuf 3.20.0 --" && -pwd && -( wget -nc -q https://github.com/protocolbuffers/protobuf/releases/download/v3.20.0/protobuf-all-3.20.0.tar.gz; echo "" ) && -tar -xzf protobuf-all-3.20.0.tar.gz && -cd protobuf-3.20.0/ && -./autogen.sh && ./configure --prefix=/usr --disable-shared link=static && make -j8 && make install && -cd .. && -echo "-- Build LLD --" && -pwd && -ln /usr/bin/llvm-config-13 /usr/bin/llvm-config && -mv /opt/rh/devtoolset-9/root/usr/bin/ar /opt/rh/devtoolset-9/root/usr/bin/ar-9 && -ln /opt/rh/devtoolset-10/root/usr/bin/ar /opt/rh/devtoolset-9/root/usr/bin/ar && -( wget -nc -q https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.1/lld-13.0.1.src.tar.xz; echo "" ) && -( wget -nc -q https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.1/libunwind-13.0.1.src.tar.xz; echo "" ) && -tar -xf lld-13.0.1.src.tar.xz && -tar -xf libunwind-13.0.1.src.tar.xz && -cp -r libunwind-13.0.1.src/include libunwind-13.0.1.src/src lld-13.0.1.src/ && -cd lld-13.0.1.src && -rm -rf build CMakeCache.txt && -mkdir build && -cd build && -cmake .. -DLLVM_LIBRARY_DIR=/usr/lib64/llvm13/lib/ -DCMAKE_INSTALL_PREFIX=/usr/lib64/llvm13/ -DCMAKE_BUILD_TYPE=Release && -make -j8 install && -ln -s /usr/lib64/llvm13/lib/include/lld /usr/include/lld && -cp /usr/lib64/llvm13/lib/liblld*.a /usr/local/lib/ && -cd ../../ && -echo "-- Build WasmEdge --" && -( wget -nc -q https://github.com/WasmEdge/WasmEdge/archive/refs/tags/0.11.2.zip; unzip -o 0.11.2.zip; ) && -cd WasmEdge-0.11.2 && -( mkdir build; echo "" ) && -cd build && -export BOOST_ROOT="/usr/local/src/boost_1_75_0" && -export Boost_LIBRARY_DIRS="/usr/local/lib" && -export BOOST_INCLUDEDIR="/usr/local/src/boost_1_75_0" && -export PATH=`echo $PATH | sed -E "s/devtoolset-7/devtoolset-9/g"` && -cmake .. \ - -DWASMEDGE_BUILD_SHARED_LIB=OFF \ - -DWASMEDGE_BUILD_STATIC_LIB=ON \ - -DWASMEDGE_BUILD_AOT_RUNTIME=ON \ - -DWASMEDGE_FORCE_DISABLE_LTO=ON \ - -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ - -DWASMEDGE_LINK_LLVM_STATIC=ON \ - -DWASMEDGE_BUILD_PLUGINS=OFF \ - -DWASMEDGE_LINK_TOOLS_STATIC=ON \ - -DBoost_NO_BOOST_CMAKE=ON -DLLVM_DIR=/usr/lib64/llvm13/lib/cmake/llvm/ -DLLVM_LIBRARY_DIR=/usr/lib64/llvm13/lib/ && -make -j8 install && -export PATH=`echo $PATH | sed -E "s/devtoolset-9/devtoolset-10/g"` && -cp -r include/api/wasmedge /usr/include/ && -cd /io/ && -echo "-- Build Rippled --" && -pwd && -cp Builds/CMake/deps/Rocksdb.cmake Builds/CMake/deps/Rocksdb.cmake.old && -perl -i -pe "s/^(\\s*)-DBUILD_SHARED_LIBS=OFF/\\1-DBUILD_SHARED_LIBS=OFF\\n\\1-DROCKSDB_BUILD_SHARED=OFF/g" Builds/CMake/deps/Rocksdb.cmake && -mv Builds/CMake/deps/WasmEdge.cmake Builds/CMake/deps/WasmEdge.old && -echo "find_package(LLVM REQUIRED CONFIG) -message(STATUS \"Found LLVM ${LLVM_PACKAGE_VERSION}\") -message(STATUS \"Using LLVMConfig.cmake in: \${LLVM_DIR}\") -add_library (wasmedge STATIC IMPORTED GLOBAL) -set_target_properties(wasmedge PROPERTIES IMPORTED_LOCATION \${WasmEdge_LIB}) -target_link_libraries (ripple_libs INTERFACE wasmedge) -add_library (NIH::WasmEdge ALIAS wasmedge) -message(\"WasmEdge DONE\") -" > Builds/CMake/deps/WasmEdge.cmake && -cd release-build && -cmake .. -DBoost_NO_BOOST_CMAKE=ON -DLLVM_DIR=/usr/lib64/llvm13/lib/cmake/llvm/ -DLLVM_LIBRARY_DIR=/usr/lib64/llvm13/lib/ -DWasmEdge_LIB=/usr/local/lib64/libwasmedge.a && -make -j8 VERBOSE=1 && -strip -s rippled && -mv rippled xahaud && -echo "Build host: `hostname`" > release.info && -echo "Build date: `date`" >> release.info && -echo "Build md5: `md5sum xahaud`" >> release.info && -echo "Git remotes:" >> release.info && -git remote -v >> release.info -echo "Git status:" >> release.info && -git status -v >> release.info && -echo "Git log [last 20]:" >> release.info && -git log -n 20 >> release.info; -cd ..; -mv src/ripple/net/impl/RegisterSSLCerts.cpp.old src/ripple/net/impl/RegisterSSLCerts.cpp; -mv Builds/CMake/deps/Rocksdb.cmake.old Builds/CMake/deps/Rocksdb.cmake; -mv Builds/CMake/deps/WasmEdge.old Builds/CMake/deps/WasmEdge.cmake;' +echo "DONE BUILDING (HOST)" From 70c694ea4a964d3e38519d3b51487875ea3f2289 Mon Sep 17 00:00:00 2001 From: Wietse Wind Date: Tue, 31 Jan 2023 01:44:30 +0100 Subject: [PATCH 6/8] Specify dedicated runner for builds --- .github/workflows/docker-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 98b9dfb0f..58f005d2d 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -8,7 +8,7 @@ on: jobs: build: - runs-on: self-hosted + runs-on: [self-hosted, vanity] steps: - uses: actions/checkout@v3 - name: Build using Docker From 15b449b1194182d68bcdebc607e561697e9b37b7 Mon Sep 17 00:00:00 2001 From: Wietse Wind Date: Tue, 31 Jan 2023 02:08:15 +0100 Subject: [PATCH 7/8] Add test brach --- .../workflows/{docker-image.yml => build-in-docker.yml} | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) rename .github/workflows/{docker-image.yml => build-in-docker.yml} (58%) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/build-in-docker.yml similarity index 58% rename from .github/workflows/docker-image.yml rename to .github/workflows/build-in-docker.yml index 58f005d2d..a4f6704f0 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/build-in-docker.yml @@ -2,7 +2,7 @@ name: Build using Docker on: push: - branches: [ "dev" ] + branches: [ "dev", "test" ] pull_request: branches: [ "dev" ] @@ -11,5 +11,11 @@ jobs: runs-on: [self-hosted, vanity] steps: - uses: actions/checkout@v3 + with: + clean: false - name: Build using Docker run: /bin/bash release-builder.sh + - uses: actions/upload-artifact@v3 + with: + name: build + path: /io/release-build/xahaud From 61750dcb0dbc44e19a4b68c208dbbf04e988053c Mon Sep 17 00:00:00 2001 From: Wietse Wind Date: Tue, 31 Jan 2023 02:39:46 +0100 Subject: [PATCH 8/8] Change artifact upload path --- .github/workflows/build-in-docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-in-docker.yml b/.github/workflows/build-in-docker.yml index a4f6704f0..28464a64f 100644 --- a/.github/workflows/build-in-docker.yml +++ b/.github/workflows/build-in-docker.yml @@ -18,4 +18,4 @@ jobs: - uses: actions/upload-artifact@v3 with: name: build - path: /io/release-build/xahaud + path: release-build/xahaud