Add full & partial release build

commit 2349db5aa9deec436d97169cd5ae58bc57f28093
Author: Wietse Wind <mail@wietse.com>
Date:   Tue Jan 31 13:37:58 2023 +0100

    Semver

commit 2969e6c31561be28fbb0979f21a61305ccb49be9
Author: Wietse Wind <mail@wietse.com>
Date:   Tue Jan 31 13:11:49 2023 +0100

    Small build works (?)

commit 3241f147c3dbbbf29d2a5c364b64bf42c0419450
Author: Wietse Wind <mail@wietse.com>
Date:   Tue Jan 31 13:01:32 2023 +0100

    Semver

commit 411f5c9c5042da16685d069cabe8318c239cf888
Author: Wietse Wind <mail@wietse.com>
Date:   Tue Jan 31 12:25:38 2023 +0100

    Start on path, need more info

commit 8ea7819bf7648cd81c434ff3b4826dd410047718
Author: Wietse Wind <mail@wietse.com>
Date:   Tue Jan 31 12:09:05 2023 +0100

    Change build, timestamp, branch, runner

commit 1579e540ff77b0517090a1ac7e32cd277f986d1e
Author: Richard Holland <richard.holland@starstone.co.nz>
Date:   Tue Jan 31 10:44:19 2023 +0000

    Change version string to 0.0.0 ahead of build numbers from GHActions

commit 1e78245e201bdeb1b238f23b6821da2ce8f3f171
Author: Wietse Wind <mail@wietse.com>
Date:   Tue Jan 31 11:39:26 2023 +0100

    Change compiler toolset

commit 47910f553ff5e4fe4ee90143ff097ac2e49ad281
Author: Wietse Wind <mail@wietse.com>
Date:   Tue Jan 31 11:35:17 2023 +0100

    Add docker unit tests

commit 7df50898d9dc8245fa766de74de0d4c85cf0259b
Author: Wietse Wind <mail@wietse.com>
Date:   Tue Jan 31 10:40:14 2023 +0100

    BUild full on new container

commit 174135fba27bbbf66d2b13c061851c47a9acb047
Author: Wietse Wind <mail@wietse.com>
Date:   Tue Jan 31 10:39:13 2023 +0100

    No full build on action runner

commit 96c7fe1f3e366a2eef9261d7f56b7889b1385daf
Author: Wietse Wind <mail@wietse.com>
Date:   Tue Jan 31 10:32:11 2023 +0100

    Fix auto clean

commit 2f7d707a6149181a7576cfab426e479c56713e97
Author: Wietse Wind <mail@wietse.com>
Date:   Tue Jan 31 10:25:39 2023 +0100

    clean container on GH action full build

commit 1b12c99b88ed253639826dc644af8d9b7fb16f72
Author: Wietse Wind <mail@wietse.com>
Date:   Tue Jan 31 10:25:02 2023 +0100

    Wrong logic

commit d0a08c27367ec7680a29e7ae2e0c64e1fa3866c4
Author: Wietse Wind <mail@wietse.com>
Date:   Tue Jan 31 10:23:17 2023 +0100

    Try to fix fast build

commit 1e2093821282aca904682942387645dabdc0dc1c
Author: Wietse Wind <mail@wietse.com>
Date:   Tue Jan 31 09:55:11 2023 +0100

    Persistent container

commit 68d9b4cf1c6e30c17a38e7259d71c642f3b7cd11
Author: Wietse Wind <mail@wietse.com>
Date:   Tue Jan 31 09:53:59 2023 +0100

    Persistent container

commit 651d5dec8aa4ab7cb4971861fbc67b9a3dc27313
Author: Wietse Wind <mail@wietse.com>
Date:   Tue Jan 31 09:46:32 2023 +0100

    Cached builder, unit tests

commit 50f7a901543faf34361b3d5d24826bad0dbb3b49
Author: Wietse Wind <mail@wietse.com>
Date:   Tue Jan 31 09:31:35 2023 +0100

    Add build cores

commit 841bf1c4fc0f432e35393f4bd393a772cc7f9ebc
Author: Wietse Wind <mail@wietse.com>
Date:   Tue Jan 31 09:26:07 2023 +0100

    Testing runner sh code

.github/workflows/build-in-docker.yml

@@ -17,5 +17,9 @@ jobs:
       run: /bin/bash release-builder.sh
     - uses: actions/upload-artifact@v3
       with:
-        name: build
+        name: build-${{ github.run_number }}
-        path: release-build/xahaud
+        path: |
+          release-build/xahaud
+          release-build/release.info
+    - name: Unit tests
+      run: /bin/bash docker-unit-tests.sh

build-core.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+
+echo "START INSIDE CONTAINER - CORE"
+
+echo "-- BUILD CORES:       $3"
+echo "-- GITHUB_REPOSITORY: $1"
+echo "-- GITHUB_SHA:        $2"
+echo "-- GITHUB_RUN_NUMBER: $4"
+
+umask 0000;
+
+export CPLUS_INCLUDE_PATH="/hbb_exe/include"
+export MANPATH="/opt/rh/devtoolset-10/root/usr/share/man:"
+export LDFLAGS="-L/hbb_exe/lib -static-libstdc++"
+export HOSTNAME="vanity"
+export CPPFLAGS="-I/hbb_exe/include"
+export LIBRARY_PATH="/hbb_exe/lib"
+export LDPATHFLAGS="-L/hbb_exe/lib"
+export STATICLIB_CFLAGS="-g -O2 -fvisibility=hidden -I/hbb_exe/include "
+export SHLIB_CXXFLAGS="-g -O2 -fvisibility=hidden -I/hbb_exe/include "
+export PCP_DIR="/opt/rh/devtoolset-10/root"
+export LD_LIBRARY_PATH="/hbb/lib:/hbb_exe/lib"
+export O3_ALLOWED="true"
+export CXXFLAGS="-g -O2 -fvisibility=hidden -I/hbb_exe/include "
+export PATH="/hbb_exe/bin:/hbb/bin:/opt/rh/devtoolset-10/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+export C_INCLUDE_PATH="/hbb_exe/include"
+export STATICLIB_CXXFLAGS="-g -O2 -fvisibility=hidden -I/hbb_exe/include "
+export SHLIB_LDFLAGS="-L/hbb_exe/lib -static-libstdc++"
+export BOOST_INCLUDEDIR="/usr/local/src/boost_1_75_0"
+export SHLVL="1"
+export SHLIB_CFLAGS="-g -O2 -fvisibility=hidden -I/hbb_exe/include "
+export CFLAGS="-g -O2 -fvisibility=hidden -I/hbb_exe/include "
+export BOOST_ROOT="/usr/local/src/boost_1_75_0"
+export PKG_CONFIG_PATH="/hbb_exe/lib/pkgconfig:/usr/lib/pkgconfig"
+export INFOPATH="/opt/rh/devtoolset-10/root/usr/share/info"
+export Boost_LIBRARY_DIRS="/usr/local/lib"
+
+cd /io/ &&
+perl -i -pe "s/^(\\s*)-DBUILD_SHARED_LIBS=OFF/\\1-DBUILD_SHARED_LIBS=OFF\\n\\1-DROCKSDB_BUILD_SHARED=OFF/g" Builds/CMake/deps/Rocksdb.cmake &&
+mv Builds/CMake/deps/WasmEdge.cmake Builds/CMake/deps/WasmEdge.old &&
+echo "find_package(LLVM REQUIRED CONFIG)
+message(STATUS \"Found LLVM ${LLVM_PACKAGE_VERSION}\")
+message(STATUS \"Using LLVMConfig.cmake in: \${LLVM_DIR}\")
+add_library (wasmedge STATIC IMPORTED GLOBAL)
+set_target_properties(wasmedge PROPERTIES IMPORTED_LOCATION \${WasmEdge_LIB})
+target_link_libraries (ripple_libs INTERFACE wasmedge)
+add_library (NIH::WasmEdge ALIAS wasmedge)
+message(\"WasmEdge DONE\")
+" > Builds/CMake/deps/WasmEdge.cmake &&
+git checkout src/ripple/protocol/impl/BuildInfo.cpp &&
+sed -i s/\"0.0.0\"/\"$(date +%Y).$(date +-%m).$(date +%-d)-$(git rev-parse --abbrev-ref HEAD)+$4\"/g src/ripple/protocol/impl/BuildInfo.cpp &&
+cd release-build &&
+cmake .. -DBoost_NO_BOOST_CMAKE=ON -DLLVM_DIR=/usr/lib64/llvm13/lib/cmake/llvm/ -DLLVM_LIBRARY_DIR=/usr/lib64/llvm13/lib/ -DWasmEdge_LIB=/usr/local/lib64/libwasmedge.a &&
+make -j$3 VERBOSE=1 &&
+strip -s rippled &&
+mv rippled xahaud &&
+echo "Build host: `hostname`" > release.info &&
+echo "Build date: `date`" >> release.info &&
+echo "Build md5: `md5sum xahaud`" >> release.info &&
+echo "Git remotes:" >> release.info && 
+git remote -v >> release.info 
+echo "Git status:" >> release.info &&
+git status -v >> release.info &&
+echo "Git log [last 20]:" >> release.info &&
+git log -n 20 >> release.info;
+cd ..;
+mv src/ripple/net/impl/RegisterSSLCerts.cpp.old src/ripple/net/impl/RegisterSSLCerts.cpp;
+mv Builds/CMake/deps/Rocksdb.cmake.old Builds/CMake/deps/Rocksdb.cmake;
+mv Builds/CMake/deps/WasmEdge.old Builds/CMake/deps/WasmEdge.cmake;
+
+echo "END INSIDE CONTAINER - CORE"

build-full.sh

@@ -1,8 +1,14 @@
 #!/bin/bash
 
-echo "START INSIDE CONTAINER"
+echo "START INSIDE CONTAINER - FULL"
+
+echo "-- BUILD CORES:       $3"
+echo "-- GITHUB_REPOSITORY: $1"
+echo "-- GITHUB_SHA:        $2"
+echo "-- GITHUB_RUN_NUMBER: $4"
 
 umask 0000;
+
 cd /io;
 mkdir src/certs;
 curl --silent -k https://raw.githubusercontent.com/RichardAH/rippled-release-builder/main/ca-bundle/certbundle.h -o src/certs/certbundle.h;
@@ -51,8 +57,6 @@ then
     #endif/g" src/ripple/net/impl/RegisterSSLCerts.cpp &&
     sed -i "s/#include <ripple\/net\/RegisterSSLCerts.h>/\0\n#include <certs\/certbundle.h>/g" src/ripple/net/impl/RegisterSSLCerts.cpp
 fi
-rm -rf release-build;
-mkdir release-build;
 mkdir .nih_c;
 mkdir .nih_toolchain;
 cd .nih_toolchain &&
@@ -71,7 +75,7 @@ ZSTD_VERSION="1.1.3" &&
 ( wget -nc -q -O zstd-${ZSTD_VERSION}.tar.gz https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz; echo "" ) &&
 tar xzvf zstd-${ZSTD_VERSION}.tar.gz &&
 cd zstd-${ZSTD_VERSION} &&
-make -j8 install &&
+make -j$3 install &&
 cd .. &&
 echo "-- Install Cmake 3.23.1 --" &&
 pwd &&
@@ -81,14 +85,14 @@ echo "-- Install Boost 1.75.0 --" &&
 pwd &&
 ( wget -nc -q https://boostorg.jfrog.io/artifactory/main/release/1.75.0/source/boost_1_75_0.tar.gz; echo "" ) &&
 tar -xzf boost_1_75_0.tar.gz &&
-cd boost_1_75_0 && ./bootstrap.sh && ./b2  link=static -j8 && ./b2 install &&
+cd boost_1_75_0 && ./bootstrap.sh && ./b2  link=static -j$3 && ./b2 install &&
 cd ../ &&
 echo "-- Install Protobuf 3.20.0 --" &&
 pwd &&
 ( wget -nc -q https://github.com/protocolbuffers/protobuf/releases/download/v3.20.0/protobuf-all-3.20.0.tar.gz; echo "" ) &&
 tar -xzf protobuf-all-3.20.0.tar.gz &&
 cd protobuf-3.20.0/ &&
-./autogen.sh && ./configure --prefix=/usr --disable-shared link=static && make -j8 && make install &&
+./autogen.sh && ./configure --prefix=/usr --disable-shared link=static && make -j$3 && make install &&
 cd .. &&
 echo "-- Build LLD --" &&
 pwd &&
@@ -105,7 +109,7 @@ rm -rf build CMakeCache.txt &&
 mkdir build &&
 cd build &&
 cmake .. -DLLVM_LIBRARY_DIR=/usr/lib64/llvm13/lib/ -DCMAKE_INSTALL_PREFIX=/usr/lib64/llvm13/ -DCMAKE_BUILD_TYPE=Release &&
-make -j8 install &&
+make -j$3 install &&
 ln -s /usr/lib64/llvm13/lib/include/lld /usr/include/lld &&
 cp /usr/lib64/llvm13/lib/liblld*.a /usr/local/lib/ &&
 cd ../../ &&
@@ -128,41 +132,20 @@ cmake .. \
     -DWASMEDGE_BUILD_PLUGINS=OFF \
     -DWASMEDGE_LINK_TOOLS_STATIC=ON \
     -DBoost_NO_BOOST_CMAKE=ON -DLLVM_DIR=/usr/lib64/llvm13/lib/cmake/llvm/ -DLLVM_LIBRARY_DIR=/usr/lib64/llvm13/lib/ &&
-make -j8 install &&
+make -j$3 install &&
 export PATH=`echo $PATH | sed -E "s/devtoolset-9/devtoolset-10/g"` &&
 cp -r include/api/wasmedge /usr/include/ &&
 cd /io/ &&
 echo "-- Build Rippled --" &&
 pwd &&
 cp Builds/CMake/deps/Rocksdb.cmake Builds/CMake/deps/Rocksdb.cmake.old &&
-perl -i -pe "s/^(\\s*)-DBUILD_SHARED_LIBS=OFF/\\1-DBUILD_SHARED_LIBS=OFF\\n\\1-DROCKSDB_BUILD_SHARED=OFF/g" Builds/CMake/deps/Rocksdb.cmake &&
-mv Builds/CMake/deps/WasmEdge.cmake Builds/CMake/deps/WasmEdge.old &&
-echo "find_package(LLVM REQUIRED CONFIG)
-message(STATUS \"Found LLVM ${LLVM_PACKAGE_VERSION}\")
-message(STATUS \"Using LLVMConfig.cmake in: \${LLVM_DIR}\")
-add_library (wasmedge STATIC IMPORTED GLOBAL)
-set_target_properties(wasmedge PROPERTIES IMPORTED_LOCATION \${WasmEdge_LIB})
-target_link_libraries (ripple_libs INTERFACE wasmedge)
-add_library (NIH::WasmEdge ALIAS wasmedge)
-message(\"WasmEdge DONE\")
-" > Builds/CMake/deps/WasmEdge.cmake &&
-cd release-build &&
-cmake .. -DBoost_NO_BOOST_CMAKE=ON -DLLVM_DIR=/usr/lib64/llvm13/lib/cmake/llvm/ -DLLVM_LIBRARY_DIR=/usr/lib64/llvm13/lib/ -DWasmEdge_LIB=/usr/local/lib64/libwasmedge.a &&
-make -j8 VERBOSE=1 &&
-strip -s rippled &&
-mv rippled xahaud &&
-echo "Build host: `hostname`" > release.info &&
-echo "Build date: `date`" >> release.info &&
-echo "Build md5: `md5sum xahaud`" >> release.info &&
-echo "Git remotes:" >> release.info && 
-git remote -v >> release.info 
-echo "Git status:" >> release.info &&
-git status -v >> release.info &&
-echo "Git log [last 20]:" >> release.info &&
-git log -n 20 >> release.info;
-cd ..;
-mv src/ripple/net/impl/RegisterSSLCerts.cpp.old src/ripple/net/impl/RegisterSSLCerts.cpp;
-mv Builds/CMake/deps/Rocksdb.cmake.old Builds/CMake/deps/Rocksdb.cmake;
-mv Builds/CMake/deps/WasmEdge.old Builds/CMake/deps/WasmEdge.cmake;
 
-echo "END INSIDE CONTAINER"
+echo "MOVING TO [ build-core.sh ]"
+cd /io;
+./build-core.sh "$1" "$2" "$3" "$4"
+echo "END [ build-core.sh ]"
+
+echo "END INSIDE CONTAINER - FULL"
+
+echo "-- Built with env vars:"
+printenv

docker-unit-tests.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+docker run --rm -i -v $(pwd):/io ubuntu sh -c '/io/release-build/xahaud -u'
+

release-builder.sh

@@ -2,6 +2,21 @@
 
 echo "START BUILDING (HOST)"
 
+echo "Cleaning previously built binary"
+rm -f release-build/xahaud
+
+BUILD_CORES=$(echo "scale=0 ; `nproc` / 3" | bc)
+
+if [[ "$GITHUB_REPOSITORY" == "" ]]; then
+  #Default
+  BUILD_CORES=8
+fi
+
+echo "-- BUILD CORES:       $BUILD_CORES"
+echo "-- GITHUB_REPOSITORY: $GITHUB_REPOSITORY"
+echo "-- GITHUB_SHA:        $GITHUB_SHA"
+echo "-- GITHUB_RUN_NUMBER: $GITHUB_RUN_NUMBER"
+
 which docker 2> /dev/null 2> /dev/null
 if [ "$?" -eq "1" ]
 then
@@ -15,6 +30,31 @@ then
   echo 'Run this inside the source directory. (.git dir not found).'
   exit 1
 fi
-docker run -i --user 0:$(id -g) --rm  -v `pwd`:/io --network host ghcr.io/foobarwidget/holy-build-box-x64 /hbb_exe/activate-exec bash -x /io/build-inner.sh
+
+STATIC_CONTAINER=$(docker ps -a | grep xahaud_cached_builder |wc -l)
+
+if [[ "$STATIC_CONTAINER" -gt "0" && "$GITHUB_REPOSITORY" != "" ]]; then
+  echo "Static container, execute in static container to have max. cache"
+  docker start xahaud_cached_builder
+  docker exec -i xahaud_cached_builder /hbb_exe/activate-exec bash -x /io/build-core.sh "$GITHUB_REPOSITORY" "$GITHUB_SHA" "$BUILD_CORES" "$GITHUB_RUN_NUMBER"
+  docker stop xahaud_cached_builder
+else
+  echo "No static container, build on temp container"
+  rm -rf release-build;
+  mkdir -p release-build;
+
+  if [[ "$GITHUB_REPOSITORY" == "" ]]; then
+    # Non GH, local building
+    echo "Non-GH runner, local building, temp container"
+    docker run -i --user 0:$(id -g) --rm -v `pwd`:/io --network host ghcr.io/foobarwidget/holy-build-box-x64 /hbb_exe/activate-exec bash -x /io/build-full.sh "$GITHUB_REPOSITORY" "$GITHUB_SHA" "$BUILD_CORES" "$GITHUB_RUN_NUMBER"
+  else
+    # GH Action, runner
+    echo "GH Action, runner, clean & re-create create persistent container"
+    docker rm -f xahaud_cached_builder
+    docker run -di --user 0:$(id -g) --name xahaud_cached_builder -v `pwd`:/io --network host ghcr.io/foobarwidget/holy-build-box-x64 /hbb_exe/activate-exec bash
+    docker exec -i xahaud_cached_builder /hbb_exe/activate-exec bash -x /io/build-full.sh "$GITHUB_REPOSITORY" "$GITHUB_SHA" "$BUILD_CORES" "$GITHUB_RUN_NUMBER"
+    docker stop xahaud_cached_builder
+  fi
+fi
 
 echo "DONE BUILDING (HOST)"