diff --git a/Builds/CMake/conan/Boost.cmake b/Builds/CMake/conan/Boost.cmake new file mode 100644 index 000000000..161ab20c5 --- /dev/null +++ b/Builds/CMake/conan/Boost.cmake @@ -0,0 +1,51 @@ +find_package(Boost 1.83 REQUIRED + COMPONENTS + chrono + container + context + coroutine + date_time + filesystem + program_options + regex + system + thread +) + +add_library(ripple_boost INTERFACE) +add_library(Ripple::boost ALIAS ripple_boost) +if(XCODE) + target_include_directories(ripple_boost BEFORE INTERFACE ${Boost_INCLUDE_DIRS}) + target_compile_options(ripple_boost INTERFACE --system-header-prefix="boost/") +else() + target_include_directories(ripple_boost SYSTEM BEFORE INTERFACE ${Boost_INCLUDE_DIRS}) +endif() + +target_link_libraries(ripple_boost + INTERFACE + Boost::boost + Boost::chrono + Boost::container + Boost::coroutine + Boost::date_time + Boost::filesystem + Boost::program_options + Boost::regex + Boost::system + Boost::thread) +if(Boost_COMPILER) + target_link_libraries(ripple_boost INTERFACE Boost::disable_autolinking) +endif() +if(san AND is_clang) + # TODO: gcc does not support -fsanitize-blacklist...can we do something else + # for gcc ? + if(NOT Boost_INCLUDE_DIRS AND TARGET Boost::headers) + get_target_property(Boost_INCLUDE_DIRS Boost::headers INTERFACE_INCLUDE_DIRECTORIES) + endif() + message(STATUS "Adding [${Boost_INCLUDE_DIRS}] to sanitizer blacklist") + file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/san_bl.txt "src:${Boost_INCLUDE_DIRS}/*") + target_compile_options(opts + INTERFACE + # ignore boost headers for sanitizing + -fsanitize-blacklist=${CMAKE_CURRENT_BINARY_DIR}/san_bl.txt) +endif() \ No newline at end of file diff --git a/Builds/CMake/conan/Protobuf.cmake b/Builds/CMake/conan/Protobuf.cmake new file mode 100644 index 000000000..4d4016e49 --- /dev/null +++ b/Builds/CMake/conan/Protobuf.cmake @@ -0,0 +1,22 @@ +find_package(Protobuf 3.8) + +file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/proto_gen) +set(ccbd ${CMAKE_CURRENT_BINARY_DIR}) +set(CMAKE_CURRENT_BINARY_DIR ${CMAKE_BINARY_DIR}/proto_gen) +protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS src/ripple/proto/ripple.proto) +set(CMAKE_CURRENT_BINARY_DIR ${ccbd}) + +add_library(pbufs STATIC ${PROTO_SRCS} ${PROTO_HDRS}) +target_include_directories(pbufs SYSTEM PUBLIC + ${CMAKE_BINARY_DIR}/proto_gen + ${CMAKE_BINARY_DIR}/proto_gen/src/ripple/proto +) +target_link_libraries(pbufs protobuf::libprotobuf) +target_compile_options(pbufs + PUBLIC + $<$: + --system-header-prefix="google/protobuf" + -Wno-deprecated-dynamic-exception-spec + > +) +add_library(Ripple::pbufs ALIAS pbufs) \ No newline at end of file diff --git a/Builds/CMake/conan/gRPC.cmake b/Builds/CMake/conan/gRPC.cmake new file mode 100644 index 000000000..0930ba4dd --- /dev/null +++ b/Builds/CMake/conan/gRPC.cmake @@ -0,0 +1,62 @@ +find_package(gRPC 1.23) + +#[=================================[ + generate protobuf sources for + grpc defs and bundle into a + static lib +#]=================================] +set(GRPC_GEN_DIR "${CMAKE_BINARY_DIR}/proto_gen_grpc") +file(MAKE_DIRECTORY ${GRPC_GEN_DIR}) +set(GRPC_PROTO_SRCS) +set(GRPC_PROTO_HDRS) +set(GRPC_PROTO_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/src/ripple/proto/org") +file(GLOB_RECURSE GRPC_DEFINITION_FILES LIST_DIRECTORIES false "${GRPC_PROTO_ROOT}/*.proto") +foreach(file ${GRPC_DEFINITION_FILES}) + get_filename_component(_abs_file ${file} ABSOLUTE) + get_filename_component(_abs_dir ${_abs_file} DIRECTORY) + get_filename_component(_basename ${file} NAME_WE) + get_filename_component(_proto_inc ${GRPC_PROTO_ROOT} DIRECTORY) # updir one level + file(RELATIVE_PATH _rel_root_file ${_proto_inc} ${_abs_file}) + get_filename_component(_rel_root_dir ${_rel_root_file} DIRECTORY) + file(RELATIVE_PATH _rel_dir ${CMAKE_CURRENT_SOURCE_DIR} ${_abs_dir}) + + set(src_1 "${GRPC_GEN_DIR}/${_rel_root_dir}/${_basename}.grpc.pb.cc") + set(src_2 "${GRPC_GEN_DIR}/${_rel_root_dir}/${_basename}.pb.cc") + set(hdr_1 "${GRPC_GEN_DIR}/${_rel_root_dir}/${_basename}.grpc.pb.h") + set(hdr_2 "${GRPC_GEN_DIR}/${_rel_root_dir}/${_basename}.pb.h") + add_custom_command( + OUTPUT ${src_1} ${src_2} ${hdr_1} ${hdr_2} + COMMAND protobuf::protoc + ARGS --grpc_out=${GRPC_GEN_DIR} + --cpp_out=${GRPC_GEN_DIR} + --plugin=protoc-gen-grpc=$ + -I ${_proto_inc} -I ${_rel_dir} + ${_abs_file} + DEPENDS ${_abs_file} protobuf::protoc gRPC::grpc_cpp_plugin + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + COMMENT "Running gRPC C++ protocol buffer compiler on ${file}" + VERBATIM) + set_source_files_properties(${src_1} ${src_2} ${hdr_1} ${hdr_2} PROPERTIES GENERATED TRUE) + list(APPEND GRPC_PROTO_SRCS ${src_1} ${src_2}) + list(APPEND GRPC_PROTO_HDRS ${hdr_1} ${hdr_2}) +endforeach() + +add_library(grpc_pbufs STATIC ${GRPC_PROTO_SRCS} ${GRPC_PROTO_HDRS}) +#target_include_directories(grpc_pbufs PRIVATE src) +target_include_directories(grpc_pbufs SYSTEM PUBLIC ${GRPC_GEN_DIR}) +target_link_libraries(grpc_pbufs + "gRPC::grpc++" + # libgrpc is missing references. + absl::random_random +) +target_compile_options(grpc_pbufs + PRIVATE + $<$:-wd4065> + $<$>:-Wno-deprecated-declarations> + PUBLIC + $<$:-wd4996> + $<$: + --system-header-prefix="google/protobuf" + -Wno-deprecated-dynamic-exception-spec + >) +add_library(Ripple::grpc_pbufs ALIAS grpc_pbufs) \ No newline at end of file diff --git a/Builds/CMake/deps/Rocksdb.cmake b/Builds/CMake/deps/Rocksdb.cmake index 18dc67508..2c832c593 100644 --- a/Builds/CMake/deps/Rocksdb.cmake +++ b/Builds/CMake/deps/Rocksdb.cmake @@ -64,13 +64,13 @@ if (local_rocksdb) PATCH_COMMAND # only used by windows build ${CMAKE_COMMAND} -E copy_if_different - ${CMAKE_CURRENT_SOURCE_DIR}/Builds/CMake/deps/rocks_thirdparty.inc + ${CMAKE_CURRENT_SOURCE_DIR}/Builds/CMake/rocks_thirdparty.inc /thirdparty.inc COMMAND # fixup their build version file to keep the values # from changing always ${CMAKE_COMMAND} -E copy_if_different - ${CMAKE_CURRENT_SOURCE_DIR}/Builds/CMake/deps/rocksdb_build_version.cc.in + ${CMAKE_CURRENT_SOURCE_DIR}/Builds/CMake/rocksdb_build_version.cc.in /util/build_version.cc.in CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} diff --git a/Builds/CMake/deps/Soci.cmake b/Builds/CMake/deps/Soci.cmake index 87322cd81..d165d6e1f 100644 --- a/Builds/CMake/deps/Soci.cmake +++ b/Builds/CMake/deps/Soci.cmake @@ -52,7 +52,7 @@ else() # whenever we update the GIT_TAG above. PATCH_COMMAND ${CMAKE_COMMAND} -D RIPPLED_SOURCE=${CMAKE_CURRENT_SOURCE_DIR} - -P ${CMAKE_CURRENT_SOURCE_DIR}/Builds/CMake/deps/soci_patch.cmake + -P ${CMAKE_CURRENT_SOURCE_DIR}/Builds/CMake/soci_patch.cmake CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} diff --git a/Builds/CMake/deps/SociConfig.cmake.patched b/Builds/CMake/deps/SociConfig.cmake.patched deleted file mode 100644 index 11bcd1f3d..000000000 --- a/Builds/CMake/deps/SociConfig.cmake.patched +++ /dev/null @@ -1,106 +0,0 @@ -################################################################################ -# SociConfig.cmake - CMake build configuration of SOCI library -################################################################################ -# Copyright (C) 2010 Mateusz Loskot -# -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt or copy at -# http://www.boost.org/LICENSE_1_0.txt) -################################################################################ - -include(CheckCXXSymbolExists) - -if(WIN32) - check_cxx_symbol_exists("_M_AMD64" "" SOCI_TARGET_ARCH_X64) - if(NOT RTC_ARCH_X64) - check_cxx_symbol_exists("_M_IX86" "" SOCI_TARGET_ARCH_X86) - endif(NOT RTC_ARCH_X64) - # add check for arm here - # see http://msdn.microsoft.com/en-us/library/b0084kay.aspx -else(WIN32) - check_cxx_symbol_exists("__i386__" "" SOCI_TARGET_ARCH_X86) - check_cxx_symbol_exists("__x86_64__" "" SOCI_TARGET_ARCH_X64) - check_cxx_symbol_exists("__arm__" "" SOCI_TARGET_ARCH_ARM) -endif(WIN32) - -if(NOT DEFINED LIB_SUFFIX) - if(SOCI_TARGET_ARCH_X64) - set(_lib_suffix "64") - else() - set(_lib_suffix "") - endif() - set(LIB_SUFFIX ${_lib_suffix} CACHE STRING "Specifies suffix for the lib directory") -endif() - -# -# C++11 Option -# - -if(NOT SOCI_CXX_C11) - set (SOCI_CXX_C11 OFF CACHE BOOL "Build to the C++11 standard") -endif() - -# -# Force compilation flags and set desired warnings level -# - -if (MSVC) - add_definitions(-D_CRT_SECURE_NO_DEPRECATE) - add_definitions(-D_CRT_SECURE_NO_WARNINGS) - add_definitions(-D_CRT_NONSTDC_NO_WARNING) - add_definitions(-D_SCL_SECURE_NO_WARNINGS) - - if(CMAKE_CXX_FLAGS MATCHES "/W[0-4]") - string(REGEX REPLACE "/W[0-4]" "/W4" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") - else() - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W4 /we4266") - endif() - -else() - - set(SOCI_GCC_CLANG_COMMON_FLAGS "") - # "-pedantic -Werror -Wno-error=parentheses -Wall -Wextra -Wpointer-arith -Wcast-align -Wcast-qual -Wfloat-equal -Woverloaded-virtual -Wredundant-decls -Wno-long-long") - - - if (SOCI_CXX_C11) - set(SOCI_CXX_VERSION_FLAGS "-std=c++11") - else() - set(SOCI_CXX_VERSION_FLAGS "-std=gnu++98") - endif() - - if("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang" OR "${CMAKE_CXX_COMPILER}" MATCHES "clang") - - if(NOT CMAKE_CXX_COMPILER_VERSION LESS 3.1 AND SOCI_ASAN) - set(SOCI_GCC_CLANG_COMMON_FLAGS "${SOCI_GCC_CLANG_COMMON_FLAGS} -fsanitize=address") - endif() - - # enforce C++11 for Clang - set(SOCI_CXX_C11 ON) - set(SOCI_CXX_VERSION_FLAGS "-std=c++11") - add_definitions(-DCATCH_CONFIG_CPP11_NO_IS_ENUM) - - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SOCI_GCC_CLANG_COMMON_FLAGS} ${SOCI_CXX_VERSION_FLAGS}") - - elseif(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_GNUCXX) - - if(NOT CMAKE_CXX_COMPILER_VERSION LESS 4.8 AND SOCI_ASAN) - set(SOCI_GCC_CLANG_COMMON_FLAGS "${SOCI_GCC_CLANG_COMMON_FLAGS} -fsanitize=address") - endif() - - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SOCI_GCC_CLANG_COMMON_FLAGS} ${SOCI_CXX_VERSION_FLAGS} ") - if (CMAKE_COMPILER_IS_GNUCXX) - if (CMAKE_SYSTEM_NAME MATCHES "FreeBSD") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") - else() - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-variadic-macros") - endif() - endif() - - else() - message(WARNING "Unknown toolset - using default flags to build SOCI") - endif() - -endif() - -# Set SOCI_HAVE_* variables for soci-config.h generator -set(SOCI_HAVE_CXX_C11 ${SOCI_CXX_C11} CACHE INTERNAL "Enables C++11 support") diff --git a/Builds/CMake/deps/WasmEdge.cmake b/Builds/CMake/deps/WasmEdge.cmake index 56f145f1d..a66240576 100644 --- a/Builds/CMake/deps/WasmEdge.cmake +++ b/Builds/CMake/deps/WasmEdge.cmake @@ -81,4 +81,4 @@ if(XAR_LIBRARY) else() message(WARNING "xar library not found... (only important for mac builds)") endif() -add_library (NIH::WasmEdge ALIAS wasmedge) +add_library (wasmedge::wasmedge ALIAS wasmedge) diff --git a/Builds/CMake/deps/rocks_thirdparty.inc b/Builds/CMake/rocks_thirdparty.inc similarity index 100% rename from Builds/CMake/deps/rocks_thirdparty.inc rename to Builds/CMake/rocks_thirdparty.inc diff --git a/Builds/CMake/deps/rocksdb_build_version.cc.in b/Builds/CMake/rocksdb_build_version.cc.in similarity index 100% rename from Builds/CMake/deps/rocksdb_build_version.cc.in rename to Builds/CMake/rocksdb_build_version.cc.in diff --git a/Builds/CMake/deps/soci_patch.cmake b/Builds/CMake/soci_patch.cmake similarity index 97% rename from Builds/CMake/deps/soci_patch.cmake rename to Builds/CMake/soci_patch.cmake index 1b5122e47..0c2a75c0d 100644 --- a/Builds/CMake/deps/soci_patch.cmake +++ b/Builds/CMake/soci_patch.cmake @@ -8,7 +8,7 @@ # those warnings. if (RIPPLED_SOURCE) execute_process( COMMAND ${CMAKE_COMMAND} -E copy_if_different - ${RIPPLED_SOURCE}/Builds/CMake/deps/SociConfig.cmake.patched + ${RIPPLED_SOURCE}/Builds/CMake/SociConfig.cmake.patched cmake/SociConfig.cmake ) endif () diff --git a/CMakeLists.txt b/CMakeLists.txt index ff3ca78cd..d1430a744 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -60,66 +60,95 @@ include(RippledCompiler) include(RippledInterface) ### - -include(deps/Boost) -find_package(OpenSSL 1.1.1 REQUIRED) -set_target_properties(OpenSSL::SSL PROPERTIES - INTERFACE_COMPILE_DEFINITIONS OPENSSL_NO_SSL2 -) -add_subdirectory(src/secp256k1) -add_subdirectory(src/ed25519-donna) -find_package(lz4 REQUIRED) -# Target names with :: are not allowed in a generator expression. -# We need to pull the include directories and imported location properties -# from separate targets. -find_package(LibArchive REQUIRED) -find_package(SOCI REQUIRED) -find_package(SQLite3 REQUIRED) -find_package(Snappy REQUIRED) -find_package(wasmedge REQUIRED) - -option(rocksdb "Enable RocksDB" ON) -if(rocksdb) - find_package(RocksDB REQUIRED) - set_target_properties(RocksDB::rocksdb PROPERTIES - INTERFACE_COMPILE_DEFINITIONS RIPPLE_ROCKSDB_AVAILABLE=1 - ) - target_link_libraries(ripple_libs INTERFACE RocksDB::rocksdb) +option(USE_CONAN "Use Conan package manager for dependencies" OFF) +# Then, auto-detect if conan_toolchain.cmake is being used +if(CMAKE_TOOLCHAIN_FILE) + # Check if the toolchain file path contains "conan_toolchain" + if(CMAKE_TOOLCHAIN_FILE MATCHES "conan_toolchain") + set(USE_CONAN ON CACHE BOOL "Using Conan detected from toolchain file" FORCE) + message(STATUS "Conan toolchain detected: ${CMAKE_TOOLCHAIN_FILE}") + message(STATUS "Building with Conan dependencies") + endif() endif() - -find_package(nudb REQUIRED) -find_package(date REQUIRED) -include(deps/Protobuf) -include(deps/gRPC) - -target_link_libraries(ripple_libs INTERFACE - ed25519::ed25519 - LibArchive::LibArchive - lz4::lz4 - OpenSSL::Crypto - OpenSSL::SSL - Ripple::grpc_pbufs - Ripple::pbufs - secp256k1::secp256k1 - soci::soci - SQLite::SQLite3 -) - -if(TARGET nudb::core) - set(nudb nudb::core) -elseif(TARGET NuDB::nudb) - set(nudb NuDB::nudb) +if (NOT USE_CONAN) + list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/Builds/CMake") + list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/Builds/CMake/deps") + add_subdirectory(src/secp256k1) + add_subdirectory(src/ed25519-donna) + include(deps/Boost) + include(deps/OpenSSL) + # include(deps/Secp256k1) + # include(deps/Ed25519-donna) + include(deps/Lz4) + include(deps/Libarchive) + include(deps/Sqlite) + include(deps/Soci) + include(deps/Snappy) + include(deps/Rocksdb) + include(deps/Nudb) + include(deps/date) + include(deps/Protobuf) + include(deps/gRPC) + include(deps/cassandra) + include(deps/Postgres) + include(deps/WasmEdge) else() - message(FATAL_ERROR "unknown nudb target") -endif() -target_link_libraries(ripple_libs INTERFACE ${nudb}) + include(conan/Boost) + find_package(OpenSSL 1.1.1 REQUIRED) + set_target_properties(OpenSSL::SSL PROPERTIES + INTERFACE_COMPILE_DEFINITIONS OPENSSL_NO_SSL2 + ) + add_subdirectory(src/secp256k1) + add_subdirectory(src/ed25519-donna) + find_package(lz4 REQUIRED) + # Target names with :: are not allowed in a generator expression. + # We need to pull the include directories and imported location properties + # from separate targets. + find_package(LibArchive REQUIRED) + find_package(SOCI REQUIRED) + find_package(SQLite3 REQUIRED) + find_package(Snappy REQUIRED) + find_package(wasmedge REQUIRED) + option(rocksdb "Enable RocksDB" ON) + if(rocksdb) + find_package(RocksDB REQUIRED) + set_target_properties(RocksDB::rocksdb PROPERTIES + INTERFACE_COMPILE_DEFINITIONS RIPPLE_ROCKSDB_AVAILABLE=1 + ) + target_link_libraries(ripple_libs INTERFACE RocksDB::rocksdb) + endif() + find_package(nudb REQUIRED) + find_package(date REQUIRED) + include(conan/Protobuf) + include(conan/gRPC) + if(TARGET nudb::core) + set(nudb nudb::core) + elseif(TARGET NuDB::nudb) + set(nudb NuDB::nudb) + else() + message(FATAL_ERROR "unknown nudb target") + endif() + target_link_libraries(ripple_libs INTERFACE ${nudb}) -if(reporting) - find_package(cassandra-cpp-driver REQUIRED) - find_package(PostgreSQL REQUIRED) + if(reporting) + find_package(cassandra-cpp-driver REQUIRED) + find_package(PostgreSQL REQUIRED) + target_link_libraries(ripple_libs INTERFACE + cassandra-cpp-driver::cassandra-cpp-driver + PostgreSQL::PostgreSQL + ) + endif() target_link_libraries(ripple_libs INTERFACE - cassandra-cpp-driver::cassandra-cpp-driver - PostgreSQL::PostgreSQL + ed25519::ed25519 + LibArchive::LibArchive + lz4::lz4 + OpenSSL::Crypto + OpenSSL::SSL + Ripple::grpc_pbufs + Ripple::pbufs + secp256k1::secp256k1 + soci::soci + SQLite::SQLite3 ) endif() diff --git a/build-core.sh b/build-core.sh index 072468916..909444330 100755 --- a/build-core.sh +++ b/build-core.sh @@ -28,9 +28,6 @@ if [[ "$?" -ne "0" ]]; then fi #!/bin/bash -sed -i 's/ed25519.h/ed25519-donna\/ed25519.h/g' src/ripple/protocol/impl/PublicKey.cpp && -sed -i 's/ed25519.h/ed25519-donna\/ed25519.h/g' src/ripple/protocol/impl/SecretKey.cpp && -sed -i 's/ed25519.h/ed25519-donna\/ed25519.h/g' src/ripple/rpc/handlers/WalletPropose.cpp && perl -i -pe "s/^(\\s*)-DBUILD_SHARED_LIBS=OFF/\\1-DBUILD_SHARED_LIBS=OFF\\n\\1-DROCKSDB_BUILD_SHARED=OFF/g" Builds/CMake/deps/Rocksdb.cmake && mv Builds/CMake/deps/WasmEdge.cmake Builds/CMake/deps/WasmEdge.old && cp Builds/CMake/deps/Rocksdb.cmake Builds/CMake/deps/Rocksdb.cmake.old && @@ -40,7 +37,7 @@ message(STATUS \"Using LLVMConfig.cmake in: \${LLVM_DIR}\") add_library (wasmedge STATIC IMPORTED GLOBAL) set_target_properties(wasmedge PROPERTIES IMPORTED_LOCATION \${WasmEdge_LIB}) target_link_libraries (ripple_libs INTERFACE wasmedge) -add_library (NIH::WasmEdge ALIAS wasmedge) +add_library (wasmedge::wasmedge ALIAS wasmedge) message(\"WasmEdge DONE\") " > Builds/CMake/deps/WasmEdge.cmake && git checkout src/ripple/protocol/impl/BuildInfo.cpp && diff --git a/build-full.sh b/build-deps.sh similarity index 60% rename from build-full.sh rename to build-deps.sh index 302135a22..cd3647dd5 100755 --- a/build-full.sh +++ b/build-deps.sh @@ -5,14 +5,12 @@ # debugging. set -ex -set -e - echo "START INSIDE CONTAINER - FULL" -echo "-- BUILD CORES: $3" -echo "-- GITHUB_REPOSITORY: $1" -echo "-- GITHUB_SHA: $2" -echo "-- GITHUB_RUN_NUMBER: $4" +# Set default BUILD_CORES if not defined +: ${BUILD_CORES:=8} + +echo "-- BUILD CORES: $BUILD_CORES" umask 0000; @@ -25,56 +23,11 @@ yum-config-manager --disable centos-sclo-sclo #### -cd /io; -mkdir -p src/certs; -curl --silent -k https://raw.githubusercontent.com/RichardAH/rippled-release-builder/main/ca-bundle/certbundle.h -o src/certs/certbundle.h; -if [ "`grep certbundle.h src/ripple/net/impl/RegisterSSLCerts.cpp | wc -l`" -eq "0" ] -then - cp src/ripple/net/impl/RegisterSSLCerts.cpp src/ripple/net/impl/RegisterSSLCerts.cpp.old - perl -i -pe "s/^{/{ - #ifdef EMBEDDED_CA_BUNDLE - BIO *cbio = BIO_new_mem_buf(ca_bundle.data(), ca_bundle.size()); - X509_STORE *cts = SSL_CTX_get_cert_store(ctx.native_handle()); - if(!cts || !cbio) - JLOG(j.warn()) - << \"Failed to create cts\/cbio when loading embedded certs.\"; - else - { - X509_INFO *itmp; - int i, count = 0, type = X509_FILETYPE_PEM; - STACK_OF(X509_INFO) *inf = PEM_X509_INFO_read_bio(cbio, NULL, NULL, NULL); +mkdir -p /io +cd /io - if (!inf) - { - BIO_free(cbio); - JLOG(j.warn()) - << \"Failed to read cbio when loading embedded certs.\"; - } - else - { - for (i = 0; i < sk_X509_INFO_num(inf); i++) - { - itmp = sk_X509_INFO_value(inf, i); - if (itmp->x509) - { - X509_STORE_add_cert(cts, itmp->x509); - count++; - } - if (itmp->crl) - { - X509_STORE_add_crl(cts, itmp->crl); - count++; - } - } - sk_X509_INFO_pop_free(inf, X509_INFO_free); - BIO_free(cbio); - } - } - #endif/g" src/ripple/net/impl/RegisterSSLCerts.cpp && - sed -i "s/#include /\0\n#include /g" src/ripple/net/impl/RegisterSSLCerts.cpp -fi -mkdir -p .nih_c; -mkdir -p .nih_toolchain; +mkdir -p .nih_c +mkdir -p .nih_toolchain cd .nih_toolchain && yum install -y wget lz4 lz4-devel git llvm13-static.x86_64 llvm13-devel.x86_64 devtoolset-10-binutils zlib-static ncurses-static -y \ devtoolset-7-gcc-c++ \ @@ -91,7 +44,7 @@ ZSTD_VERSION="1.1.3" && ( wget -nc -q -O zstd-${ZSTD_VERSION}.tar.gz https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz; echo "" ) && tar xzvf zstd-${ZSTD_VERSION}.tar.gz && cd zstd-${ZSTD_VERSION} && -make -j$3 install && +make -j$BUILD_CORES install && cd .. && echo "-- Install Cmake 3.23.1 --" && pwd && @@ -101,20 +54,23 @@ echo "-- Install Boost 1.86.0 --" && pwd && ( wget -nc -q https://archives.boost.io/release/1.86.0/source/boost_1_86_0.tar.gz; echo "" ) && tar -xzf boost_1_86_0.tar.gz && -cd boost_1_86_0 && ./bootstrap.sh && ./b2 link=static -j$3 && ./b2 install && +cd boost_1_86_0 && ./bootstrap.sh && ./b2 link=static -j$BUILD_CORES && ./b2 install && cd ../ && +# Copy Boost to the expected location +mkdir -p /usr/local/src/ && +cp -r boost_1_86_0 /usr/local/src/ && echo "-- Install Protobuf 3.20.0 --" && pwd && ( wget -nc -q https://github.com/protocolbuffers/protobuf/releases/download/v3.20.0/protobuf-all-3.20.0.tar.gz; echo "" ) && tar -xzf protobuf-all-3.20.0.tar.gz && cd protobuf-3.20.0/ && -./autogen.sh && ./configure --prefix=/usr --disable-shared link=static && make -j$3 && make install && +./autogen.sh && ./configure --prefix=/usr --disable-shared link=static && make -j$BUILD_CORES && make install && cd .. && echo "-- Build LLD --" && pwd && -ln /usr/bin/llvm-config-13 /usr/bin/llvm-config && +ln -sf /usr/bin/llvm-config-13 /usr/bin/llvm-config && mv /opt/rh/devtoolset-9/root/usr/bin/ar /opt/rh/devtoolset-9/root/usr/bin/ar-9 && -ln /opt/rh/devtoolset-10/root/usr/bin/ar /opt/rh/devtoolset-9/root/usr/bin/ar && +ln -sf /opt/rh/devtoolset-10/root/usr/bin/ar /opt/rh/devtoolset-9/root/usr/bin/ar && ( wget -nc -q https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.1/lld-13.0.1.src.tar.xz; echo "" ) && ( wget -nc -q https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.1/libunwind-13.0.1.src.tar.xz; echo "" ) && tar -xf lld-13.0.1.src.tar.xz && @@ -125,12 +81,12 @@ rm -rf build CMakeCache.txt && mkdir -p build && cd build && cmake .. -DLLVM_LIBRARY_DIR=/usr/lib64/llvm13/lib/ -DCMAKE_INSTALL_PREFIX=/usr/lib64/llvm13/ -DCMAKE_BUILD_TYPE=Release && -make -j$3 install && +make -j$BUILD_CORES install && ln -s /usr/lib64/llvm13/lib/include/lld /usr/include/lld && cp /usr/lib64/llvm13/lib/liblld*.a /usr/local/lib/ && cd ../../ && echo "-- Build WasmEdge --" && -( wget -nc -q https://github.com/WasmEdge/WasmEdge/archive/refs/tags/0.11.2.zip; unzip -o 0.11.2.zip; ) && +( wget -nc -q https://github.com/WasmEdge/WasmEdge/archive/refs/tags/0.11.2.zip; echo ""; unzip -o 0.11.2.zip; ) && cd WasmEdge-0.11.2 && ( mkdir -p build; echo "" ) && cd build && @@ -149,31 +105,36 @@ cmake .. \ -DWASMEDGE_BUILD_PLUGINS=OFF \ -DWASMEDGE_LINK_TOOLS_STATIC=ON \ -DBoost_NO_BOOST_CMAKE=ON -DLLVM_DIR=/usr/lib64/llvm13/lib/cmake/llvm/ -DLLVM_LIBRARY_DIR=/usr/lib64/llvm13/lib/ && -make -j$3 install && +make -j$BUILD_CORES install && export PATH=`echo $PATH | sed -E "s/devtoolset-9/devtoolset-10/g"` && cp -r include/api/wasmedge /usr/include/ && cd /io/ && echo "-- Build Rippled --" && echo "MOVING TO [ build-core.sh ]" -cd /io; -printenv > .env.temp; -cat .env.temp | grep '=' | sed s/\\\(^[^=]\\+=\\\)/\\1\\\"/g|sed s/\$/\\\"/g > .env; -rm .env.temp; +cd /io + +# Save current environment to .env file +printenv > .env.temp +cat .env.temp | grep '=' | sed s/\\\(^[^=]\\+=\\\)/\\1\\\"/g|sed s/\$/\\\"/g > .env +rm .env.temp echo "Persisting ENV:" cat .env -./build-core.sh "$1" "$2" "$3" "$4" +# Create a deps summary +mkdir -p /usr/local +DEPS_SUMMARY="/usr/local/deps-summary.txt" +echo "Dependencies built at: $(date)" > $DEPS_SUMMARY +echo "Boost dir: $(ls -la /usr/local/src/boost_1_86_0 2>/dev/null || echo 'NOT FOUND')" >> $DEPS_SUMMARY +echo "ZStd version: $(zstd --version | head -n 1 || echo 'NOT FOUND')" >> $DEPS_SUMMARY +echo "Protobuf version: $(protoc --version || echo 'NOT FOUND')" >> $DEPS_SUMMARY +echo "CMAKE version: $(/hbb/cmake-3.23.1-linux-x86_64/bin/cmake --version | head -n 1 || echo 'NOT FOUND')" >> $DEPS_SUMMARY -echo $? -if [[ "$?" -ne "0" ]]; then - echo "ERR build-core.sh non 0 exit code" - exit 127 -fi - -echo "END [ build-core.sh ]" +echo "-------- DEPENDENCY SUMMARY --------" +cat $DEPS_SUMMARY +echo "------------------------------------" echo "END INSIDE CONTAINER - FULL" -echo "-- Built with env vars:" +echo "-- Built with env vars:" \ No newline at end of file diff --git a/build.hbb.dockerfile b/build.hbb.dockerfile new file mode 100644 index 000000000..c17968da9 --- /dev/null +++ b/build.hbb.dockerfile @@ -0,0 +1,20 @@ + +# docker build -t xahaud-hbb:latest build.hbb.dockerfile . +FROM ghcr.io/foobarwidget/holy-build-box-x64 + +# Set environment variables +ENV BUILD_CORES=8 + +# Create /io directory for mounting the working directory +RUN mkdir -p /io + +# Copy dependency build script +COPY build-deps.sh /hbb/build-deps.sh +RUN chmod +x /hbb/build-deps.sh + +# Run the dependency build script with proper error handling +RUN /hbb_exe/activate-exec bash -ex /hbb/build-deps.sh || (echo "ERROR: Dependency build failed" && exit 1) + +# Set the entrypoint to activate the HBB environment and load our env settings +ENTRYPOINT ["/hbb_exe/activate-exec"] +CMD ["bash"] \ No newline at end of file diff --git a/release-builder.sh b/release-builder.sh index 2fa86d933..466920a1d 100755 --- a/release-builder.sh +++ b/release-builder.sh @@ -11,32 +11,24 @@ echo "Cleaning previously built binary" rm -f release-build/xahaud BUILD_CORES=$(echo "scale=0 ; `nproc` / 1.337" | bc) +GITHUB_REPOSITORY=${GITHUB_REPOSITORY:-""} +GITHUB_SHA=${GITHUB_SHA:-"local"} +GITHUB_RUN_NUMBER=${GITHUB_RUN_NUMBER:-"0"} +GITHUB_WORKFLOW=${GITHUB_WORKFLOW:-"local"} +GITHUB_REF=${GITHUB_REF:-"local"} if [[ "$GITHUB_REPOSITORY" == "" ]]; then #Default BUILD_CORES=8 fi -EXIT_IF_CONTAINER_RUNNING=${EXIT_IF_CONTAINER_RUNNING:-1} # Ensure still works outside of GH Actions by setting these to /dev/null # GA will run this script and then delete it at the end of the job JOB_CLEANUP_SCRIPT=${JOB_CLEANUP_SCRIPT:-/dev/null} NORMALIZED_WORKFLOW=$(echo "$GITHUB_WORKFLOW" | tr -c 'a-zA-Z0-9' '-') NORMALIZED_REF=$(echo "$GITHUB_REF" | tr -c 'a-zA-Z0-9' '-') CONTAINER_NAME="xahaud_cached_builder_${NORMALIZED_WORKFLOW}-${NORMALIZED_REF}" - -# Check if the container is already running -if docker ps --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then - echo "⚠️ A running container (${CONTAINER_NAME}) was detected." - - if [[ "$EXIT_IF_CONTAINER_RUNNING" -eq 1 ]]; then - echo "❌ EXIT_IF_CONTAINER_RUNNING is set. Exiting." - exit 1 - else - echo "🛑 Stopping the running container: ${CONTAINER_NAME}" - docker stop "${CONTAINER_NAME}" - fi -fi +DEPENDENCY_IMAGE="xahaud-hbb:latest" echo "-- BUILD CORES: $BUILD_CORES" echo "-- GITHUB_REPOSITORY: $GITHUB_REPOSITORY" @@ -58,33 +50,52 @@ then exit 1 fi +# Check if dependency image exists +if ! docker image inspect "$DEPENDENCY_IMAGE" &> /dev/null; then + echo "Dependency image doesn't exist. Building it now..." + + # Create a temporary directory for dependency build files + TMP_DIR=$(mktemp -d) + + # Copy the dependency build files + cp ./build-deps.sh "$TMP_DIR/" + cp ./build.hbb.dockerfile "$TMP_DIR/Dockerfile" + + # Build the dependency image + docker build -t "$DEPENDENCY_IMAGE" "$TMP_DIR" + + # Clean up + rm -rf "$TMP_DIR" + + echo "Dependency image built successfully." +fi + STATIC_CONTAINER=$(docker ps -a | grep $CONTAINER_NAME |wc -l) -#if [[ "$STATIC_CONTAINER" -gt "0" && "$GITHUB_REPOSITORY" != "" ]]; then -if false; then +if [[ "$STATIC_CONTAINER" -gt "0" && "$GITHUB_REPOSITORY" != "" ]]; then echo "Static container, execute in static container to have max. cache" docker start $CONTAINER_NAME - docker exec -i $CONTAINER_NAME /hbb_exe/activate-exec bash -x /io/build-core.sh "$GITHUB_REPOSITORY" "$GITHUB_SHA" "$BUILD_CORES" "$GITHUB_RUN_NUMBER" + docker exec -i $CONTAINER_NAME bash -x /io/build-core.sh "$GITHUB_REPOSITORY" "$GITHUB_SHA" "$BUILD_CORES" "$GITHUB_RUN_NUMBER" docker stop $CONTAINER_NAME else - echo "No static container, build on temp container" + echo "No static container, build using dependency image" rm -rf release-build; mkdir -p release-build; if [[ "$GITHUB_REPOSITORY" == "" ]]; then # Non GH, local building echo "Non-GH runner, local building, temp container" - docker run -i --user 0:$(id -g) --rm -v /data/builds:/data/builds -v `pwd`:/io --network host ghcr.io/foobarwidget/holy-build-box-x64 /hbb_exe/activate-exec bash -x /io/build-full.sh "$GITHUB_REPOSITORY" "$GITHUB_SHA" "$BUILD_CORES" "$GITHUB_RUN_NUMBER" + docker run -i --user 0:$(id -g) --rm -v /data/builds:/data/builds -v `pwd`:/io --network host "$DEPENDENCY_IMAGE" bash -x /io/build-core.sh "$GITHUB_REPOSITORY" "$GITHUB_SHA" "$BUILD_CORES" "$GITHUB_RUN_NUMBER" else # GH Action, runner echo "GH Action, runner, clean & re-create create persistent container" docker rm -f $CONTAINER_NAME echo "echo 'Stopping container: $CONTAINER_NAME'" >> "$JOB_CLEANUP_SCRIPT" echo "docker stop --time=15 \"$CONTAINER_NAME\" || echo 'Failed to stop container or container not running'" >> "$JOB_CLEANUP_SCRIPT" - docker run -di --user 0:$(id -g) --name $CONTAINER_NAME -v /data/builds:/data/builds -v `pwd`:/io --network host ghcr.io/foobarwidget/holy-build-box-x64 /hbb_exe/activate-exec bash - docker exec -i $CONTAINER_NAME /hbb_exe/activate-exec bash -x /io/build-full.sh "$GITHUB_REPOSITORY" "$GITHUB_SHA" "$BUILD_CORES" "$GITHUB_RUN_NUMBER" + docker run -di --user 0:$(id -g) --name $CONTAINER_NAME -v /data/builds:/data/builds -v `pwd`:/io --network host "$DEPENDENCY_IMAGE" bash + docker exec -i $CONTAINER_NAME bash -x /io/build-core.sh "$GITHUB_REPOSITORY" "$GITHUB_SHA" "$BUILD_CORES" "$GITHUB_RUN_NUMBER" docker stop $CONTAINER_NAME fi fi -echo "DONE BUILDING (HOST)" +echo "DONE BUILDING (HOST)" \ No newline at end of file