CI improvements (#867)

* Generate conan profile in CI

* Move linux build into main workflow

* Add saving/restoring conan data

* Move cache to Linux

* Fix error

* Change key to hash from conanfile

* Fix path error

* Populate cache only in develop branch

* Big refactor

- Move duplicated code to actions
- Isolate mac build from home directory
- Separate ccache and conan caches

* Fix errors

* Change ccache cache name and fix errors

* Always populate cache

* Use newer ccache on Linux

* Strip tests

* Better conan hash

.github/actions/build_clio/action.yml

@@ -0,0 +1,31 @@
+name: Build clio
+description: Build clio in build directory
+inputs:
+  conan_profile:
+    description: Conan profile name
+    required: true
+    default: default
+runs:
+  using: composite
+  steps:
+    - name: Get number of threads on mac
+      id: mac_threads
+      if: ${{ runner.os == 'macOS' }}
+      shell: bash
+      run: echo "num=$(($(sysctl -n hw.logicalcpu) - 2))" >> $GITHUB_OUTPUT
+
+    - name: Get number of threads on Linux
+      id: linux_threads
+      if: ${{ runner.os == 'Linux' }}
+      shell: bash
+      run: echo "num=$(($(nproc) - 2))" >> $GITHUB_OUTPUT
+
+    - name: Build Clio
+      shell: bash
+      run: |
+        mkdir -p build
+        cd build
+        threads_num=${{ steps.mac_threads.outputs.num || steps.linux_threads.outputs.num }}
+        conan install .. -of . -b missing -s build_type=Release -o clio:tests=True --profile ${{ inputs.conan_profile }}
+        cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake -DCMAKE_BUILD_TYPE=Release .. -G Ninja
+        cmake --build . --parallel $threads_num

.github/actions/clang_format/action.yml

@@ -1,3 +1,5 @@
+name: Check format
+description: Check format using clang-format-11
 runs:
   using: composite
   steps:

.github/actions/git_common_ancestor/action.yml

@@ -0,0 +1,14 @@
+name: Git common ancestor
+description: Find the closest common commit
+outputs:
+  commit:
+    description: Hash of commit
+    value: ${{ steps.find_common_ancestor.outputs.commit }}
+runs:
+  using: composite
+  steps:
+    - name: Find common git ancestor
+      id: find_common_ancestor
+      shell: bash
+      run: |
+        echo "commit=$(git merge-base --fork-point origin/develop)" >> $GITHUB_OUTPUT

.github/actions/linux_build/build.sh

@@ -1,20 +0,0 @@
-#!/usr/bin/env bash
-
-conan profile new default --detect
-conan profile update settings.compiler.cppstd=20 default
-conan profile update settings.compiler.libcxx=libstdc++11 default
-conan remote add --insert 0 conan-non-prod http://18.143.149.228:8081/artifactory/api/conan/conan-non-prod
-
-cd rippled
-conan export external/snappy snappy/1.1.10@
-conan export external/soci soci/4.0.3@
-conan export .
-conan install --output-folder build_rippled -install-folder build_rippled --build missing --settings build_type=Release
-cmake -B build_rippled -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake -DCMAKE_BUILD_TYPE=Release
-cmake --build build_rippled --target xrpl_core --parallel $(($(nproc) - 2))
-cd ..
-
-conan export external/cassandra
-conan install . -if build_clio -of build_clio --build missing --settings build_type=Release -o tests=True
-cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake -DCMAKE_BUILD_TYPE=Release -B build_clio
-cmake --build build_clio --parallel $(($(nproc) - 2))

.github/actions/restore_cache/action.yml

@@ -0,0 +1,41 @@
+name: Restore cache
+description: Find and restores conan and ccache cache
+inputs:
+  conan_dir:
+    description: Path to .conan directory
+    required: true
+  ccache_dir:
+    description: Path to .ccache directory
+    required: true
+outputs:
+  conan_hash:
+    description: Hash to use as a part of conan cache key
+    value: ${{ steps.conan_hash.outputs.hash }}
+runs:
+  using: composite
+  steps:
+    - name: Find common commit
+      id: git_common_ancestor
+      uses: ./.github/actions/git_common_ancestor
+
+    - name: Calculate conan hash
+      id: conan_hash
+      shell: bash
+      run: |
+        conan info . -j info.json
+        hash=$(cat info.json | jq '.[] | "\(.id), \(.package_revision)"' | shasum -a 256 | cut -d ' ' -f 1)
+        rm info.json
+        echo "hash=$hash" >> $GITHUB_OUTPUT
+
+    - name: Restore conan cache
+      uses: actions/cache/restore@v3
+      id: conan_cache
+      with:
+        path: ${{ inputs.conan_dir }}/data
+        key: clio-conan_data-${{ runner.os }}-develop-${{ steps.conan_hash.outputs.hash }}
+
+    - name: Restore ccache cache
+      uses: actions/cache/restore@v3
+      with:
+        path: ${{ inputs.ccache_dir }}
+        key: clio-ccache-${{ runner.os }}-develop-${{ steps.git_common_ancestor.outputs.commit }}

.github/actions/save_cache/action.yml

@@ -0,0 +1,37 @@
+name: Save cache
+description: Save conan and ccache cache for develop branch
+inputs:
+  conan_dir:
+    description: Path to .conan directory
+    required: true
+  conan_hash:
+    description: Hash to use as a part of conan cache key
+    required: true
+  ccache_dir:
+    description: Path to .ccache directory
+    required: true
+runs:
+  using: composite
+  steps:
+    - name: Find common commit
+      id: git_common_ancestor
+      uses: ./.github/actions/git_common_ancestor
+
+    - name: Cleanup conan directory from extra data
+      shell: bash
+      run: |
+        conan remove "*" -s -b -f
+
+    - name: Save conan cache
+      uses: actions/cache/save@v3
+      with:
+        path: ${{ inputs.conan_dir }}/data
+        key: clio-conan_data-${{ runner.os }}-develop-${{ inputs.conan_hash }}
+
+    - name: Save ccache cache
+      uses: actions/cache/save@v3
+      with:
+        path: ${{ inputs.ccache_dir }}
+        key: clio-ccache-${{ runner.os }}-develop-${{ steps.git_common_ancestor.outputs.commit }}
+
+

.github/actions/setup_conan/action.yml

@@ -0,0 +1,55 @@
+name: Setup conan
+description: Setup conan profile and artifactory
+outputs:
+  conan_profile:
+    description: Created conan profile name
+    value: ${{ steps.conan_export_output.outputs.conan_profile }}
+runs:
+  using: composite
+  steps:
+    - name: On mac
+      if: ${{ runner.os == 'macOS' }}
+      shell: bash
+      env:
+        CONAN_PROFILE: clio_clang_14
+      id: conan_setup_mac
+      run: |
+        echo "Creating $CONAN_PROFILE conan profile";
+        clang_path="$(brew --prefix llvm@14)/bin/clang"
+        clang_cxx_path="$(brew --prefix llvm@14)/bin/clang++"
+        conan profile new $CONAN_PROFILE --detect --force
+        conan profile update settings.compiler=clang $CONAN_PROFILE
+        conan profile update settings.compiler.version=14 $CONAN_PROFILE
+        conan profile update settings.compiler.cppstd=20 $CONAN_PROFILE
+        conan profile update "conf.tools.build:compiler_executables={\"c\": \"$clang_path\", \"cpp\": \"$clang_cxx_path\"}" $CONAN_PROFILE
+        conan profile update env.CC="$clang_path" $CONAN_PROFILE
+        conan profile update env.CXX="$clang_cxx_path" $CONAN_PROFILE
+        echo "created_conan_profile=$CONAN_PROFILE" >> $GITHUB_OUTPUT
+
+    - name: On linux
+      if: ${{ runner.os == 'Linux' }}
+      shell: bash
+      id: conan_setup_linux
+      run: |
+        conan profile new default --detect
+        conan profile update settings.compiler.cppstd=20 default
+        conan profile update settings.compiler.libcxx=libstdc++11 default
+        echo "created_conan_profile=default" >> $GITHUB_OUTPUT
+
+    - name: Export output variable
+      shell: bash
+      id: conan_export_output
+      run: |
+        echo "conan_profile=${{ steps.conan_setup_mac.outputs.created_conan_profile || steps.conan_setup_linux.outputs.created_conan_profile }}" >> $GITHUB_OUTPUT
+
+    - name: Add conan-non-prod artifactory
+      shell: bash
+      run: |
+        if [[ -z $(conan remote list | grep conan-non-prod) ]]; then
+            echo "Adding conan-non-prod"
+            conan remote add conan-non-prod http://18.143.149.228:8081/artifactory/api/conan/conan-non-prod
+        else
+            echo "Conan-non-prod is available"
+        fi
+
+

.github/workflows/build.yml

@@ -13,82 +13,114 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Run clang-format
-        uses: ./.github/actions/lint
+        uses: ./.github/actions/clang_format
 
   build_mac:
     name: Build macOS
     needs: lint
-    continue-on-error: true
     runs-on: [self-hosted, macOS]
+    env:
+      CCACHE_DIR: ${{ github.workspace }}/.ccache
+      CONAN_USER_HOME: ${{ github.workspace }}
     steps:
       - uses: actions/checkout@v3
         with:
-          path: clio
+            fetch-depth: 0
 
-      - name: List conan artifactory
+      - name: Install packages
         run: |
-          conan search
-          conan remote list
-          if [[ $(conan remote list |grep conan-non-prod| wc -c) -ne 0 ]]; then
-              echo "conan-non-prod is available"
-          else
-              echo "adding conan-non-prod"
-              conan remote add conan-non-prod http://18.143.149.228:8081/artifactory/api/conan/conan-non-prod
-          fi
+          brew install llvm@14 pkg-config ninja bison cmake ccache jq
 
-      - name: Install dependencies
-        run: |
-          brew install llvm@14 pkg-config ninja bison cmake
+      - name: Setup conan
+        uses: ./.github/actions/setup_conan
+        id: conan
 
-      - name: Setup environment for llvm-14
-        run: |
-          export PATH="/usr/local/opt/llvm@14/bin:$PATH"
-          export LDFLAGS="-L/usr/local/opt/llvm@14/lib -L/usr/local/opt/llvm@14/lib/c++ -Wl,-rpath,/usr/local/opt/llvm@14/lib/c++"
-          export CPPFLAGS="-I/usr/local/opt/llvm@14/include"
+      - name: Restore cache
+        uses: ./.github/actions/restore_cache
+        id: restore_cache
+        with:
+          conan_dir: ${{ env.CONAN_USER_HOME }}/.conan
+          ccache_dir: ${{ env.CCACHE_DIR }}
 
       - name: Build Clio
-        run: |
-          cd clio
-          mkdir -p build
-          cd build
-          conan install .. -of . -b missing -s build_type=Release -o clio:tests=True
-          cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake -DCMAKE_BUILD_TYPE=Release ..
-          cmake --build . --parallel $(($(sysctl -n hw.logicalcpu) - 2))
+        uses: ./.github/actions/build_clio
+        with:
+          conan_profile: ${{ steps.conan.outputs.conan_profile }}
+
+      - name: Strip tests
+        run: strip build/clio_tests
 
       - name: Upload clio_tests
         uses: actions/upload-artifact@v3
         with:
           name: clio_tests_mac
-          path: ./clio/build/clio_tests
+          path: build/clio_tests
+
+      - name: Save cache
+        uses: ./.github/actions/save_cache
+        with:
+          conan_dir: ${{ env.CONAN_USER_HOME }}/.conan
+          conan_hash: ${{ steps.restore_cache.outputs.conan_hash }}
+          ccache_dir: ${{ env.CCACHE_DIR }}
 
   build_linux:
     name: Build linux
     needs: lint
-    continue-on-error: true
     runs-on: [self-hosted, Linux]
     container:
-      image: thejohnfreeman/rippled-build-ubuntu:12e19cd9034b
+      image: conanio/gcc11:1.61.0
       options: --user root
+    env:
+      CCACHE_DIR: /root/.ccache
+      CONAN_USER_HOME: /root/
     steps:
       - name: Get Clio
         uses: actions/checkout@v3
-
-      - name: Get rippled
-        uses: actions/checkout@v3
         with:
-          repository: thejohnfreeman/rippled
-          ref: clio
-          path: rippled
+          fetch-depth: 0
+
+      - name: Install packages
+        run: |
+          apt update -qq
+          apt install -y jq
+
+      - name: Install ccache
+        run: |
+          wget https://github.com/ccache/ccache/releases/download/v4.8.3/ccache-4.8.3-linux-x86_64.tar.xz
+          tar xf ./ccache-4.8.3-linux-x86_64.tar.xz
+          mv ./ccache-4.8.3-linux-x86_64/ccache /usr/bin/ccache
+
+      - name: Fix git permissions
+        run: git config --global --add safe.directory $PWD
+
+      - name: Setup conan
+        uses: ./.github/actions/setup_conan
+
+      - name: Restore cache
+        uses: ./.github/actions/restore_cache
+        id: restore_cache
+        with:
+          conan_dir: ${{ env.CONAN_USER_HOME }}/.conan
+          ccache_dir: ${{ env.CCACHE_DIR }}
 
       - name: Build Clio
-        run: |
-          ./.github/actions/linux_build/build.sh
+        uses: ./.github/actions/build_clio
+
+      - name: Strip tests
+        run: strip build/clio_tests
 
       - name: Upload clio_tests
         uses: actions/upload-artifact@v3
         with:
           name: clio_tests_linux
-          path: ./build_clio/clio_tests
+          path: build/clio_tests
+
+      - name: Save cache
+        uses: ./.github/actions/save_cache
+        with:
+          conan_dir: ${{ env.CONAN_USER_HOME }}/.conan
+          conan_hash: ${{ steps.restore_cache.outputs.conan_hash }}
+          ccache_dir: ${{ env.CCACHE_DIR }}
 
   test_mac:
     needs: build_mac

.gitignore

@@ -1,5 +1,5 @@
 *clio*.log
-build*/
+/build*/
 .build
 .cache
 .vscode

CMake/Ccache.cmake

@@ -0,0 +1,5 @@
+find_program (CCACHE_PATH "ccache")
+if (CCACHE_PATH)
+    set (CMAKE_CXX_COMPILER_LAUNCHER "${CCACHE_PATH}")
+    message (STATUS "Using ccache: ${CCACHE_PATH}")
+endif ()

CMakeLists.txt

@@ -16,6 +16,7 @@ set_property (CACHE san PROPERTY STRINGS ";undefined;memory;address;thread")
 # ========================================================================== #
 
 # Include required modules
+include (CMake/Ccache.cmake)
 include (CheckCXXCompilerFlag)
 
 if (verbose)