ci: Generate protocol_autogen only once in CI (#7918)

.github/workflows/on-pr.yml

@@ -82,6 +82,7 @@ jobs:
             .github/scripts/strategy-matrix/**
             .github/workflows/reusable-build-test-config.yml
             .github/workflows/reusable-build-test.yml
+            .github/workflows/reusable-check-autogen.yml
             .github/workflows/reusable-clang-tidy.yml
             .github/workflows/reusable-package.yml
             .github/workflows/reusable-strategy-matrix.yml
@@ -126,6 +127,11 @@ jobs:
     outputs:
       go: ${{ steps.go.outputs.go == 'true' }}
 
+  check-autogen:
+    needs: should-run
+    if: ${{ needs.should-run.outputs.go == 'true' }}
+    uses: ./.github/workflows/reusable-check-autogen.yml
+
   check-levelization:
     needs: should-run
     if: ${{ needs.should-run.outputs.go == 'true' }}
@@ -200,6 +206,7 @@ jobs:
   passed:
     if: failure() || cancelled()
     needs:
+      - check-autogen
       - check-levelization
       - check-rename
       - clang-tidy

.github/workflows/on-trigger.yml

@@ -20,6 +20,7 @@ on:
       - ".github/scripts/strategy-matrix/**"
       - ".github/workflows/reusable-build-test-config.yml"
       - ".github/workflows/reusable-build-test.yml"
+      - ".github/workflows/reusable-check-autogen.yml"
       - ".github/workflows/reusable-clang-tidy.yml"
       - ".github/workflows/reusable-package.yml"
       - ".github/workflows/reusable-strategy-matrix.yml"
@@ -67,6 +68,9 @@ defaults:
     shell: bash
 
 jobs:
+  check-autogen:
+    uses: ./.github/workflows/reusable-check-autogen.yml
+
   clang-tidy:
     uses: ./.github/workflows/reusable-clang-tidy.yml
     permissions:

.github/workflows/reusable-build-test-config.yml

@@ -176,9 +176,9 @@ jobs:
               ..
 
       # Export the sanitizer options before any instrumented binary runs. The
-      # protocol code-gen and build steps below invoke instrumented dependency
-      # tools (protoc, grpc), so setting UBSAN_OPTIONS here lets the UBSan
-      # suppression list silence their diagnostics too, not just at test time.
+      # build step below invokes instrumented dependency tools (protoc, grpc),
+      # so setting UBSAN_OPTIONS here lets the UBSan suppression list silence
+      # their diagnostics too, not just at test time.
       # GITHUB_WORKSPACE (not the github.workspace context) is used so the path
       # resolves correctly inside the container job.
       - name: Set sanitizer options
@@ -196,32 +196,6 @@ jobs:
           echo "UBSAN_OPTIONS=include=${SUPP}/runtime-ubsan-options.txt:suppressions=${SUPP}/ubsan.supp" >>${GITHUB_ENV}
           echo "LSAN_OPTIONS=include=${SUPP}/runtime-lsan-options.txt:suppressions=${SUPP}/lsan.supp" >>${GITHUB_ENV}
 
-      - name: Check protocol autogen files are up-to-date
-        working-directory: ${{ env.BUILD_DIR }}
-        env:
-          MESSAGE: |
-
-            The generated protocol wrapper classes are out of date.
-
-            This typically happens when the macro files or generator scripts
-            have changed but the generated files were not regenerated.
-
-            To fix this:
-              1. Run: cmake --build . --target setup_code_gen
-              2. Run: cmake --build . --target code_gen
-              3. Commit and push the regenerated files
-        run: |
-          set -e
-          cmake --build . --target setup_code_gen
-          cmake --build . --target code_gen
-          DIFF=$(git -C .. status --porcelain -- include/xrpl/protocol_autogen src/tests/libxrpl/protocol_autogen)
-          if [ -n "${DIFF}" ]; then
-              echo "::error::Generated protocol files are out of date"
-              git -C .. diff -- include/xrpl/protocol_autogen src/tests/libxrpl/protocol_autogen
-              echo "${MESSAGE}"
-              exit 1
-          fi
-
       - name: Build the binary
         working-directory: ${{ env.BUILD_DIR }}
         env:

.github/workflows/reusable-check-autogen.yml

@@ -0,0 +1,76 @@
+# This workflow checks that the generated protocol wrapper classes are
+# up-to-date with the macro files and generator scripts they are produced from,
+# see more info in include/xrpl/protocol_autogen/README.md.
+name: Check autogen
+
+# This workflow can only be triggered by other workflows.
+on: workflow_call
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-autogen
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  BUILD_DIR: build/codegen
+
+jobs:
+  autogen:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Set up Python
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: "3.13"
+
+      # Code generation is pure Python, so the standalone project below offers
+      # the same targets as the main build without needing its dependencies or
+      # a compiler, which keeps this job down to a few seconds.
+      - name: Configure CMake
+        run: cmake -S cmake/codegen -B "${BUILD_DIR}"
+
+      - name: Install code generation dependencies
+        run: cmake --build "${BUILD_DIR}" --target setup_code_gen
+
+      - name: Generate code
+        run: cmake --build "${BUILD_DIR}" --target code_gen
+
+      - name: Check for differences
+        env:
+          MESSAGE: |
+
+            The generated protocol wrapper classes are out of date.
+
+            This typically happens when the macro files or generator scripts
+            have changed but the generated files were not regenerated.
+
+            Run the following from the repository root, then commit and push
+            the regenerated files. This needs neither the dependencies nor a
+            compiler. See include/xrpl/protocol_autogen/README.md for more info.
+
+              cmake -S cmake/codegen -B build/codegen
+              cmake --build build/codegen --target setup_code_gen
+              cmake --build build/codegen --target code_gen
+
+            In an already configured build directory, the 'setup_code_gen' and
+            'code_gen' targets do the same thing.
+        run: |
+          # Record untracked files in the index without staging their contents,
+          # so that classes generated for a newly added transaction or ledger
+          # entry type show up in the diff below rather than silently as an
+          # empty one.
+          git add --intent-to-add .
+          DIFF=$(git status --porcelain)
+          if [ -n "${DIFF}" ]; then
+              # Print the differences to give the contributor a hint about what to
+              # expect when running code generation on their own machine.
+              git diff
+              echo "${MESSAGE}"
+              exit 1
+          fi

BUILD.md

@@ -247,7 +247,17 @@ cmake --build . --target setup_code_gen  # create venv and install dependencies
 cmake --build . --target code_gen        # regenerate code
 ```
 
-The regenerated files should be committed alongside your changes.
+The same targets are also available as a standalone project, which does not
+need the dependencies to be configured first:
+
+```
+cmake -S cmake/codegen -B build/codegen
+cmake --build build/codegen --target setup_code_gen
+cmake --build build/codegen --target code_gen
+```
+
+The regenerated files should be committed alongside your changes. CI verifies
+that they are up-to-date.
 
 ## Coverage report
 

cmake/XrplProtocolAutogen.cmake

@@ -2,21 +2,22 @@
    Protocol Autogen - Code generation for protocol wrapper classes
 #]===================================================================]
 
+# The repository root, derived from the location of this file rather than from
+# the including project, so that the targets below can also be offered on their
+# own by cmake/codegen/CMakeLists.txt.
+get_filename_component(XRPL_ROOT "${CMAKE_CURRENT_LIST_DIR}/.." ABSOLUTE)
+
 set(CODEGEN_VENV_DIR
-    "${CMAKE_CURRENT_SOURCE_DIR}/.venv"
+    "${XRPL_ROOT}/.venv"
     CACHE PATH
     "Path to a Python virtual environment for code generation. A venv will be created here by setup_code_gen and used to run generation scripts."
 )
 
 # Directory paths
-set(MACRO_DIR "${CMAKE_CURRENT_SOURCE_DIR}/include/xrpl/protocol/detail")
-set(AUTOGEN_HEADER_DIR
-    "${CMAKE_CURRENT_SOURCE_DIR}/include/xrpl/protocol_autogen"
-)
-set(AUTOGEN_TEST_DIR
-    "${CMAKE_CURRENT_SOURCE_DIR}/src/tests/libxrpl/protocol_autogen"
-)
-set(SCRIPTS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/cmake/scripts/codegen")
+set(MACRO_DIR "${XRPL_ROOT}/include/xrpl/protocol/detail")
+set(AUTOGEN_HEADER_DIR "${XRPL_ROOT}/include/xrpl/protocol_autogen")
+set(AUTOGEN_TEST_DIR "${XRPL_ROOT}/src/tests/libxrpl/protocol_autogen")
+set(SCRIPTS_DIR "${XRPL_ROOT}/cmake/scripts/codegen")
 
 # Input macro files
 set(TRANSACTIONS_MACRO "${MACRO_DIR}/transactions.macro")
@@ -114,14 +115,14 @@ if(CODEGEN_VENV_DIR)
         setup_code_gen
         COMMAND ${Python3_EXECUTABLE} -m venv "${CODEGEN_VENV_DIR}"
         COMMAND ${CODEGEN_PYTHON} -m pip install -r "${REQUIREMENTS_FILE}"
-        WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+        WORKING_DIRECTORY "${XRPL_ROOT}"
         COMMENT "Creating venv and installing code generation dependencies..."
     )
 else()
     add_custom_target(
         setup_code_gen
         COMMAND ${Python3_EXECUTABLE} -m pip install -r "${REQUIREMENTS_FILE}"
-        WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+        WORKING_DIRECTORY "${XRPL_ROOT}"
         COMMENT "Installing code generation dependencies..."
     )
 endif()
@@ -139,8 +140,8 @@ add_custom_target(
         -DSFIELDS_MACRO=${SFIELDS_MACRO}
         -DAUTOGEN_HEADER_DIR=${AUTOGEN_HEADER_DIR}
         -DAUTOGEN_TEST_DIR=${AUTOGEN_TEST_DIR} -P
-        "${CMAKE_CURRENT_SOURCE_DIR}/cmake/XrplProtocolAutogenRun.cmake"
-    WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+        "${CMAKE_CURRENT_LIST_DIR}/XrplProtocolAutogenRun.cmake"
+    WORKING_DIRECTORY "${XRPL_ROOT}"
     COMMENT "Running protocol code generation..."
     SOURCES ${ALL_INPUT_FILES}
 )

cmake/codegen/CMakeLists.txt

@@ -0,0 +1,21 @@
+#[===================================================================[
+   Protocol Autogen - Standalone project
+
+   Exposes the 'setup_code_gen' and 'code_gen' targets on their own, without
+   configuring the rest of xrpl. Code generation is pure Python, so this needs
+   neither the dependencies nor a compiler, which makes it usable in CI and by
+   contributors who only want to regenerate the protocol wrapper classes:
+
+     cmake -S cmake/codegen -B build/codegen
+     cmake --build build/codegen --target setup_code_gen
+     cmake --build build/codegen --target code_gen
+
+   The targets are identical to the ones offered by the top-level build, since
+   both come from cmake/XrplProtocolAutogen.cmake.
+#]===================================================================]
+
+cmake_minimum_required(VERSION 3.16)
+
+project(xrpl_codegen LANGUAGES NONE)
+
+include("${CMAKE_CURRENT_LIST_DIR}/../XrplProtocolAutogen.cmake")

include/xrpl/protocol_autogen/README.md

@@ -23,6 +23,16 @@ By default, `CODEGEN_VENV_DIR` points to `.venv` in the project root. The
 `setup_code_gen` target creates a venv there and installs the required packages.
 The `code_gen` target then uses the venv's Python interpreter to run generation.
 
+Generation is pure Python, so the same targets are also available as a
+standalone project that needs neither the dependencies nor a compiler. This is
+what CI uses, and it is handy if you only want to regenerate these files:
+
+```bash
+cmake -S cmake/codegen -B build/codegen
+cmake --build build/codegen --target setup_code_gen
+cmake --build build/codegen --target code_gen
+```
+
 ### Python Dependencies
 
 The code generation requires the following Python packages (installed by `setup_code_gen`):