chore: Move codegen venv setup into build stage (#6617)

Co-authored-by: JCW <a1q123456@users.noreply.github.com>
Co-authored-by: Bart <bthomee@users.noreply.github.com>

.github/workflows/reusable-build-test-config.yml

@@ -153,6 +153,32 @@ jobs:
             ${CMAKE_ARGS} \
             ..
 
+      - name: Check protocol autogen files are up-to-date
+        working-directory: ${{ env.BUILD_DIR }}
+        env:
+          MESSAGE: |
+
+            The generated protocol wrapper classes are out of date.
+
+            This typically happens when the macro files or generator scripts
+            have changed but the generated files were not regenerated.
+
+            To fix this:
+              1. Run: cmake --build . --target setup_code_gen
+              2. Run: cmake --build . --target code_gen
+              3. Commit and push the regenerated files
+        run: |
+          set -e
+          cmake --build . --target setup_code_gen
+          cmake --build . --target code_gen
+          DIFF=$(git -C .. status --porcelain -- include/xrpl/protocol_autogen src/tests/libxrpl/protocol_autogen)
+          if [ -n "${DIFF}" ]; then
+            echo "::error::Generated protocol files are out of date"
+            git -C .. diff -- include/xrpl/protocol_autogen src/tests/libxrpl/protocol_autogen
+            echo "${MESSAGE}"
+            exit 1
+          fi
+
       - name: Build the binary
         working-directory: ${{ env.BUILD_DIR }}
         env:
@@ -166,29 +192,6 @@ jobs:
             --parallel "${BUILD_NPROC}" \
             --target "${CMAKE_TARGET}"
 
-      - name: Check protocol autogen files are up-to-date
-        env:
-          MESSAGE: |
-
-            The generated protocol wrapper classes are out of date.
-
-            This typically happens when your branch is behind develop and
-            the macro files or generator scripts have changed.
-
-            To fix this:
-              1. Update your branch from develop (merge or rebase)
-              2. Build with code generation enabled (XRPL_NO_CODEGEN=OFF)
-              3. Commit and push the regenerated files
-        run: |
-          set -e
-          DIFF=$(git status --porcelain -- include/xrpl/protocol_autogen src/tests/libxrpl/protocol_autogen)
-          if [ -n "${DIFF}" ]; then
-            echo "::error::Generated protocol files are out of date"
-            git diff -- include/xrpl/protocol_autogen src/tests/libxrpl/protocol_autogen
-            echo "${MESSAGE}"
-            exit 1
-          fi
-
       - name: Show ccache statistics
         if: ${{ inputs.ccache_enabled }}
         run: |

.gitignore

@@ -13,6 +13,7 @@
 Debug/
 Release/
 /.build/
+/.venv/
 /build/
 /db/
 /out.txt

BUILD.md

@@ -459,6 +459,21 @@ install ccache --version 4.11.3 --allow-downgrade`.
    The location of `xrpld` binary in your build directory depends on your
    CMake generator. Pass `--help` to see the rest of the command line options.
 
+## Code generation
+
+The protocol wrapper classes in `include/xrpl/protocol_autogen/` are generated
+from macro definition files in `include/xrpl/protocol/detail/`. If you modify
+the macro files (e.g. `transactions.macro`, `ledger_entries.macro`) or the
+generation scripts/templates in `cmake/scripts/codegen/`, you need to regenerate the
+files:
+
+```
+cmake --build . --target setup_code_gen  # create venv and install dependencies (once)
+cmake --build . --target code_gen        # regenerate code
+```
+
+The regenerated files should be committed alongside your changes.
+
 ## Coverage report
 
 The coverage report is intended for developers using compilers GCC

CMakeLists.txt

@@ -132,6 +132,7 @@ if(coverage)
 endif()
 
 include(XrplCore)
+include(XrplProtocolAutogen)
 include(XrplInstall)
 include(XrplValidatorKeys)
 

cmake/XrplCore.cmake

@@ -108,24 +108,12 @@ target_link_libraries(
 )
 
 # Level 05
-## Set up code generation for protocol_autogen module
-include(XrplProtocolAutogen)
-# Must call setup_protocol_autogen before add_module so that:
-# 1. Stale generated files are cleared before GLOB runs
-# 2. Output file list is known for custom commands
-setup_protocol_autogen()
-
 add_module(xrpl protocol_autogen)
 target_link_libraries(
     xrpl.libxrpl.protocol_autogen
     PUBLIC xrpl.libxrpl.protocol
 )
 
-# Ensure code generation runs before compiling protocol_autogen
-if(TARGET protocol_autogen_generate)
-    add_dependencies(xrpl.libxrpl.protocol_autogen protocol_autogen_generate)
-endif()
-
 # Level 06
 add_module(xrpl core)
 target_link_libraries(

cmake/XrplProtocolAutogen.cmake

@@ -2,308 +2,145 @@
    Protocol Autogen - Code generation for protocol wrapper classes
 #]===================================================================]
 
-# Options for code generation
-option(
-    XRPL_NO_CODEGEN
-    "Disable code generation (use pre-generated files from repository)"
-    OFF
-)
 set(CODEGEN_VENV_DIR
-    ""
+    "${CMAKE_CURRENT_SOURCE_DIR}/.venv"
     CACHE PATH
-    "Path to Python virtual environment for code generation. If provided, automatic venv setup is skipped."
+    "Path to a Python virtual environment for code generation. A venv will be created here by setup_code_gen and used to run generation scripts."
 )
 
-# Function to set up code generation for protocol_autogen module
-# This runs at configure time to generate C++ wrapper classes from macro files
-function(setup_protocol_autogen)
-    # Directory paths
-    set(MACRO_DIR "${CMAKE_CURRENT_SOURCE_DIR}/include/xrpl/protocol/detail")
-    set(AUTOGEN_HEADER_DIR
-        "${CMAKE_CURRENT_SOURCE_DIR}/include/xrpl/protocol_autogen"
-    )
-    set(AUTOGEN_TEST_DIR
-        "${CMAKE_CURRENT_SOURCE_DIR}/src/tests/libxrpl/protocol_autogen"
-    )
-    set(SCRIPTS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/scripts")
+# Directory paths
+set(MACRO_DIR "${CMAKE_CURRENT_SOURCE_DIR}/include/xrpl/protocol/detail")
+set(AUTOGEN_HEADER_DIR
+    "${CMAKE_CURRENT_SOURCE_DIR}/include/xrpl/protocol_autogen"
+)
+set(AUTOGEN_TEST_DIR
+    "${CMAKE_CURRENT_SOURCE_DIR}/src/tests/libxrpl/protocol_autogen"
+)
+set(SCRIPTS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/cmake/scripts/codegen")
 
-    # Input macro files
-    set(TRANSACTIONS_MACRO "${MACRO_DIR}/transactions.macro")
-    set(LEDGER_ENTRIES_MACRO "${MACRO_DIR}/ledger_entries.macro")
-    set(SFIELDS_MACRO "${MACRO_DIR}/sfields.macro")
+# Input macro files
+set(TRANSACTIONS_MACRO "${MACRO_DIR}/transactions.macro")
+set(LEDGER_ENTRIES_MACRO "${MACRO_DIR}/ledger_entries.macro")
+set(SFIELDS_MACRO "${MACRO_DIR}/sfields.macro")
 
-    # Python scripts and templates
-    set(GENERATE_TX_SCRIPT "${SCRIPTS_DIR}/generate_tx_classes.py")
-    set(GENERATE_LEDGER_SCRIPT "${SCRIPTS_DIR}/generate_ledger_classes.py")
-    set(REQUIREMENTS_FILE "${SCRIPTS_DIR}/requirements.txt")
-    set(MACRO_PARSER_COMMON "${SCRIPTS_DIR}/macro_parser_common.py")
-    set(TX_TEMPLATE "${SCRIPTS_DIR}/templates/Transaction.h.mako")
-    set(TX_TEST_TEMPLATE "${SCRIPTS_DIR}/templates/TransactionTests.cpp.mako")
-    set(LEDGER_TEMPLATE "${SCRIPTS_DIR}/templates/LedgerEntry.h.mako")
-    set(LEDGER_TEST_TEMPLATE
-        "${SCRIPTS_DIR}/templates/LedgerEntryTests.cpp.mako"
+# Python scripts and templates
+set(GENERATE_TX_SCRIPT "${SCRIPTS_DIR}/generate_tx_classes.py")
+set(GENERATE_LEDGER_SCRIPT "${SCRIPTS_DIR}/generate_ledger_classes.py")
+set(REQUIREMENTS_FILE "${SCRIPTS_DIR}/requirements.txt")
+set(MACRO_PARSER_COMMON "${SCRIPTS_DIR}/macro_parser_common.py")
+set(TX_TEMPLATE "${SCRIPTS_DIR}/templates/Transaction.h.mako")
+set(TX_TEST_TEMPLATE "${SCRIPTS_DIR}/templates/TransactionTests.cpp.mako")
+set(LEDGER_TEMPLATE "${SCRIPTS_DIR}/templates/LedgerEntry.h.mako")
+set(LEDGER_TEST_TEMPLATE "${SCRIPTS_DIR}/templates/LedgerEntryTests.cpp.mako")
+set(ALL_INPUT_FILES
+    "${TRANSACTIONS_MACRO}"
+    "${LEDGER_ENTRIES_MACRO}"
+    "${SFIELDS_MACRO}"
+    "${GENERATE_TX_SCRIPT}"
+    "${GENERATE_LEDGER_SCRIPT}"
+    "${REQUIREMENTS_FILE}"
+    "${MACRO_PARSER_COMMON}"
+    "${TX_TEMPLATE}"
+    "${TX_TEST_TEMPLATE}"
+    "${LEDGER_TEMPLATE}"
+    "${LEDGER_TEST_TEMPLATE}"
+)
+
+# Create output directories
+file(MAKE_DIRECTORY "${AUTOGEN_HEADER_DIR}/transactions")
+file(MAKE_DIRECTORY "${AUTOGEN_HEADER_DIR}/ledger_entries")
+file(MAKE_DIRECTORY "${AUTOGEN_TEST_DIR}/ledger_entries")
+file(MAKE_DIRECTORY "${AUTOGEN_TEST_DIR}/transactions")
+
+# Find Python3
+if(NOT Python3_EXECUTABLE)
+    find_package(Python3 COMPONENTS Interpreter QUIET)
+endif()
+
+if(NOT Python3_EXECUTABLE)
+    find_program(Python3_EXECUTABLE NAMES python3 python)
+endif()
+
+if(NOT Python3_EXECUTABLE)
+    message(
+        WARNING
+        "Python3 not found. The 'code_gen' and 'setup_code_gen' targets will not be available."
     )
+    return()
+endif()
 
-    # Check if code generation is disabled
-    if(XRPL_NO_CODEGEN)
+# Warn if pip is configured with a non-default index (may need VPN).
+execute_process(
+    COMMAND ${Python3_EXECUTABLE} -m pip config get global.index-url
+    OUTPUT_VARIABLE PIP_INDEX_URL
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+    ERROR_QUIET
+    RESULT_VARIABLE PIP_CONFIG_RESULT
+)
+if(PIP_CONFIG_RESULT EQUAL 0 AND PIP_INDEX_URL)
+    if(
+        NOT PIP_INDEX_URL STREQUAL "https://pypi.org/simple"
+        AND NOT PIP_INDEX_URL STREQUAL "https://pypi.python.org/simple"
+    )
         message(
             WARNING
-            "Protocol autogen: Code generation is disabled (XRPL_NO_CODEGEN=ON). "
-            "Generated files may be out of date."
+            "Private pip index URL detected: ${PIP_INDEX_URL}\n"
+            "You may need to connect to VPN to access this URL."
         )
-        return()
     endif()
+endif()
 
-    # Create output directories
-    file(MAKE_DIRECTORY "${AUTOGEN_HEADER_DIR}/transactions")
-    file(MAKE_DIRECTORY "${AUTOGEN_HEADER_DIR}/ledger_entries")
-    file(MAKE_DIRECTORY "${AUTOGEN_TEST_DIR}/ledger_entries")
-    file(MAKE_DIRECTORY "${AUTOGEN_TEST_DIR}/transactions")
-
-    # Find Python3 - check if already found by Conan or find it ourselves
-    if(NOT Python3_EXECUTABLE)
-        find_package(Python3 COMPONENTS Interpreter QUIET)
-    endif()
-
-    if(NOT Python3_EXECUTABLE)
-        # Try finding python3 executable directly
-        find_program(Python3_EXECUTABLE NAMES python3 python)
-    endif()
-
-    if(NOT Python3_EXECUTABLE)
-        message(
-            FATAL_ERROR
-            "Python3 not found. Code generation cannot proceed.\n"
-            "Please install Python 3, or set -DXRPL_NO_CODEGEN=ON to use existing generated files."
-        )
-        return()
-    endif()
-
-    message(STATUS "Using Python3 for code generation: ${Python3_EXECUTABLE}")
-
-    # Set up Python virtual environment for code generation
-    if(CODEGEN_VENV_DIR)
-        # User-provided venv - skip automatic setup
-        set(VENV_DIR "${CODEGEN_VENV_DIR}")
-        message(STATUS "Using user-provided Python venv: ${VENV_DIR}")
-    else()
-        # Use default venv in build directory
-        set(VENV_DIR "${CMAKE_CURRENT_BINARY_DIR}/codegen_venv")
-    endif()
-
-    # Determine the Python executable path in the venv
+# Determine which Python interpreter to use for code generation.
+if(CODEGEN_VENV_DIR)
     if(WIN32)
-        set(VENV_PYTHON "${VENV_DIR}/Scripts/python.exe")
-        set(VENV_PIP "${VENV_DIR}/Scripts/pip.exe")
+        set(CODEGEN_PYTHON "${CODEGEN_VENV_DIR}/Scripts/python.exe")
     else()
-        set(VENV_PYTHON "${VENV_DIR}/bin/python")
-        set(VENV_PIP "${VENV_DIR}/bin/pip")
+        set(CODEGEN_PYTHON "${CODEGEN_VENV_DIR}/bin/python")
     endif()
-
-    # Only auto-setup venv if not user-provided
-    if(NOT CODEGEN_VENV_DIR)
-        # Check if venv needs to be created or updated
-        set(VENV_NEEDS_UPDATE FALSE)
-        if(NOT EXISTS "${VENV_PYTHON}")
-            set(VENV_NEEDS_UPDATE TRUE)
-            message(
-                STATUS
-                "Creating Python virtual environment for code generation..."
-            )
-        elseif(
-            "${REQUIREMENTS_FILE}"
-                IS_NEWER_THAN
-                "${VENV_DIR}/.requirements_installed"
-        )
-            set(VENV_NEEDS_UPDATE TRUE)
-            message(
-                STATUS
-                "Updating Python virtual environment (requirements changed)..."
-            )
-        endif()
-
-        # Create/update virtual environment if needed
-        if(VENV_NEEDS_UPDATE)
-            message(
-                STATUS
-                "Setting up Python virtual environment at ${VENV_DIR}"
-            )
-            execute_process(
-                COMMAND ${Python3_EXECUTABLE} -m venv "${VENV_DIR}"
-                RESULT_VARIABLE VENV_RESULT
-                ERROR_VARIABLE VENV_ERROR
-            )
-            if(NOT VENV_RESULT EQUAL 0)
-                message(
-                    FATAL_ERROR
-                    "Failed to create virtual environment: ${VENV_ERROR}"
-                )
-            endif()
-
-            # Check pip index URL configuration
-            execute_process(
-                COMMAND ${VENV_PIP} config get global.index-url
-                OUTPUT_VARIABLE PIP_INDEX_URL
-                OUTPUT_STRIP_TRAILING_WHITESPACE
-                ERROR_QUIET
-            )
-
-            # Default PyPI URL
-            set(DEFAULT_PIP_INDEX "https://pypi.org/simple")
-
-            # Show warning if using non-default index
-            if(PIP_INDEX_URL AND NOT PIP_INDEX_URL STREQUAL "")
-                if(NOT PIP_INDEX_URL STREQUAL DEFAULT_PIP_INDEX)
-                    message(
-                        WARNING
-                        "Private pip index URL detected: ${PIP_INDEX_URL}\n"
-                        "You may need to connect to VPN to access this URL."
-                    )
-                endif()
-            endif()
-
-            message(STATUS "Installing Python dependencies...")
-            execute_process(
-                COMMAND ${VENV_PIP} install --upgrade pip
-                RESULT_VARIABLE PIP_UPGRADE_RESULT
-                OUTPUT_QUIET
-                ERROR_VARIABLE PIP_UPGRADE_ERROR
-            )
-            if(NOT PIP_UPGRADE_RESULT EQUAL 0)
-                message(WARNING "Failed to upgrade pip: ${PIP_UPGRADE_ERROR}")
-            endif()
-
-            execute_process(
-                COMMAND ${VENV_PIP} install -r "${REQUIREMENTS_FILE}"
-                RESULT_VARIABLE PIP_INSTALL_RESULT
-                ERROR_VARIABLE PIP_INSTALL_ERROR
-            )
-            if(NOT PIP_INSTALL_RESULT EQUAL 0)
-                message(
-                    FATAL_ERROR
-                    "Failed to install Python dependencies: ${PIP_INSTALL_ERROR}"
-                )
-            endif()
-
-            # Mark requirements as installed
-            file(TOUCH "${VENV_DIR}/.requirements_installed")
-            message(STATUS "Python virtual environment ready")
-        endif()
-    endif()
-
-    # At configure time - get list of output files for transactions
-    execute_process(
-        COMMAND
-            ${VENV_PYTHON} "${GENERATE_TX_SCRIPT}" "${TRANSACTIONS_MACRO}"
-            --header-dir "${AUTOGEN_HEADER_DIR}/transactions" --test-dir
-            "${AUTOGEN_TEST_DIR}/transactions" --list-outputs
-        OUTPUT_VARIABLE TX_OUTPUT_FILES
-        OUTPUT_STRIP_TRAILING_WHITESPACE
-        RESULT_VARIABLE TX_LIST_RESULT
-        ERROR_VARIABLE TX_LIST_ERROR
+else()
+    set(CODEGEN_PYTHON "${Python3_EXECUTABLE}")
+    message(
+        WARNING
+        "CODEGEN_VENV_DIR is not set. Dependencies will be installed globally.\n"
+        "If this is not intended, reconfigure with:\n"
+        "  cmake . -UCODEGEN_VENV_DIR"
     )
-    if(NOT TX_LIST_RESULT EQUAL 0)
-        message(
-            FATAL_ERROR
-            "Failed to list transaction output files:\n${TX_LIST_ERROR}"
-        )
-    endif()
-    # Convert newline-separated list to CMake list
-    string(REPLACE "\\" "/" TX_OUTPUT_FILES "${TX_OUTPUT_FILES}")
-    string(REPLACE "\n" ";" TX_OUTPUT_FILES "${TX_OUTPUT_FILES}")
+endif()
 
-    # At configure time - get list of output files for ledger entries
-    execute_process(
-        COMMAND
-            ${VENV_PYTHON} "${GENERATE_LEDGER_SCRIPT}" "${LEDGER_ENTRIES_MACRO}"
-            --header-dir "${AUTOGEN_HEADER_DIR}/ledger_entries" --test-dir
-            "${AUTOGEN_TEST_DIR}/ledger_entries" --list-outputs
-        OUTPUT_VARIABLE LEDGER_OUTPUT_FILES
-        OUTPUT_STRIP_TRAILING_WHITESPACE
-        RESULT_VARIABLE LEDGER_LIST_RESULT
-        ERROR_VARIABLE LEDGER_LIST_ERROR
-    )
-    if(NOT LEDGER_LIST_RESULT EQUAL 0)
-        message(
-            FATAL_ERROR
-            "Failed to list ledger entry output files:\n${LEDGER_LIST_ERROR}"
-        )
-    endif()
-    # Convert newline-separated list to CMake list
-    string(REPLACE "\\" "/" LEDGER_OUTPUT_FILES "${LEDGER_OUTPUT_FILES}")
-    string(REPLACE "\n" ";" LEDGER_OUTPUT_FILES "${LEDGER_OUTPUT_FILES}")
-
-    # Custom command to generate transaction classes at build time
-    add_custom_command(
-        OUTPUT ${TX_OUTPUT_FILES}
-        COMMAND
-            ${VENV_PYTHON} "${GENERATE_TX_SCRIPT}" "${TRANSACTIONS_MACRO}"
-            --header-dir "${AUTOGEN_HEADER_DIR}/transactions" --test-dir
-            "${AUTOGEN_TEST_DIR}/transactions" --sfields-macro
-            "${SFIELDS_MACRO}"
-        WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
-        DEPENDS
-            "${TRANSACTIONS_MACRO}"
-            "${SFIELDS_MACRO}"
-            "${GENERATE_TX_SCRIPT}"
-            "${MACRO_PARSER_COMMON}"
-            "${TX_TEMPLATE}"
-            "${TX_TEST_TEMPLATE}"
-            "${REQUIREMENTS_FILE}"
-        COMMENT "Generating transaction classes from transactions.macro..."
-        VERBATIM
-    )
-
-    # Custom command to generate ledger entry classes at build time
-    add_custom_command(
-        OUTPUT ${LEDGER_OUTPUT_FILES}
-        COMMAND
-            ${VENV_PYTHON} "${GENERATE_LEDGER_SCRIPT}" "${LEDGER_ENTRIES_MACRO}"
-            --header-dir "${AUTOGEN_HEADER_DIR}/ledger_entries" --test-dir
-            "${AUTOGEN_TEST_DIR}/ledger_entries" --sfields-macro
-            "${SFIELDS_MACRO}"
-        WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
-        DEPENDS
-            "${LEDGER_ENTRIES_MACRO}"
-            "${SFIELDS_MACRO}"
-            "${GENERATE_LEDGER_SCRIPT}"
-            "${MACRO_PARSER_COMMON}"
-            "${LEDGER_TEMPLATE}"
-            "${LEDGER_TEST_TEMPLATE}"
-            "${REQUIREMENTS_FILE}"
-        COMMENT "Generating ledger entry classes from ledger_entries.macro..."
-        VERBATIM
-    )
-
-    # Create a custom target that depends on all generated files
+# Custom target to create a venv and install Python dependencies.
+# Run manually with: cmake --build . --target setup_code_gen
+if(CODEGEN_VENV_DIR)
     add_custom_target(
-        protocol_autogen_generate
-        DEPENDS ${TX_OUTPUT_FILES} ${LEDGER_OUTPUT_FILES}
-        COMMENT "Protocol autogen code generation"
+        setup_code_gen
+        COMMAND ${Python3_EXECUTABLE} -m venv "${CODEGEN_VENV_DIR}"
+        COMMAND ${CODEGEN_PYTHON} -m pip install -r "${REQUIREMENTS_FILE}"
+        WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+        COMMENT "Creating venv and installing code generation dependencies..."
     )
+else()
+    add_custom_target(
+        setup_code_gen
+        COMMAND ${Python3_EXECUTABLE} -m pip install -r "${REQUIREMENTS_FILE}"
+        WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+        COMMENT "Installing code generation dependencies..."
+    )
+endif()
 
-    # Extract test files from output lists (files ending in Tests.cpp)
-    set(PROTOCOL_AUTOGEN_TEST_SOURCES "")
-    foreach(FILE ${TX_OUTPUT_FILES} ${LEDGER_OUTPUT_FILES})
-        if(FILE MATCHES "Tests\\.cpp$")
-            list(APPEND PROTOCOL_AUTOGEN_TEST_SOURCES "${FILE}")
-        endif()
-    endforeach()
-    # Export test sources to parent scope for use in test CMakeLists.txt
-    set(PROTOCOL_AUTOGEN_TEST_SOURCES
-        "${PROTOCOL_AUTOGEN_TEST_SOURCES}"
-        CACHE INTERNAL
-        "Generated protocol_autogen test sources"
-    )
-
-    # Register dependencies so CMake reconfigures when macro files change
-    # (to update the list of output files)
-    set_property(
-        DIRECTORY
-        APPEND
-        PROPERTY
-            CMAKE_CONFIGURE_DEPENDS
-                "${TRANSACTIONS_MACRO}"
-                "${LEDGER_ENTRIES_MACRO}"
-    )
-endfunction()
+# Custom target for code generation, excluded from ALL.
+# Run manually with: cmake --build . --target code_gen
+add_custom_target(
+    code_gen
+    COMMAND
+        ${CMAKE_COMMAND} -DCODEGEN_PYTHON=${CODEGEN_PYTHON}
+        -DGENERATE_TX_SCRIPT=${GENERATE_TX_SCRIPT}
+        -DGENERATE_LEDGER_SCRIPT=${GENERATE_LEDGER_SCRIPT}
+        -DTRANSACTIONS_MACRO=${TRANSACTIONS_MACRO}
+        -DLEDGER_ENTRIES_MACRO=${LEDGER_ENTRIES_MACRO}
+        -DSFIELDS_MACRO=${SFIELDS_MACRO}
+        -DAUTOGEN_HEADER_DIR=${AUTOGEN_HEADER_DIR}
+        -DAUTOGEN_TEST_DIR=${AUTOGEN_TEST_DIR} -P
+        "${CMAKE_CURRENT_SOURCE_DIR}/cmake/XrplProtocolAutogenRun.cmake"
+    WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+    COMMENT "Running protocol code generation..."
+    SOURCES ${ALL_INPUT_FILES}
+)

cmake/XrplProtocolAutogenRun.cmake

@@ -0,0 +1,39 @@
+#[===================================================================[
+   Protocol Autogen - Run script invoked by the 'code_gen' target
+#]===================================================================]
+
+# Generate transaction classes.
+execute_process(
+    COMMAND
+        ${CODEGEN_PYTHON} "${GENERATE_TX_SCRIPT}" "${TRANSACTIONS_MACRO}"
+        --header-dir "${AUTOGEN_HEADER_DIR}/transactions" --test-dir
+        "${AUTOGEN_TEST_DIR}/transactions" --sfields-macro "${SFIELDS_MACRO}"
+    RESULT_VARIABLE TX_RESULT
+    OUTPUT_VARIABLE TX_OUTPUT
+    ERROR_VARIABLE TX_ERROR
+)
+if(NOT TX_RESULT EQUAL 0)
+    message(
+        FATAL_ERROR
+        "Transaction code generation failed:\n${TX_OUTPUT}\n${TX_ERROR}\n${TX_RESULT}"
+    )
+endif()
+
+# Generate ledger entry classes.
+execute_process(
+    COMMAND
+        ${CODEGEN_PYTHON} "${GENERATE_LEDGER_SCRIPT}" "${LEDGER_ENTRIES_MACRO}"
+        --header-dir "${AUTOGEN_HEADER_DIR}/ledger_entries" --test-dir
+        "${AUTOGEN_TEST_DIR}/ledger_entries" --sfields-macro "${SFIELDS_MACRO}"
+    RESULT_VARIABLE LEDGER_RESULT
+    OUTPUT_VARIABLE LEDGER_OUTPUT
+    ERROR_VARIABLE LEDGER_ERROR
+)
+if(NOT LEDGER_RESULT EQUAL 0)
+    message(
+        FATAL_ERROR
+        "Ledger entry code generation failed:\n${LEDGER_OUTPUT}\n${LEDGER_ERROR}\n${TX_RESULT}"
+    )
+endif()
+
+message(STATUS "Protocol autogen: code generation complete")

cmake/scripts/codegen/generate_ledger_classes.py

@@ -138,28 +138,12 @@ def main():
         "--sfields-macro",
         help="Path to sfields.macro (default: auto-detect from macro_path)",
     )
-    parser.add_argument(
-        "--list-outputs",
-        action="store_true",
-        help="List output files without generating (one per line)",
-    )
-
+    parser.add_argument("--venv-dir", help=argparse.SUPPRESS)
     args = parser.parse_args()
 
     # Parse the macro file to get ledger entry names
     entries = parse_macro_file(args.macro_path)
 
-    # If --list-outputs, just print the output file paths and exit
-    if args.list_outputs:
-        header_dir = Path(args.header_dir)
-        for entry in entries:
-            print(header_dir / f"{entry['name']}.h")
-        if args.test_dir:
-            test_dir = Path(args.test_dir)
-            for entry in entries:
-                print(test_dir / f"{entry['name']}Tests.cpp")
-        return
-
     # Auto-detect sfields.macro path if not provided
     if args.sfields_macro:
         sfields_path = Path(args.sfields_macro)

cmake/scripts/codegen/generate_tx_classes.py

@@ -147,28 +147,12 @@ def main():
         "--sfields-macro",
         help="Path to sfields.macro (default: auto-detect from macro_path)",
     )
-    parser.add_argument(
-        "--list-outputs",
-        action="store_true",
-        help="List output files without generating (one per line)",
-    )
-
+    parser.add_argument("--venv-dir", help=argparse.SUPPRESS)
     args = parser.parse_args()
 
     # Parse the macro file to get transaction names
     transactions = parse_macro_file(args.macro_path)
 
-    # If --list-outputs, just print the output file paths and exit
-    if args.list_outputs:
-        header_dir = Path(args.header_dir)
-        for tx in transactions:
-            print(header_dir / f"{tx['name']}.h")
-        if args.test_dir:
-            test_dir = Path(args.test_dir)
-            for tx in transactions:
-                print(test_dir / f"{tx['name']}Tests.cpp")
-        return
-
     # Auto-detect sfields.macro path if not provided
     if args.sfields_macro:
         sfields_path = Path(args.sfields_macro)

include/xrpl/protocol_autogen/README.md

@@ -4,42 +4,33 @@ This directory contains auto-generated C++ wrapper classes for XRP Ledger protoc
 
 ## Generated Files
 
-The files in this directory are automatically generated at **CMake configure time** from macro definition files:
+The files in this directory are generated from macro definition files:
 
-- **Transaction classes** (in `transactions/`): Generated from `include/xrpl/protocol/detail/transactions.macro` by `scripts/generate_tx_classes.py`
-- **Ledger entry classes** (in `ledger_entries/`): Generated from `include/xrpl/protocol/detail/ledger_entries.macro` by `scripts/generate_ledger_classes.py`
+- **Transaction classes** (in `transactions/`): Generated from `include/xrpl/protocol/detail/transactions.macro` by `cmake/scripts/codegen/generate_tx_classes.py`
+- **Ledger entry classes** (in `ledger_entries/`): Generated from `include/xrpl/protocol/detail/ledger_entries.macro` by `cmake/scripts/codegen/generate_ledger_classes.py`
 
 ## Generation Process
 
-The generation happens automatically when you **configure** the project (not during build). When you run CMake, the system:
+Generation requires a one-time setup step to create a virtual environment
+and install Python dependencies, followed by running the generation target:
 
-1. Creates a Python virtual environment in the build directory (`codegen_venv`)
-2. Installs Python dependencies from `scripts/requirements.txt` into the venv (only if needed)
-3. Runs the Python generation scripts using the venv Python interpreter
-4. Parses the macro files to extract type definitions
-5. Generates type-safe C++ wrapper classes using Mako templates
-6. Places the generated headers in this directory
+```bash
+cmake --build . --target setup_code_gen  # create venv and install dependencies (once)
+cmake --build . --target code_gen        # generate code
+```
 
-### When Regeneration Happens
-
-The code is regenerated when:
-
-- You run CMake configure for the first time
-- The Python virtual environment doesn't exist
-- `scripts/requirements.txt` has been modified
-
-To force regeneration, delete the build directory and reconfigure.
+By default, `CODEGEN_VENV_DIR` points to `.venv` in the project root. The
+`setup_code_gen` target creates a venv there and installs the required packages.
+The `code_gen` target then uses the venv's Python interpreter to run generation.
 
 ### Python Dependencies
 
-The code generation requires the following Python packages (automatically installed):
+The code generation requires the following Python packages (installed by `setup_code_gen`):
 
 - `pcpp` - C preprocessor for Python
 - `pyparsing` - Parser combinator library
 - `Mako` - Template engine
 
-These are isolated in a virtual environment and won't affect your system Python installation.
-
 ## Version Control
 
 The generated `.h` files **are checked into version control**. This means:
@@ -50,15 +41,15 @@ The generated `.h` files **are checked into version control**. This means:
 
 ## Modifying Generated Code
 
-**Do not manually edit generated files.** Any changes will be overwritten the next time CMake configure runs.
+**Do not manually edit generated files.** Any changes will be overwritten the next time `code_gen` is run.
 
 To modify the generated classes:
 
 - Edit the macro files in `include/xrpl/protocol/detail/`
-- Edit the Mako templates in `scripts/templates/`
-- Edit the generation scripts in `scripts/`
-- Update Python dependencies in `scripts/requirements.txt`
-- Run CMake configure to regenerate
+- Edit the Mako templates in `cmake/scripts/codegen/templates/`
+- Edit the generation scripts in `cmake/scripts/codegen/`
+- Update Python dependencies in `cmake/scripts/codegen/requirements.txt`
+- Run `cmake --build . --target code_gen` to regenerate
 
 ## Adding Common Fields
 
@@ -73,7 +64,7 @@ Base classes:
 
 Templates (update to pass required common fields to base class constructors):
 
-- `scripts/templates/Transaction.h.mako`
-- `scripts/templates/LedgerEntry.h.mako`
+- `cmake/scripts/codegen/templates/Transaction.h.mako`
+- `cmake/scripts/codegen/templates/LedgerEntry.h.mako`
 
 These files are **not auto-generated** and must be updated by hand.

src/tests/libxrpl/CMakeLists.txt

@@ -32,20 +32,9 @@ xrpl_add_test(json)
 target_link_libraries(xrpl.test.json PRIVATE xrpl.imports.test)
 add_dependencies(xrpl.tests xrpl.test.json)
 
-# protocol_autogen tests use explicit source list (not GLOB) because sources are generated
-# Mark generated sources so CMake knows they'll be created at build time
-set_source_files_properties(
-    ${PROTOCOL_AUTOGEN_TEST_SOURCES}
-    PROPERTIES GENERATED TRUE
-)
-add_executable(xrpl.test.protocol_autogen ${PROTOCOL_AUTOGEN_TEST_SOURCES})
+xrpl_add_test(protocol_autogen)
 target_link_libraries(xrpl.test.protocol_autogen PRIVATE xrpl.imports.test)
 add_dependencies(xrpl.tests xrpl.test.protocol_autogen)
-add_test(NAME xrpl.test.protocol_autogen COMMAND xrpl.test.protocol_autogen)
-# Ensure code generation runs before compiling tests
-if(TARGET protocol_autogen_generate)
-    add_dependencies(xrpl.test.protocol_autogen protocol_autogen_generate)
-endif()
 
 # Network unit tests are currently not supported on Windows
 if(NOT WIN32)