From d52d735543223a6993173f7d7fbd9beb6eb8cbed Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Wed, 15 Apr 2026 19:50:49 +0100 Subject: [PATCH] chore: Move codegen venv setup into build stage (#6617) Co-authored-by: JCW Co-authored-by: Bart --- .../workflows/reusable-build-test-config.yml | 49 ++- .gitignore | 1 + BUILD.md | 15 + CMakeLists.txt | 1 + cmake/XrplCore.cmake | 12 - cmake/XrplProtocolAutogen.cmake | 405 ++++++------------ cmake/XrplProtocolAutogenRun.cmake | 39 ++ .../codegen}/generate_ledger_classes.py | 18 +- .../scripts/codegen}/generate_tx_classes.py | 18 +- .../scripts/codegen}/macro_parser_common.py | 0 .../scripts/codegen}/requirements.txt | 0 .../codegen}/templates/LedgerEntry.h.mako | 0 .../templates/LedgerEntryTests.cpp.mako | 0 .../codegen}/templates/Transaction.h.mako | 0 .../templates/TransactionTests.cpp.mako | 0 include/xrpl/protocol_autogen/README.md | 49 +-- src/tests/libxrpl/CMakeLists.txt | 13 +- 17 files changed, 226 insertions(+), 394 deletions(-) create mode 100644 cmake/XrplProtocolAutogenRun.cmake rename {scripts => cmake/scripts/codegen}/generate_ledger_classes.py (92%) rename {scripts => cmake/scripts/codegen}/generate_tx_classes.py (92%) rename {scripts => cmake/scripts/codegen}/macro_parser_common.py (100%) rename {scripts => cmake/scripts/codegen}/requirements.txt (100%) rename {scripts => cmake/scripts/codegen}/templates/LedgerEntry.h.mako (100%) rename {scripts => cmake/scripts/codegen}/templates/LedgerEntryTests.cpp.mako (100%) rename {scripts => cmake/scripts/codegen}/templates/Transaction.h.mako (100%) rename {scripts => cmake/scripts/codegen}/templates/TransactionTests.cpp.mako (100%) diff --git a/.github/workflows/reusable-build-test-config.yml b/.github/workflows/reusable-build-test-config.yml index 57112ed96a..8053c64f8b 100644 --- a/.github/workflows/reusable-build-test-config.yml +++ b/.github/workflows/reusable-build-test-config.yml @@ -153,6 +153,32 @@ jobs: ${CMAKE_ARGS} \ .. + - name: Check protocol autogen files are up-to-date + working-directory: ${{ env.BUILD_DIR }} + env: + MESSAGE: | + + The generated protocol wrapper classes are out of date. + + This typically happens when the macro files or generator scripts + have changed but the generated files were not regenerated. + + To fix this: + 1. Run: cmake --build . --target setup_code_gen + 2. Run: cmake --build . --target code_gen + 3. Commit and push the regenerated files + run: | + set -e + cmake --build . --target setup_code_gen + cmake --build . --target code_gen + DIFF=$(git -C .. status --porcelain -- include/xrpl/protocol_autogen src/tests/libxrpl/protocol_autogen) + if [ -n "${DIFF}" ]; then + echo "::error::Generated protocol files are out of date" + git -C .. diff -- include/xrpl/protocol_autogen src/tests/libxrpl/protocol_autogen + echo "${MESSAGE}" + exit 1 + fi + - name: Build the binary working-directory: ${{ env.BUILD_DIR }} env: @@ -166,29 +192,6 @@ jobs: --parallel "${BUILD_NPROC}" \ --target "${CMAKE_TARGET}" - - name: Check protocol autogen files are up-to-date - env: - MESSAGE: | - - The generated protocol wrapper classes are out of date. - - This typically happens when your branch is behind develop and - the macro files or generator scripts have changed. - - To fix this: - 1. Update your branch from develop (merge or rebase) - 2. Build with code generation enabled (XRPL_NO_CODEGEN=OFF) - 3. Commit and push the regenerated files - run: | - set -e - DIFF=$(git status --porcelain -- include/xrpl/protocol_autogen src/tests/libxrpl/protocol_autogen) - if [ -n "${DIFF}" ]; then - echo "::error::Generated protocol files are out of date" - git diff -- include/xrpl/protocol_autogen src/tests/libxrpl/protocol_autogen - echo "${MESSAGE}" - exit 1 - fi - - name: Show ccache statistics if: ${{ inputs.ccache_enabled }} run: | diff --git a/.gitignore b/.gitignore index 7ee6d0c70a..6bd34ece04 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ Debug/ Release/ /.build/ +/.venv/ /build/ /db/ /out.txt diff --git a/BUILD.md b/BUILD.md index 7406d60949..cf0a685abd 100644 --- a/BUILD.md +++ b/BUILD.md @@ -459,6 +459,21 @@ install ccache --version 4.11.3 --allow-downgrade`. The location of `xrpld` binary in your build directory depends on your CMake generator. Pass `--help` to see the rest of the command line options. +## Code generation + +The protocol wrapper classes in `include/xrpl/protocol_autogen/` are generated +from macro definition files in `include/xrpl/protocol/detail/`. If you modify +the macro files (e.g. `transactions.macro`, `ledger_entries.macro`) or the +generation scripts/templates in `cmake/scripts/codegen/`, you need to regenerate the +files: + +``` +cmake --build . --target setup_code_gen # create venv and install dependencies (once) +cmake --build . --target code_gen # regenerate code +``` + +The regenerated files should be committed alongside your changes. + ## Coverage report The coverage report is intended for developers using compilers GCC diff --git a/CMakeLists.txt b/CMakeLists.txt index 33f68451c5..80ff8fec13 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -132,6 +132,7 @@ if(coverage) endif() include(XrplCore) +include(XrplProtocolAutogen) include(XrplInstall) include(XrplValidatorKeys) diff --git a/cmake/XrplCore.cmake b/cmake/XrplCore.cmake index a50e30f660..9b1dc74049 100644 --- a/cmake/XrplCore.cmake +++ b/cmake/XrplCore.cmake @@ -108,24 +108,12 @@ target_link_libraries( ) # Level 05 -## Set up code generation for protocol_autogen module -include(XrplProtocolAutogen) -# Must call setup_protocol_autogen before add_module so that: -# 1. Stale generated files are cleared before GLOB runs -# 2. Output file list is known for custom commands -setup_protocol_autogen() - add_module(xrpl protocol_autogen) target_link_libraries( xrpl.libxrpl.protocol_autogen PUBLIC xrpl.libxrpl.protocol ) -# Ensure code generation runs before compiling protocol_autogen -if(TARGET protocol_autogen_generate) - add_dependencies(xrpl.libxrpl.protocol_autogen protocol_autogen_generate) -endif() - # Level 06 add_module(xrpl core) target_link_libraries( diff --git a/cmake/XrplProtocolAutogen.cmake b/cmake/XrplProtocolAutogen.cmake index e48d28959d..dd9ef6a9a4 100644 --- a/cmake/XrplProtocolAutogen.cmake +++ b/cmake/XrplProtocolAutogen.cmake @@ -2,308 +2,145 @@ Protocol Autogen - Code generation for protocol wrapper classes #]===================================================================] -# Options for code generation -option( - XRPL_NO_CODEGEN - "Disable code generation (use pre-generated files from repository)" - OFF -) set(CODEGEN_VENV_DIR - "" + "${CMAKE_CURRENT_SOURCE_DIR}/.venv" CACHE PATH - "Path to Python virtual environment for code generation. If provided, automatic venv setup is skipped." + "Path to a Python virtual environment for code generation. A venv will be created here by setup_code_gen and used to run generation scripts." ) -# Function to set up code generation for protocol_autogen module -# This runs at configure time to generate C++ wrapper classes from macro files -function(setup_protocol_autogen) - # Directory paths - set(MACRO_DIR "${CMAKE_CURRENT_SOURCE_DIR}/include/xrpl/protocol/detail") - set(AUTOGEN_HEADER_DIR - "${CMAKE_CURRENT_SOURCE_DIR}/include/xrpl/protocol_autogen" - ) - set(AUTOGEN_TEST_DIR - "${CMAKE_CURRENT_SOURCE_DIR}/src/tests/libxrpl/protocol_autogen" - ) - set(SCRIPTS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/scripts") +# Directory paths +set(MACRO_DIR "${CMAKE_CURRENT_SOURCE_DIR}/include/xrpl/protocol/detail") +set(AUTOGEN_HEADER_DIR + "${CMAKE_CURRENT_SOURCE_DIR}/include/xrpl/protocol_autogen" +) +set(AUTOGEN_TEST_DIR + "${CMAKE_CURRENT_SOURCE_DIR}/src/tests/libxrpl/protocol_autogen" +) +set(SCRIPTS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/cmake/scripts/codegen") - # Input macro files - set(TRANSACTIONS_MACRO "${MACRO_DIR}/transactions.macro") - set(LEDGER_ENTRIES_MACRO "${MACRO_DIR}/ledger_entries.macro") - set(SFIELDS_MACRO "${MACRO_DIR}/sfields.macro") +# Input macro files +set(TRANSACTIONS_MACRO "${MACRO_DIR}/transactions.macro") +set(LEDGER_ENTRIES_MACRO "${MACRO_DIR}/ledger_entries.macro") +set(SFIELDS_MACRO "${MACRO_DIR}/sfields.macro") - # Python scripts and templates - set(GENERATE_TX_SCRIPT "${SCRIPTS_DIR}/generate_tx_classes.py") - set(GENERATE_LEDGER_SCRIPT "${SCRIPTS_DIR}/generate_ledger_classes.py") - set(REQUIREMENTS_FILE "${SCRIPTS_DIR}/requirements.txt") - set(MACRO_PARSER_COMMON "${SCRIPTS_DIR}/macro_parser_common.py") - set(TX_TEMPLATE "${SCRIPTS_DIR}/templates/Transaction.h.mako") - set(TX_TEST_TEMPLATE "${SCRIPTS_DIR}/templates/TransactionTests.cpp.mako") - set(LEDGER_TEMPLATE "${SCRIPTS_DIR}/templates/LedgerEntry.h.mako") - set(LEDGER_TEST_TEMPLATE - "${SCRIPTS_DIR}/templates/LedgerEntryTests.cpp.mako" +# Python scripts and templates +set(GENERATE_TX_SCRIPT "${SCRIPTS_DIR}/generate_tx_classes.py") +set(GENERATE_LEDGER_SCRIPT "${SCRIPTS_DIR}/generate_ledger_classes.py") +set(REQUIREMENTS_FILE "${SCRIPTS_DIR}/requirements.txt") +set(MACRO_PARSER_COMMON "${SCRIPTS_DIR}/macro_parser_common.py") +set(TX_TEMPLATE "${SCRIPTS_DIR}/templates/Transaction.h.mako") +set(TX_TEST_TEMPLATE "${SCRIPTS_DIR}/templates/TransactionTests.cpp.mako") +set(LEDGER_TEMPLATE "${SCRIPTS_DIR}/templates/LedgerEntry.h.mako") +set(LEDGER_TEST_TEMPLATE "${SCRIPTS_DIR}/templates/LedgerEntryTests.cpp.mako") +set(ALL_INPUT_FILES + "${TRANSACTIONS_MACRO}" + "${LEDGER_ENTRIES_MACRO}" + "${SFIELDS_MACRO}" + "${GENERATE_TX_SCRIPT}" + "${GENERATE_LEDGER_SCRIPT}" + "${REQUIREMENTS_FILE}" + "${MACRO_PARSER_COMMON}" + "${TX_TEMPLATE}" + "${TX_TEST_TEMPLATE}" + "${LEDGER_TEMPLATE}" + "${LEDGER_TEST_TEMPLATE}" +) + +# Create output directories +file(MAKE_DIRECTORY "${AUTOGEN_HEADER_DIR}/transactions") +file(MAKE_DIRECTORY "${AUTOGEN_HEADER_DIR}/ledger_entries") +file(MAKE_DIRECTORY "${AUTOGEN_TEST_DIR}/ledger_entries") +file(MAKE_DIRECTORY "${AUTOGEN_TEST_DIR}/transactions") + +# Find Python3 +if(NOT Python3_EXECUTABLE) + find_package(Python3 COMPONENTS Interpreter QUIET) +endif() + +if(NOT Python3_EXECUTABLE) + find_program(Python3_EXECUTABLE NAMES python3 python) +endif() + +if(NOT Python3_EXECUTABLE) + message( + WARNING + "Python3 not found. The 'code_gen' and 'setup_code_gen' targets will not be available." ) + return() +endif() - # Check if code generation is disabled - if(XRPL_NO_CODEGEN) +# Warn if pip is configured with a non-default index (may need VPN). +execute_process( + COMMAND ${Python3_EXECUTABLE} -m pip config get global.index-url + OUTPUT_VARIABLE PIP_INDEX_URL + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET + RESULT_VARIABLE PIP_CONFIG_RESULT +) +if(PIP_CONFIG_RESULT EQUAL 0 AND PIP_INDEX_URL) + if( + NOT PIP_INDEX_URL STREQUAL "https://pypi.org/simple" + AND NOT PIP_INDEX_URL STREQUAL "https://pypi.python.org/simple" + ) message( WARNING - "Protocol autogen: Code generation is disabled (XRPL_NO_CODEGEN=ON). " - "Generated files may be out of date." + "Private pip index URL detected: ${PIP_INDEX_URL}\n" + "You may need to connect to VPN to access this URL." ) - return() endif() +endif() - # Create output directories - file(MAKE_DIRECTORY "${AUTOGEN_HEADER_DIR}/transactions") - file(MAKE_DIRECTORY "${AUTOGEN_HEADER_DIR}/ledger_entries") - file(MAKE_DIRECTORY "${AUTOGEN_TEST_DIR}/ledger_entries") - file(MAKE_DIRECTORY "${AUTOGEN_TEST_DIR}/transactions") - - # Find Python3 - check if already found by Conan or find it ourselves - if(NOT Python3_EXECUTABLE) - find_package(Python3 COMPONENTS Interpreter QUIET) - endif() - - if(NOT Python3_EXECUTABLE) - # Try finding python3 executable directly - find_program(Python3_EXECUTABLE NAMES python3 python) - endif() - - if(NOT Python3_EXECUTABLE) - message( - FATAL_ERROR - "Python3 not found. Code generation cannot proceed.\n" - "Please install Python 3, or set -DXRPL_NO_CODEGEN=ON to use existing generated files." - ) - return() - endif() - - message(STATUS "Using Python3 for code generation: ${Python3_EXECUTABLE}") - - # Set up Python virtual environment for code generation - if(CODEGEN_VENV_DIR) - # User-provided venv - skip automatic setup - set(VENV_DIR "${CODEGEN_VENV_DIR}") - message(STATUS "Using user-provided Python venv: ${VENV_DIR}") - else() - # Use default venv in build directory - set(VENV_DIR "${CMAKE_CURRENT_BINARY_DIR}/codegen_venv") - endif() - - # Determine the Python executable path in the venv +# Determine which Python interpreter to use for code generation. +if(CODEGEN_VENV_DIR) if(WIN32) - set(VENV_PYTHON "${VENV_DIR}/Scripts/python.exe") - set(VENV_PIP "${VENV_DIR}/Scripts/pip.exe") + set(CODEGEN_PYTHON "${CODEGEN_VENV_DIR}/Scripts/python.exe") else() - set(VENV_PYTHON "${VENV_DIR}/bin/python") - set(VENV_PIP "${VENV_DIR}/bin/pip") + set(CODEGEN_PYTHON "${CODEGEN_VENV_DIR}/bin/python") endif() - - # Only auto-setup venv if not user-provided - if(NOT CODEGEN_VENV_DIR) - # Check if venv needs to be created or updated - set(VENV_NEEDS_UPDATE FALSE) - if(NOT EXISTS "${VENV_PYTHON}") - set(VENV_NEEDS_UPDATE TRUE) - message( - STATUS - "Creating Python virtual environment for code generation..." - ) - elseif( - "${REQUIREMENTS_FILE}" - IS_NEWER_THAN - "${VENV_DIR}/.requirements_installed" - ) - set(VENV_NEEDS_UPDATE TRUE) - message( - STATUS - "Updating Python virtual environment (requirements changed)..." - ) - endif() - - # Create/update virtual environment if needed - if(VENV_NEEDS_UPDATE) - message( - STATUS - "Setting up Python virtual environment at ${VENV_DIR}" - ) - execute_process( - COMMAND ${Python3_EXECUTABLE} -m venv "${VENV_DIR}" - RESULT_VARIABLE VENV_RESULT - ERROR_VARIABLE VENV_ERROR - ) - if(NOT VENV_RESULT EQUAL 0) - message( - FATAL_ERROR - "Failed to create virtual environment: ${VENV_ERROR}" - ) - endif() - - # Check pip index URL configuration - execute_process( - COMMAND ${VENV_PIP} config get global.index-url - OUTPUT_VARIABLE PIP_INDEX_URL - OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_QUIET - ) - - # Default PyPI URL - set(DEFAULT_PIP_INDEX "https://pypi.org/simple") - - # Show warning if using non-default index - if(PIP_INDEX_URL AND NOT PIP_INDEX_URL STREQUAL "") - if(NOT PIP_INDEX_URL STREQUAL DEFAULT_PIP_INDEX) - message( - WARNING - "Private pip index URL detected: ${PIP_INDEX_URL}\n" - "You may need to connect to VPN to access this URL." - ) - endif() - endif() - - message(STATUS "Installing Python dependencies...") - execute_process( - COMMAND ${VENV_PIP} install --upgrade pip - RESULT_VARIABLE PIP_UPGRADE_RESULT - OUTPUT_QUIET - ERROR_VARIABLE PIP_UPGRADE_ERROR - ) - if(NOT PIP_UPGRADE_RESULT EQUAL 0) - message(WARNING "Failed to upgrade pip: ${PIP_UPGRADE_ERROR}") - endif() - - execute_process( - COMMAND ${VENV_PIP} install -r "${REQUIREMENTS_FILE}" - RESULT_VARIABLE PIP_INSTALL_RESULT - ERROR_VARIABLE PIP_INSTALL_ERROR - ) - if(NOT PIP_INSTALL_RESULT EQUAL 0) - message( - FATAL_ERROR - "Failed to install Python dependencies: ${PIP_INSTALL_ERROR}" - ) - endif() - - # Mark requirements as installed - file(TOUCH "${VENV_DIR}/.requirements_installed") - message(STATUS "Python virtual environment ready") - endif() - endif() - - # At configure time - get list of output files for transactions - execute_process( - COMMAND - ${VENV_PYTHON} "${GENERATE_TX_SCRIPT}" "${TRANSACTIONS_MACRO}" - --header-dir "${AUTOGEN_HEADER_DIR}/transactions" --test-dir - "${AUTOGEN_TEST_DIR}/transactions" --list-outputs - OUTPUT_VARIABLE TX_OUTPUT_FILES - OUTPUT_STRIP_TRAILING_WHITESPACE - RESULT_VARIABLE TX_LIST_RESULT - ERROR_VARIABLE TX_LIST_ERROR +else() + set(CODEGEN_PYTHON "${Python3_EXECUTABLE}") + message( + WARNING + "CODEGEN_VENV_DIR is not set. Dependencies will be installed globally.\n" + "If this is not intended, reconfigure with:\n" + " cmake . -UCODEGEN_VENV_DIR" ) - if(NOT TX_LIST_RESULT EQUAL 0) - message( - FATAL_ERROR - "Failed to list transaction output files:\n${TX_LIST_ERROR}" - ) - endif() - # Convert newline-separated list to CMake list - string(REPLACE "\\" "/" TX_OUTPUT_FILES "${TX_OUTPUT_FILES}") - string(REPLACE "\n" ";" TX_OUTPUT_FILES "${TX_OUTPUT_FILES}") +endif() - # At configure time - get list of output files for ledger entries - execute_process( - COMMAND - ${VENV_PYTHON} "${GENERATE_LEDGER_SCRIPT}" "${LEDGER_ENTRIES_MACRO}" - --header-dir "${AUTOGEN_HEADER_DIR}/ledger_entries" --test-dir - "${AUTOGEN_TEST_DIR}/ledger_entries" --list-outputs - OUTPUT_VARIABLE LEDGER_OUTPUT_FILES - OUTPUT_STRIP_TRAILING_WHITESPACE - RESULT_VARIABLE LEDGER_LIST_RESULT - ERROR_VARIABLE LEDGER_LIST_ERROR - ) - if(NOT LEDGER_LIST_RESULT EQUAL 0) - message( - FATAL_ERROR - "Failed to list ledger entry output files:\n${LEDGER_LIST_ERROR}" - ) - endif() - # Convert newline-separated list to CMake list - string(REPLACE "\\" "/" LEDGER_OUTPUT_FILES "${LEDGER_OUTPUT_FILES}") - string(REPLACE "\n" ";" LEDGER_OUTPUT_FILES "${LEDGER_OUTPUT_FILES}") - - # Custom command to generate transaction classes at build time - add_custom_command( - OUTPUT ${TX_OUTPUT_FILES} - COMMAND - ${VENV_PYTHON} "${GENERATE_TX_SCRIPT}" "${TRANSACTIONS_MACRO}" - --header-dir "${AUTOGEN_HEADER_DIR}/transactions" --test-dir - "${AUTOGEN_TEST_DIR}/transactions" --sfields-macro - "${SFIELDS_MACRO}" - WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" - DEPENDS - "${TRANSACTIONS_MACRO}" - "${SFIELDS_MACRO}" - "${GENERATE_TX_SCRIPT}" - "${MACRO_PARSER_COMMON}" - "${TX_TEMPLATE}" - "${TX_TEST_TEMPLATE}" - "${REQUIREMENTS_FILE}" - COMMENT "Generating transaction classes from transactions.macro..." - VERBATIM - ) - - # Custom command to generate ledger entry classes at build time - add_custom_command( - OUTPUT ${LEDGER_OUTPUT_FILES} - COMMAND - ${VENV_PYTHON} "${GENERATE_LEDGER_SCRIPT}" "${LEDGER_ENTRIES_MACRO}" - --header-dir "${AUTOGEN_HEADER_DIR}/ledger_entries" --test-dir - "${AUTOGEN_TEST_DIR}/ledger_entries" --sfields-macro - "${SFIELDS_MACRO}" - WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" - DEPENDS - "${LEDGER_ENTRIES_MACRO}" - "${SFIELDS_MACRO}" - "${GENERATE_LEDGER_SCRIPT}" - "${MACRO_PARSER_COMMON}" - "${LEDGER_TEMPLATE}" - "${LEDGER_TEST_TEMPLATE}" - "${REQUIREMENTS_FILE}" - COMMENT "Generating ledger entry classes from ledger_entries.macro..." - VERBATIM - ) - - # Create a custom target that depends on all generated files +# Custom target to create a venv and install Python dependencies. +# Run manually with: cmake --build . --target setup_code_gen +if(CODEGEN_VENV_DIR) add_custom_target( - protocol_autogen_generate - DEPENDS ${TX_OUTPUT_FILES} ${LEDGER_OUTPUT_FILES} - COMMENT "Protocol autogen code generation" + setup_code_gen + COMMAND ${Python3_EXECUTABLE} -m venv "${CODEGEN_VENV_DIR}" + COMMAND ${CODEGEN_PYTHON} -m pip install -r "${REQUIREMENTS_FILE}" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + COMMENT "Creating venv and installing code generation dependencies..." ) +else() + add_custom_target( + setup_code_gen + COMMAND ${Python3_EXECUTABLE} -m pip install -r "${REQUIREMENTS_FILE}" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + COMMENT "Installing code generation dependencies..." + ) +endif() - # Extract test files from output lists (files ending in Tests.cpp) - set(PROTOCOL_AUTOGEN_TEST_SOURCES "") - foreach(FILE ${TX_OUTPUT_FILES} ${LEDGER_OUTPUT_FILES}) - if(FILE MATCHES "Tests\\.cpp$") - list(APPEND PROTOCOL_AUTOGEN_TEST_SOURCES "${FILE}") - endif() - endforeach() - # Export test sources to parent scope for use in test CMakeLists.txt - set(PROTOCOL_AUTOGEN_TEST_SOURCES - "${PROTOCOL_AUTOGEN_TEST_SOURCES}" - CACHE INTERNAL - "Generated protocol_autogen test sources" - ) - - # Register dependencies so CMake reconfigures when macro files change - # (to update the list of output files) - set_property( - DIRECTORY - APPEND - PROPERTY - CMAKE_CONFIGURE_DEPENDS - "${TRANSACTIONS_MACRO}" - "${LEDGER_ENTRIES_MACRO}" - ) -endfunction() +# Custom target for code generation, excluded from ALL. +# Run manually with: cmake --build . --target code_gen +add_custom_target( + code_gen + COMMAND + ${CMAKE_COMMAND} -DCODEGEN_PYTHON=${CODEGEN_PYTHON} + -DGENERATE_TX_SCRIPT=${GENERATE_TX_SCRIPT} + -DGENERATE_LEDGER_SCRIPT=${GENERATE_LEDGER_SCRIPT} + -DTRANSACTIONS_MACRO=${TRANSACTIONS_MACRO} + -DLEDGER_ENTRIES_MACRO=${LEDGER_ENTRIES_MACRO} + -DSFIELDS_MACRO=${SFIELDS_MACRO} + -DAUTOGEN_HEADER_DIR=${AUTOGEN_HEADER_DIR} + -DAUTOGEN_TEST_DIR=${AUTOGEN_TEST_DIR} -P + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/XrplProtocolAutogenRun.cmake" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + COMMENT "Running protocol code generation..." + SOURCES ${ALL_INPUT_FILES} +) diff --git a/cmake/XrplProtocolAutogenRun.cmake b/cmake/XrplProtocolAutogenRun.cmake new file mode 100644 index 0000000000..8bdb37a8e0 --- /dev/null +++ b/cmake/XrplProtocolAutogenRun.cmake @@ -0,0 +1,39 @@ +#[===================================================================[ + Protocol Autogen - Run script invoked by the 'code_gen' target +#]===================================================================] + +# Generate transaction classes. +execute_process( + COMMAND + ${CODEGEN_PYTHON} "${GENERATE_TX_SCRIPT}" "${TRANSACTIONS_MACRO}" + --header-dir "${AUTOGEN_HEADER_DIR}/transactions" --test-dir + "${AUTOGEN_TEST_DIR}/transactions" --sfields-macro "${SFIELDS_MACRO}" + RESULT_VARIABLE TX_RESULT + OUTPUT_VARIABLE TX_OUTPUT + ERROR_VARIABLE TX_ERROR +) +if(NOT TX_RESULT EQUAL 0) + message( + FATAL_ERROR + "Transaction code generation failed:\n${TX_OUTPUT}\n${TX_ERROR}\n${TX_RESULT}" + ) +endif() + +# Generate ledger entry classes. +execute_process( + COMMAND + ${CODEGEN_PYTHON} "${GENERATE_LEDGER_SCRIPT}" "${LEDGER_ENTRIES_MACRO}" + --header-dir "${AUTOGEN_HEADER_DIR}/ledger_entries" --test-dir + "${AUTOGEN_TEST_DIR}/ledger_entries" --sfields-macro "${SFIELDS_MACRO}" + RESULT_VARIABLE LEDGER_RESULT + OUTPUT_VARIABLE LEDGER_OUTPUT + ERROR_VARIABLE LEDGER_ERROR +) +if(NOT LEDGER_RESULT EQUAL 0) + message( + FATAL_ERROR + "Ledger entry code generation failed:\n${LEDGER_OUTPUT}\n${LEDGER_ERROR}\n${TX_RESULT}" + ) +endif() + +message(STATUS "Protocol autogen: code generation complete") diff --git a/scripts/generate_ledger_classes.py b/cmake/scripts/codegen/generate_ledger_classes.py similarity index 92% rename from scripts/generate_ledger_classes.py rename to cmake/scripts/codegen/generate_ledger_classes.py index ad773ab9af..f5513655de 100644 --- a/scripts/generate_ledger_classes.py +++ b/cmake/scripts/codegen/generate_ledger_classes.py @@ -138,28 +138,12 @@ def main(): "--sfields-macro", help="Path to sfields.macro (default: auto-detect from macro_path)", ) - parser.add_argument( - "--list-outputs", - action="store_true", - help="List output files without generating (one per line)", - ) - + parser.add_argument("--venv-dir", help=argparse.SUPPRESS) args = parser.parse_args() # Parse the macro file to get ledger entry names entries = parse_macro_file(args.macro_path) - # If --list-outputs, just print the output file paths and exit - if args.list_outputs: - header_dir = Path(args.header_dir) - for entry in entries: - print(header_dir / f"{entry['name']}.h") - if args.test_dir: - test_dir = Path(args.test_dir) - for entry in entries: - print(test_dir / f"{entry['name']}Tests.cpp") - return - # Auto-detect sfields.macro path if not provided if args.sfields_macro: sfields_path = Path(args.sfields_macro) diff --git a/scripts/generate_tx_classes.py b/cmake/scripts/codegen/generate_tx_classes.py similarity index 92% rename from scripts/generate_tx_classes.py rename to cmake/scripts/codegen/generate_tx_classes.py index f21b7101df..07baefd8b6 100644 --- a/scripts/generate_tx_classes.py +++ b/cmake/scripts/codegen/generate_tx_classes.py @@ -147,28 +147,12 @@ def main(): "--sfields-macro", help="Path to sfields.macro (default: auto-detect from macro_path)", ) - parser.add_argument( - "--list-outputs", - action="store_true", - help="List output files without generating (one per line)", - ) - + parser.add_argument("--venv-dir", help=argparse.SUPPRESS) args = parser.parse_args() # Parse the macro file to get transaction names transactions = parse_macro_file(args.macro_path) - # If --list-outputs, just print the output file paths and exit - if args.list_outputs: - header_dir = Path(args.header_dir) - for tx in transactions: - print(header_dir / f"{tx['name']}.h") - if args.test_dir: - test_dir = Path(args.test_dir) - for tx in transactions: - print(test_dir / f"{tx['name']}Tests.cpp") - return - # Auto-detect sfields.macro path if not provided if args.sfields_macro: sfields_path = Path(args.sfields_macro) diff --git a/scripts/macro_parser_common.py b/cmake/scripts/codegen/macro_parser_common.py similarity index 100% rename from scripts/macro_parser_common.py rename to cmake/scripts/codegen/macro_parser_common.py diff --git a/scripts/requirements.txt b/cmake/scripts/codegen/requirements.txt similarity index 100% rename from scripts/requirements.txt rename to cmake/scripts/codegen/requirements.txt diff --git a/scripts/templates/LedgerEntry.h.mako b/cmake/scripts/codegen/templates/LedgerEntry.h.mako similarity index 100% rename from scripts/templates/LedgerEntry.h.mako rename to cmake/scripts/codegen/templates/LedgerEntry.h.mako diff --git a/scripts/templates/LedgerEntryTests.cpp.mako b/cmake/scripts/codegen/templates/LedgerEntryTests.cpp.mako similarity index 100% rename from scripts/templates/LedgerEntryTests.cpp.mako rename to cmake/scripts/codegen/templates/LedgerEntryTests.cpp.mako diff --git a/scripts/templates/Transaction.h.mako b/cmake/scripts/codegen/templates/Transaction.h.mako similarity index 100% rename from scripts/templates/Transaction.h.mako rename to cmake/scripts/codegen/templates/Transaction.h.mako diff --git a/scripts/templates/TransactionTests.cpp.mako b/cmake/scripts/codegen/templates/TransactionTests.cpp.mako similarity index 100% rename from scripts/templates/TransactionTests.cpp.mako rename to cmake/scripts/codegen/templates/TransactionTests.cpp.mako diff --git a/include/xrpl/protocol_autogen/README.md b/include/xrpl/protocol_autogen/README.md index be90788b05..860ed1a242 100644 --- a/include/xrpl/protocol_autogen/README.md +++ b/include/xrpl/protocol_autogen/README.md @@ -4,42 +4,33 @@ This directory contains auto-generated C++ wrapper classes for XRP Ledger protoc ## Generated Files -The files in this directory are automatically generated at **CMake configure time** from macro definition files: +The files in this directory are generated from macro definition files: -- **Transaction classes** (in `transactions/`): Generated from `include/xrpl/protocol/detail/transactions.macro` by `scripts/generate_tx_classes.py` -- **Ledger entry classes** (in `ledger_entries/`): Generated from `include/xrpl/protocol/detail/ledger_entries.macro` by `scripts/generate_ledger_classes.py` +- **Transaction classes** (in `transactions/`): Generated from `include/xrpl/protocol/detail/transactions.macro` by `cmake/scripts/codegen/generate_tx_classes.py` +- **Ledger entry classes** (in `ledger_entries/`): Generated from `include/xrpl/protocol/detail/ledger_entries.macro` by `cmake/scripts/codegen/generate_ledger_classes.py` ## Generation Process -The generation happens automatically when you **configure** the project (not during build). When you run CMake, the system: +Generation requires a one-time setup step to create a virtual environment +and install Python dependencies, followed by running the generation target: -1. Creates a Python virtual environment in the build directory (`codegen_venv`) -2. Installs Python dependencies from `scripts/requirements.txt` into the venv (only if needed) -3. Runs the Python generation scripts using the venv Python interpreter -4. Parses the macro files to extract type definitions -5. Generates type-safe C++ wrapper classes using Mako templates -6. Places the generated headers in this directory +```bash +cmake --build . --target setup_code_gen # create venv and install dependencies (once) +cmake --build . --target code_gen # generate code +``` -### When Regeneration Happens - -The code is regenerated when: - -- You run CMake configure for the first time -- The Python virtual environment doesn't exist -- `scripts/requirements.txt` has been modified - -To force regeneration, delete the build directory and reconfigure. +By default, `CODEGEN_VENV_DIR` points to `.venv` in the project root. The +`setup_code_gen` target creates a venv there and installs the required packages. +The `code_gen` target then uses the venv's Python interpreter to run generation. ### Python Dependencies -The code generation requires the following Python packages (automatically installed): +The code generation requires the following Python packages (installed by `setup_code_gen`): - `pcpp` - C preprocessor for Python - `pyparsing` - Parser combinator library - `Mako` - Template engine -These are isolated in a virtual environment and won't affect your system Python installation. - ## Version Control The generated `.h` files **are checked into version control**. This means: @@ -50,15 +41,15 @@ The generated `.h` files **are checked into version control**. This means: ## Modifying Generated Code -**Do not manually edit generated files.** Any changes will be overwritten the next time CMake configure runs. +**Do not manually edit generated files.** Any changes will be overwritten the next time `code_gen` is run. To modify the generated classes: - Edit the macro files in `include/xrpl/protocol/detail/` -- Edit the Mako templates in `scripts/templates/` -- Edit the generation scripts in `scripts/` -- Update Python dependencies in `scripts/requirements.txt` -- Run CMake configure to regenerate +- Edit the Mako templates in `cmake/scripts/codegen/templates/` +- Edit the generation scripts in `cmake/scripts/codegen/` +- Update Python dependencies in `cmake/scripts/codegen/requirements.txt` +- Run `cmake --build . --target code_gen` to regenerate ## Adding Common Fields @@ -73,7 +64,7 @@ Base classes: Templates (update to pass required common fields to base class constructors): -- `scripts/templates/Transaction.h.mako` -- `scripts/templates/LedgerEntry.h.mako` +- `cmake/scripts/codegen/templates/Transaction.h.mako` +- `cmake/scripts/codegen/templates/LedgerEntry.h.mako` These files are **not auto-generated** and must be updated by hand. diff --git a/src/tests/libxrpl/CMakeLists.txt b/src/tests/libxrpl/CMakeLists.txt index a82ed1472f..0b666441d1 100644 --- a/src/tests/libxrpl/CMakeLists.txt +++ b/src/tests/libxrpl/CMakeLists.txt @@ -32,20 +32,9 @@ xrpl_add_test(json) target_link_libraries(xrpl.test.json PRIVATE xrpl.imports.test) add_dependencies(xrpl.tests xrpl.test.json) -# protocol_autogen tests use explicit source list (not GLOB) because sources are generated -# Mark generated sources so CMake knows they'll be created at build time -set_source_files_properties( - ${PROTOCOL_AUTOGEN_TEST_SOURCES} - PROPERTIES GENERATED TRUE -) -add_executable(xrpl.test.protocol_autogen ${PROTOCOL_AUTOGEN_TEST_SOURCES}) +xrpl_add_test(protocol_autogen) target_link_libraries(xrpl.test.protocol_autogen PRIVATE xrpl.imports.test) add_dependencies(xrpl.tests xrpl.test.protocol_autogen) -add_test(NAME xrpl.test.protocol_autogen COMMAND xrpl.test.protocol_autogen) -# Ensure code generation runs before compiling tests -if(TARGET protocol_autogen_generate) - add_dependencies(xrpl.test.protocol_autogen protocol_autogen_generate) -endif() # Network unit tests are currently not supported on Windows if(NOT WIN32)