ARCHIVE/rippled
1
0
Fork 0
mirror of https://github.com/XRPLF/rippled.git synced 2026-04-01 17:42:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor the code generation process

Browse Source
This commit is contained in:
JCW
2026-03-31 16:22:50 +01:00
parent 5c8dfe5456
commit a1344b91c3
14 changed files with 193 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
cmake/XrplCore.cmake
View File

@@ -108,11 +108,10 @@ target_link_libraries(
)
# Level 05
## Set up code generation for protocol_autogen module
## Set up code generation for protocol_autogen module.
## Generation runs at configure time (when the stamp is stale),
## so generated files are always present before add_module GLOBs them.
include(XrplProtocolAutogen)
# Must call setup_protocol_autogen before add_module so that:
# 1. Stale generated files are cleared before GLOB runs
# 2. Output file list is known for custom commands
setup_protocol_autogen()
add_module(xrpl protocol_autogen)
@@ -121,11 +120,6 @@ target_link_libraries(
PUBLIC xrpl.libxrpl.protocol
)
# Ensure code generation runs before compiling protocol_autogen
if(TARGET protocol_autogen_generate)
add_dependencies(xrpl.libxrpl.protocol_autogen protocol_autogen_generate)
endif()
# Level 06
add_module(xrpl core)
target_link_libraries(

203
cmake/XrplProtocolAutogen.cmake
View File

@@ -15,7 +15,6 @@ set(CODEGEN_VENV_DIR
)
# Function to set up code generation for protocol_autogen module
# This runs at configure time to generate C++ wrapper classes from macro files
function(setup_protocol_autogen)
# Directory paths
set(MACRO_DIR "${CMAKE_CURRENT_SOURCE_DIR}/include/xrpl/protocol/detail")
@@ -25,7 +24,7 @@ function(setup_protocol_autogen)
set(AUTOGEN_TEST_DIR
"${CMAKE_CURRENT_SOURCE_DIR}/src/tests/libxrpl/protocol_autogen"
)
set(SCRIPTS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/scripts")
set(SCRIPTS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/scripts/codegen")
# Input macro files
set(TRANSACTIONS_MACRO "${MACRO_DIR}/transactions.macro")
@@ -43,6 +42,7 @@ function(setup_protocol_autogen)
set(LEDGER_TEST_TEMPLATE
"${SCRIPTS_DIR}/templates/LedgerEntryTests.cpp.mako"
)
set(UPDATE_STAMP_SCRIPT "${SCRIPTS_DIR}/update_codegen_stamp.py")
# Check if code generation is disabled
if(XRPL_NO_CODEGEN)
@@ -60,7 +60,33 @@ function(setup_protocol_autogen)
file(MAKE_DIRECTORY "${AUTOGEN_TEST_DIR}/ledger_entries")
file(MAKE_DIRECTORY "${AUTOGEN_TEST_DIR}/transactions")
# Find Python3 - check if already found by Conan or find it ourselves
# === Stamp file check ===
# All input files whose content affects code generation output.
set(STAMP_FILE "${CMAKE_CURRENT_SOURCE_DIR}/scripts/codegen/.codegen_stamp")
set(ALL_INPUT_FILES
"${TRANSACTIONS_MACRO}"
"${LEDGER_ENTRIES_MACRO}"
"${SFIELDS_MACRO}"
"${GENERATE_TX_SCRIPT}"
"${GENERATE_LEDGER_SCRIPT}"
"${REQUIREMENTS_FILE}"
"${MACRO_PARSER_COMMON}"
"${TX_TEMPLATE}"
"${TX_TEST_TEMPLATE}"
"${LEDGER_TEMPLATE}"
"${LEDGER_TEST_TEMPLATE}"
)
# Tell CMake to reconfigure automatically when any input file changes.
# The reconfigure itself is cheap — it runs the stamp check below
# which only invokes stdlib Python (no venv needed).
set_property(
DIRECTORY
APPEND
PROPERTY CMAKE_CONFIGURE_DEPENDS ${ALL_INPUT_FILES}
)
# Find Python3 (needed for stamp check; no venv required).
if(NOT Python3_EXECUTABLE)
find_package(Python3 COMPONENTS Interpreter QUIET)
endif()
@@ -79,19 +105,45 @@ function(setup_protocol_autogen)
return()
endif()
message(STATUS "Using Python3 for code generation: ${Python3_EXECUTABLE}")
# Check whether the stamp is up-to-date (stdlib-only, no venv).
execute_process(
COMMAND
${Python3_EXECUTABLE} "${UPDATE_STAMP_SCRIPT}" --check
"${STAMP_FILE}" ${ALL_INPUT_FILES}
WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
RESULT_VARIABLE STAMP_CHECK_RESULT
)
# Set up Python virtual environment for code generation
# ------------------------------------------------------------------
# Fast path: stamp matches — generated files are up to date.
# ------------------------------------------------------------------
if(STAMP_CHECK_RESULT EQUAL 0)
message(
STATUS
"Protocol autogen: inputs unchanged (stamp matches), skipping generation"
)
return()
endif()
# ------------------------------------------------------------------
# Slow path: stamp mismatch — run generation at configure time.
# ------------------------------------------------------------------
message(
STATUS
"Protocol autogen: inputs changed, running code generation..."
)
# Set up Python virtual environment for code generation.
if(CODEGEN_VENV_DIR)
# User-provided venv - skip automatic setup
# User-provided venv - skip automatic setup.
set(VENV_DIR "${CODEGEN_VENV_DIR}")
message(STATUS "Using user-provided Python venv: ${VENV_DIR}")
else()
# Use default venv in build directory
# Use default venv in build directory.
set(VENV_DIR "${CMAKE_CURRENT_BINARY_DIR}/codegen_venv")
endif()
# Determine the Python executable path in the venv
# Determine the Python/pip executables inside the venv.
if(WIN32)
set(VENV_PYTHON "${VENV_DIR}/Scripts/python.exe")
set(VENV_PIP "${VENV_DIR}/Scripts/pip.exe")
@@ -100,9 +152,9 @@ function(setup_protocol_autogen)
set(VENV_PIP "${VENV_DIR}/bin/pip")
endif()
# Only auto-setup venv if not user-provided
# Create or update the virtual environment if needed.
if(NOT CODEGEN_VENV_DIR)
# Check if venv needs to be created or updated
# Check if venv needs to be created or updated.
set(VENV_NEEDS_UPDATE FALSE)
if(NOT EXISTS "${VENV_PYTHON}")
set(VENV_NEEDS_UPDATE TRUE)
@@ -122,8 +174,9 @@ function(setup_protocol_autogen)
)
endif()
# Create/update virtual environment if needed
# Create/update virtual environment if needed.
if(VENV_NEEDS_UPDATE)
# Create the venv.
message(
STATUS
"Setting up Python virtual environment at ${VENV_DIR}"
@@ -140,7 +193,7 @@ function(setup_protocol_autogen)
)
endif()
# Check pip index URL configuration
# Warn if pip is configured with a non-default index (may need VPN).
execute_process(
COMMAND ${VENV_PIP} config get global.index-url
OUTPUT_VARIABLE PIP_INDEX_URL
@@ -162,6 +215,7 @@ function(setup_protocol_autogen)
endif()
endif()
# Install dependencies.
message(STATUS "Installing Python dependencies...")
execute_process(
COMMAND ${VENV_PIP} install --upgrade pip
@@ -185,125 +239,56 @@ function(setup_protocol_autogen)
)
endif()
# Mark requirements as installed
# Mark requirements as installed.
file(TOUCH "${VENV_DIR}/.requirements_installed")
message(STATUS "Python virtual environment ready")
endif()
endif()
# At configure time - get list of output files for transactions
# Generate transaction classes.
execute_process(
COMMAND
${VENV_PYTHON} "${GENERATE_TX_SCRIPT}" "${TRANSACTIONS_MACRO}"
--header-dir "${AUTOGEN_HEADER_DIR}/transactions" --test-dir
"${AUTOGEN_TEST_DIR}/transactions" --list-outputs
OUTPUT_VARIABLE TX_OUTPUT_FILES
OUTPUT_STRIP_TRAILING_WHITESPACE
RESULT_VARIABLE TX_LIST_RESULT
ERROR_VARIABLE TX_LIST_ERROR
)
if(NOT TX_LIST_RESULT EQUAL 0)
message(
FATAL_ERROR
"Failed to list transaction output files:\n${TX_LIST_ERROR}"
)
endif()
# Convert newline-separated list to CMake list
string(REPLACE "\\" "/" TX_OUTPUT_FILES "${TX_OUTPUT_FILES}")
string(REPLACE "\n" ";" TX_OUTPUT_FILES "${TX_OUTPUT_FILES}")
# At configure time - get list of output files for ledger entries
execute_process(
COMMAND
${VENV_PYTHON} "${GENERATE_LEDGER_SCRIPT}" "${LEDGER_ENTRIES_MACRO}"
--header-dir "${AUTOGEN_HEADER_DIR}/ledger_entries" --test-dir
"${AUTOGEN_TEST_DIR}/ledger_entries" --list-outputs
OUTPUT_VARIABLE LEDGER_OUTPUT_FILES
OUTPUT_STRIP_TRAILING_WHITESPACE
RESULT_VARIABLE LEDGER_LIST_RESULT
ERROR_VARIABLE LEDGER_LIST_ERROR
)
if(NOT LEDGER_LIST_RESULT EQUAL 0)
message(
FATAL_ERROR
"Failed to list ledger entry output files:\n${LEDGER_LIST_ERROR}"
)
endif()
# Convert newline-separated list to CMake list
string(REPLACE "\\" "/" LEDGER_OUTPUT_FILES "${LEDGER_OUTPUT_FILES}")
string(REPLACE "\n" ";" LEDGER_OUTPUT_FILES "${LEDGER_OUTPUT_FILES}")
# Custom command to generate transaction classes at build time
add_custom_command(
OUTPUT ${TX_OUTPUT_FILES}
COMMAND
${VENV_PYTHON} "${GENERATE_TX_SCRIPT}" "${TRANSACTIONS_MACRO}"
--header-dir "${AUTOGEN_HEADER_DIR}/transactions" --test-dir
"${AUTOGEN_TEST_DIR}/transactions" --sfields-macro
"${SFIELDS_MACRO}"
WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
DEPENDS
"${TRANSACTIONS_MACRO}"
"${SFIELDS_MACRO}"
"${GENERATE_TX_SCRIPT}"
"${MACRO_PARSER_COMMON}"
"${TX_TEMPLATE}"
"${TX_TEST_TEMPLATE}"
"${REQUIREMENTS_FILE}"
COMMENT "Generating transaction classes from transactions.macro..."
VERBATIM
RESULT_VARIABLE TX_RESULT
ERROR_VARIABLE TX_ERROR
)
if(NOT TX_RESULT EQUAL 0)
message(FATAL_ERROR "Transaction code generation failed:\n${TX_ERROR}")
endif()
# Custom command to generate ledger entry classes at build time
add_custom_command(
OUTPUT ${LEDGER_OUTPUT_FILES}
# Generate ledger entry classes.
execute_process(
COMMAND
${VENV_PYTHON} "${GENERATE_LEDGER_SCRIPT}" "${LEDGER_ENTRIES_MACRO}"
--header-dir "${AUTOGEN_HEADER_DIR}/ledger_entries" --test-dir
"${AUTOGEN_TEST_DIR}/ledger_entries" --sfields-macro
"${SFIELDS_MACRO}"
WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
DEPENDS
"${LEDGER_ENTRIES_MACRO}"
"${SFIELDS_MACRO}"
"${GENERATE_LEDGER_SCRIPT}"
"${MACRO_PARSER_COMMON}"
"${LEDGER_TEMPLATE}"
"${LEDGER_TEST_TEMPLATE}"
"${REQUIREMENTS_FILE}"
COMMENT "Generating ledger entry classes from ledger_entries.macro..."
VERBATIM
RESULT_VARIABLE LEDGER_RESULT
ERROR_VARIABLE LEDGER_ERROR
)
if(NOT LEDGER_RESULT EQUAL 0)
message(
FATAL_ERROR
"Ledger entry code generation failed:\n${LEDGER_ERROR}"
)
endif()
# Create a custom target that depends on all generated files
add_custom_target(
protocol_autogen_generate
DEPENDS ${TX_OUTPUT_FILES} ${LEDGER_OUTPUT_FILES}
COMMENT "Protocol autogen code generation"
# Update the stamp file so subsequent configures skip generation.
execute_process(
COMMAND
${Python3_EXECUTABLE} "${UPDATE_STAMP_SCRIPT}" --update
"${STAMP_FILE}" ${ALL_INPUT_FILES}
WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
RESULT_VARIABLE STAMP_RESULT
)
if(NOT STAMP_RESULT EQUAL 0)
message(WARNING "Failed to update codegen stamp file")
endif()
# Extract test files from output lists (files ending in Tests.cpp)
set(PROTOCOL_AUTOGEN_TEST_SOURCES "")
foreach(FILE ${TX_OUTPUT_FILES} ${LEDGER_OUTPUT_FILES})
if(FILE MATCHES "Tests\\.cpp$")
list(APPEND PROTOCOL_AUTOGEN_TEST_SOURCES "${FILE}")
endif()
endforeach()
# Export test sources to parent scope for use in test CMakeLists.txt
set(PROTOCOL_AUTOGEN_TEST_SOURCES
"${PROTOCOL_AUTOGEN_TEST_SOURCES}"
CACHE INTERNAL
"Generated protocol_autogen test sources"
)
# Register dependencies so CMake reconfigures when macro files change
# (to update the list of output files)
set_property(
DIRECTORY
APPEND
PROPERTY
CMAKE_CONFIGURE_DEPENDS
"${TRANSACTIONS_MACRO}"
"${LEDGER_ENTRIES_MACRO}"
)
message(STATUS "Protocol autogen: code generation complete")
endfunction()

18
include/xrpl/protocol_autogen/README.md
View File

@@ -6,15 +6,15 @@ This directory contains auto-generated C++ wrapper classes for XRP Ledger protoc
The files in this directory are automatically generated at **CMake configure time** from macro definition files:
- **Transaction classes** (in `transactions/`): Generated from `include/xrpl/protocol/detail/transactions.macro` by `scripts/generate_tx_classes.py`
- **Ledger entry classes** (in `ledger_entries/`): Generated from `include/xrpl/protocol/detail/ledger_entries.macro` by `scripts/generate_ledger_classes.py`
- **Transaction classes** (in `transactions/`): Generated from `include/xrpl/protocol/detail/transactions.macro` by `scripts/codegen/generate_tx_classes.py`
- **Ledger entry classes** (in `ledger_entries/`): Generated from `include/xrpl/protocol/detail/ledger_entries.macro` by `scripts/codegen/generate_ledger_classes.py`
## Generation Process
The generation happens automatically when you **configure** the project (not during build). When you run CMake, the system:
1. Creates a Python virtual environment in the build directory (`codegen_venv`)
2. Installs Python dependencies from `scripts/requirements.txt` into the venv (only if needed)
2. Installs Python dependencies from `scripts/codegen/requirements.txt` into the venv (only if needed)
3. Runs the Python generation scripts using the venv Python interpreter
4. Parses the macro files to extract type definitions
5. Generates type-safe C++ wrapper classes using Mako templates
@@ -26,7 +26,7 @@ The code is regenerated when:
- You run CMake configure for the first time
- The Python virtual environment doesn't exist
- `scripts/requirements.txt` has been modified
- `scripts/codegen/requirements.txt` has been modified
To force regeneration, delete the build directory and reconfigure.
@@ -55,9 +55,9 @@ The generated `.h` files **are checked into version control**. This means:
To modify the generated classes:
- Edit the macro files in `include/xrpl/protocol/detail/`
- Edit the Mako templates in `scripts/templates/`
- Edit the generation scripts in `scripts/`
- Update Python dependencies in `scripts/requirements.txt`
- Edit the Mako templates in `scripts/codegen/templates/`
- Edit the generation scripts in `scripts/codegen/`
- Update Python dependencies in `scripts/codegen/requirements.txt`
- Run CMake configure to regenerate
## Adding Common Fields
@@ -73,7 +73,7 @@ Base classes:
Templates (update to pass required common fields to base class constructors):
- `scripts/templates/Transaction.h.mako`
- `scripts/templates/LedgerEntry.h.mako`
- `scripts/codegen/templates/Transaction.h.mako`
- `scripts/codegen/templates/LedgerEntry.h.mako`
These files are **not auto-generated** and must be updated by hand.

4
scripts/codegen/.codegen_stamp Normal file
View File

@@ -0,0 +1,4 @@
# Auto-generated by protocol autogen - do not edit manually.
# This file tracks input hashes to avoid unnecessary code regeneration.
# It should be checked into version control alongside the generated files.
COMBINED_HASH=24a9168ac6a450f09fa4e2ab288d06624a368041e91fbc7741101d3565d1e601

17
scripts/generate_ledger_classes.py → scripts/codegen/generate_ledger_classes.py
View File

@@ -138,28 +138,11 @@ def main():
"--sfields-macro",
help="Path to sfields.macro (default: auto-detect from macro_path)",
)
parser.add_argument(
"--list-outputs",
action="store_true",
help="List output files without generating (one per line)",
)
args = parser.parse_args()
# Parse the macro file to get ledger entry names
entries = parse_macro_file(args.macro_path)
# If --list-outputs, just print the output file paths and exit
if args.list_outputs:
header_dir = Path(args.header_dir)
for entry in entries:
print(header_dir / f"{entry['name']}.h")
if args.test_dir:
test_dir = Path(args.test_dir)
for entry in entries:
print(test_dir / f"{entry['name']}Tests.cpp")
return
# Auto-detect sfields.macro path if not provided
if args.sfields_macro:
sfields_path = Path(args.sfields_macro)

17
scripts/generate_tx_classes.py → scripts/codegen/generate_tx_classes.py
View File

@@ -147,28 +147,11 @@ def main():
"--sfields-macro",
help="Path to sfields.macro (default: auto-detect from macro_path)",
)
parser.add_argument(
"--list-outputs",
action="store_true",
help="List output files without generating (one per line)",
)
args = parser.parse_args()
# Parse the macro file to get transaction names
transactions = parse_macro_file(args.macro_path)
# If --list-outputs, just print the output file paths and exit
if args.list_outputs:
header_dir = Path(args.header_dir)
for tx in transactions:
print(header_dir / f"{tx['name']}.h")
if args.test_dir:
test_dir = Path(args.test_dir)
for tx in transactions:
print(test_dir / f"{tx['name']}Tests.cpp")
return
# Auto-detect sfields.macro path if not provided
if args.sfields_macro:
sfields_path = Path(args.sfields_macro)

0
scripts/macro_parser_common.py → scripts/codegen/macro_parser_common.py
View File

0
scripts/requirements.txt → scripts/codegen/requirements.txt
View File

0
scripts/templates/LedgerEntry.h.mako → scripts/codegen/templates/LedgerEntry.h.mako
View File

0
scripts/templates/LedgerEntryTests.cpp.mako → scripts/codegen/templates/LedgerEntryTests.cpp.mako
View File

0
scripts/templates/Transaction.h.mako → scripts/codegen/templates/Transaction.h.mako
View File

0
scripts/templates/TransactionTests.cpp.mako → scripts/codegen/templates/TransactionTests.cpp.mako
View File

80
scripts/codegen/update_codegen_stamp.py Normal file
View File

@@ -0,0 +1,80 @@
#!/usr/bin/env python3
"""
Check or update the codegen stamp file.
Uses only the Python standard library (hashlib, pathlib, sys) so it can
run without a virtual environment.
Modes:
--check Exit 0 if stamp is up-to-date, exit 1 if stale/missing.
--update Recompute the hash and write it to the stamp file.
Usage:
python update_codegen_stamp.py --check <stamp_file> <input_files...>
python update_codegen_stamp.py --update <stamp_file> <input_files...>
"""
import hashlib
import sys
from pathlib import Path
def compute_combined_hash(input_files: list[str]) -> str:
"""Compute a combined SHA-256 hash of all input files.
Algorithm: compute each file's SHA-256 hex digest, concatenate them
all, then SHA-256 the concatenation.
"""
parts = []
for filepath in input_files:
file_hash = hashlib.sha256(Path(filepath).read_bytes()).hexdigest()
parts.append(file_hash)
combined = "".join(parts)
return hashlib.sha256(combined.encode()).hexdigest()
def read_stamp_hash(stamp_file: str) -> str:
"""Read the COMBINED_HASH from an existing stamp file, or '' if missing."""
path = Path(stamp_file)
if not path.exists():
return ""
for line in path.read_text().splitlines():
if line.startswith("COMBINED_HASH="):
return line.split("=", 1)[1]
return ""
def main():
if len(sys.argv) < 4 or sys.argv[1] not in ("--check", "--update"):
print(
f"Usage: {sys.argv[0]} --check|--update <stamp_file> <input_files...>",
file=sys.stderr,
)
sys.exit(2)
mode = sys.argv[1]
stamp_file = sys.argv[2]
input_files = sys.argv[3:]
current_hash = compute_combined_hash(input_files)
if mode == "--check":
stamp_hash = read_stamp_hash(stamp_file)
if current_hash == stamp_hash:
sys.exit(0)
else:
sys.exit(1)
# --update
with open(stamp_file, "w") as fp:
fp.write(
"# Auto-generated by protocol autogen - do not edit manually.\n"
"# This file tracks input hashes to avoid unnecessary code regeneration.\n"
"# It should be checked into version control alongside the generated files.\n"
)
fp.write(f"COMBINED_HASH={current_hash}\n")
if __name__ == "__main__":
main()

15
src/tests/libxrpl/CMakeLists.txt
View File

@@ -32,20 +32,11 @@ xrpl_add_test(json)
target_link_libraries(xrpl.test.json PRIVATE xrpl.imports.test)
add_dependencies(xrpl.tests xrpl.test.json)
# protocol_autogen tests use explicit source list (not GLOB) because sources are generated
# Mark generated sources so CMake knows they'll be created at build time
set_source_files_properties(
${PROTOCOL_AUTOGEN_TEST_SOURCES}
PROPERTIES GENERATED TRUE
)
add_executable(xrpl.test.protocol_autogen ${PROTOCOL_AUTOGEN_TEST_SOURCES})
# protocol_autogen tests — sources are checked into git so GLOB works.
# Code generation runs at configure time when inputs change.
xrpl_add_test(protocol_autogen)
target_link_libraries(xrpl.test.protocol_autogen PRIVATE xrpl.imports.test)
add_dependencies(xrpl.tests xrpl.test.protocol_autogen)
add_test(NAME xrpl.test.protocol_autogen COMMAND xrpl.test.protocol_autogen)
# Ensure code generation runs before compiling tests
if(TARGET protocol_autogen_generate)
add_dependencies(xrpl.test.protocol_autogen protocol_autogen_generate)
endif()
# Network unit tests are currently not supported on Windows
if(NOT WIN32)