Simplify protobuf generation

2025-11-20 10:35:50 +00:00 · 2024-03-22 09:15:08 -05:00
parent 3bf1229b42
commit 13ea38bcbb
7 changed files with 114 additions and 113 deletions
--- a/Builds/CMake/RippledCore.cmake
+++ b/Builds/CMake/RippledCore.cmake
@@ -3,10 +3,55 @@
   core functionality, useable by some client software perhaps
 #]===================================================================]

+include(target_protobuf_sources)
+
 file (GLOB_RECURSE rb_headers
  src/ripple/beast/*.h
  src/ripple/beast/*.hpp)

+# Protocol buffers cannot participate in a unity build,
+# because all the generated sources
+# define a bunch of `static const` variables with the same names,
+# so we just build them as a separate library.
+add_library(xrpl.libpb)
+target_protobuf_sources(xrpl.libpb ripple/proto
+  LANGUAGE cpp
+  IMPORT_DIRS src/ripple/proto
+  PROTOS src/ripple/proto/ripple.proto
+)
+
+file(GLOB_RECURSE protos "src/ripple/proto/org/*.proto")
+target_protobuf_sources(xrpl.libpb ripple/proto
+  LANGUAGE cpp
+  IMPORT_DIRS src/ripple/proto
+  PROTOS "${protos}"
+)
+target_protobuf_sources(xrpl.libpb ripple/proto
+  LANGUAGE grpc
+  IMPORT_DIRS src/ripple/proto
+  PROTOS "${protos}"
+  PLUGIN protoc-gen-grpc=$<TARGET_FILE:gRPC::grpc_cpp_plugin>
+  GENERATE_EXTENSIONS .grpc.pb.h .grpc.pb.cc
+)
+
+target_compile_options(xrpl.libpb
+  PUBLIC
+    $<$<BOOL:${MSVC}>:-wd4996>
+    $<$<BOOL:${XCODE}>:
+      --system-header-prefix="google/protobuf"
+      -Wno-deprecated-dynamic-exception-spec
+    >
+  PRIVATE
+    $<$<BOOL:${MSVC}>:-wd4065>
+    $<$<NOT:$<BOOL:${MSVC}>>:-Wno-deprecated-declarations>
+)
+
+target_link_libraries(xrpl.libpb
+  PUBLIC
+    protobuf::libprotobuf
+    gRPC::grpc++
+)
+
 add_library (xrpl_core
  ${rb_headers}) ## headers added here for benefit of IDEs
 if (unity)
@@ -27,7 +72,6 @@ add_library(libxrpl INTERFACE)
 target_link_libraries(libxrpl INTERFACE xrpl_core)
 add_library(xrpl::libxrpl ALIAS libxrpl)

-
 #[===============================[
    beast/legacy FILES:
    TODO: review these sources for removal or replacement
@@ -163,6 +207,7 @@ target_link_libraries (xrpl_core
    Ripple::syslibs
    secp256k1::secp256k1
    ed25519::ed25519
+    xrpl.libpb
    date::date
    Ripple::opts
    xxHash::xxhash)
--- a/Builds/CMake/RippledInstall.cmake
+++ b/Builds/CMake/RippledInstall.cmake
@@ -9,6 +9,7 @@ install (
    ripple_syslibs
    ripple_boost
    xrpl_core
+    xrpl.libpb
  EXPORT RippleExports
  LIBRARY DESTINATION lib
  ARCHIVE DESTINATION lib
--- a/Builds/CMake/conan/Protobuf.cmake
+++ b/Builds/CMake/conan/Protobuf.cmake
@@ -1,27 +0,0 @@
-find_package(Protobuf 3.8)
-
-set(output_dir ${CMAKE_BINARY_DIR}/proto_gen)
-file(MAKE_DIRECTORY ${output_dir})
-set(ccbd ${CMAKE_CURRENT_BINARY_DIR})
-set(CMAKE_CURRENT_BINARY_DIR ${output_dir})
-protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS src/ripple/proto/ripple.proto)
-set(CMAKE_CURRENT_BINARY_DIR ${ccbd})
-
-target_include_directories(xrpl_core SYSTEM PUBLIC
-    # The generated implementation imports the header relative to the output
-    # directory.
-    $<BUILD_INTERFACE:${output_dir}>
-    $<BUILD_INTERFACE:${output_dir}/src>
-)
-target_sources(xrpl_core PRIVATE ${output_dir}/src/ripple/proto/ripple.pb.cc)
-install(
-  FILES ${output_dir}/src/ripple/proto/ripple.pb.h
-  DESTINATION include/ripple/proto)
-target_link_libraries(xrpl_core PUBLIC protobuf::libprotobuf)
-target_compile_options(xrpl_core
-  PUBLIC
-    $<$<BOOL:${XCODE}>:
-      --system-header-prefix="google/protobuf"
-      -Wno-deprecated-dynamic-exception-spec
-    >
-)
--- a/Builds/CMake/conan/gRPC.cmake
+++ b/Builds/CMake/conan/gRPC.cmake
@@ -1,82 +0,0 @@
-find_package(gRPC 1.23)
-
-#[=================================[
-   generate protobuf sources for
-   grpc defs and bundle into a
-   static lib
-#]=================================]
-set(output_dir "${CMAKE_BINARY_DIR}/proto_gen_grpc")
-set(GRPC_GEN_DIR "${output_dir}/ripple/proto")
-file(MAKE_DIRECTORY ${GRPC_GEN_DIR})
-set(GRPC_PROTO_SRCS)
-set(GRPC_PROTO_HDRS)
-set(GRPC_PROTO_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/src/ripple/proto/org")
-file(GLOB_RECURSE GRPC_DEFINITION_FILES "${GRPC_PROTO_ROOT}/*.proto")
-foreach(file ${GRPC_DEFINITION_FILES})
-  # /home/user/rippled/src/ripple/proto/org/.../v1/get_ledger.proto
-  get_filename_component(_abs_file ${file} ABSOLUTE)
-  # /home/user/rippled/src/ripple/proto/org/.../v1
-  get_filename_component(_abs_dir ${_abs_file} DIRECTORY)
-  # get_ledger
-  get_filename_component(_basename ${file} NAME_WE)
-  # /home/user/rippled/src/ripple/proto
-  get_filename_component(_proto_inc ${GRPC_PROTO_ROOT} DIRECTORY) # updir one level
-  # org/.../v1/get_ledger.proto
-  file(RELATIVE_PATH _rel_root_file ${_proto_inc} ${_abs_file})
-  # org/.../v1
-  get_filename_component(_rel_root_dir ${_rel_root_file} DIRECTORY)
-  # src/ripple/proto/org/.../v1
-  file(RELATIVE_PATH _rel_dir ${CMAKE_CURRENT_SOURCE_DIR} ${_abs_dir})
-
-  # .cmake/proto_gen_grpc/ripple/proto/org/.../v1/get_ledger.grpc.pb.cc
-  set(src_1 "${GRPC_GEN_DIR}/${_rel_root_dir}/${_basename}.grpc.pb.cc")
-  set(src_2 "${GRPC_GEN_DIR}/${_rel_root_dir}/${_basename}.pb.cc")
-  set(hdr_1 "${GRPC_GEN_DIR}/${_rel_root_dir}/${_basename}.grpc.pb.h")
-  set(hdr_2 "${GRPC_GEN_DIR}/${_rel_root_dir}/${_basename}.pb.h")
-  add_custom_command(
-    OUTPUT ${src_1} ${src_2} ${hdr_1} ${hdr_2}
-    COMMAND protobuf::protoc
-    ARGS --grpc_out=${GRPC_GEN_DIR}
-         --cpp_out=${GRPC_GEN_DIR}
-         --plugin=protoc-gen-grpc=$<TARGET_FILE:gRPC::grpc_cpp_plugin>
-         -I ${_proto_inc} -I ${_rel_dir}
-         ${_abs_file}
-    DEPENDS ${_abs_file} protobuf::protoc gRPC::grpc_cpp_plugin
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-    COMMENT "Running gRPC C++ protocol buffer compiler on ${file}"
-    VERBATIM)
-    set_source_files_properties(${src_1} ${src_2} ${hdr_1} ${hdr_2} PROPERTIES
-      GENERATED TRUE
-      SKIP_UNITY_BUILD_INCLUSION ON
-    )
-    list(APPEND GRPC_PROTO_SRCS ${src_1} ${src_2})
-    list(APPEND GRPC_PROTO_HDRS ${hdr_1} ${hdr_2})
-endforeach()
-
-target_include_directories(xrpl_core SYSTEM PUBLIC
-  $<BUILD_INTERFACE:${output_dir}>
-  $<BUILD_INTERFACE:${output_dir}/ripple/proto>
-  # The generated sources include headers relative to this path. Fix it later.
-  $<INSTALL_INTERFACE:include/ripple/proto>
-)
-target_sources(xrpl_core PRIVATE ${GRPC_PROTO_SRCS})
-install(
-  DIRECTORY ${output_dir}/ripple
-  DESTINATION include/
-  FILES_MATCHING PATTERN "*.h"
-)
-target_link_libraries(xrpl_core PUBLIC
-  "gRPC::grpc++"
-  # libgrpc is missing references.
-  absl::random_random
-)
-target_compile_options(xrpl_core
-  PRIVATE
-    $<$<BOOL:${MSVC}>:-wd4065>
-    $<$<NOT:$<BOOL:${MSVC}>>:-Wno-deprecated-declarations>
-  PUBLIC
-    $<$<BOOL:${MSVC}>:-wd4996>
-    $<$<BOOL:${XCODE}>:
-      --system-header-prefix="google/protobuf"
-      -Wno-deprecated-dynamic-exception-spec
-    >)
--- a/Builds/CMake/target_protobuf_sources.cmake
+++ b/Builds/CMake/target_protobuf_sources.cmake
@@ -0,0 +1,62 @@
+find_package(Protobuf REQUIRED)
+
+# .proto files import each other like this:
+#
+# import "path/to/file.proto";
+#
+# For the protobuf compiler to find these imports,
+# the parent directory of "path" must be in the import path.
+#
+# When generating C++,
+# it turns into an include statement like this:
+#
+# #include "path/to/file.pb.h"
+#
+# and the header is generated at a path relative to the output directory
+# that matches the given .proto path relative to the source directory
+# minus the first matching prefix on the import path.
+#
+# In other words, a file `include/package/path/to/file.proto`
+# with import path [`include/package`, `include`]
+# will generate files `output/path/to/file.pb.{h,cc}`
+# with includes like `#include "path/to/file.pb.h".
+#
+# During build, the generated files can find each other if the output
+# directory is an include directory, but we want to install that directory
+# under our package's include directory (`include/package`), not as a sibling.
+# After install, they can find each other if that subdirectory is an include
+# directory.
+
+# Add protocol buffer sources to an existing library target.
+# target:
+#     The name of the library target.
+# prefix:
+#     The install prefix for headers relative to `CMAKE_INSTALL_INCLUDEDIR`.
+#     This prefix should appear at the start of all your consumer includes.
+# ARGN:
+#     A list of .proto files.
+function(target_protobuf_sources target prefix)
+  set(dir "${CMAKE_CURRENT_BINARY_DIR}/pb-${target}")
+  file(MAKE_DIRECTORY "${dir}/${prefix}")
+
+  protobuf_generate(
+    TARGET ${target}
+    PROTOC_OUT_DIR "${dir}/${prefix}"
+    "${ARGN}"
+  )
+  target_include_directories(${target} SYSTEM PUBLIC
+    # Allows #include <package/path/to/file.proto> used by consumer files.
+    $<BUILD_INTERFACE:${dir}>
+    # Allows #include "path/to/file.proto" used by generated files.
+    $<BUILD_INTERFACE:${dir}/${prefix}>
+    # Allows #include <package/path/to/file.proto> used by consumer files.
+    $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+    # Allows #include "path/to/file.proto" used by generated files.
+    $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}/${prefix}>
+  )
+  install(
+    DIRECTORY ${dir}/
+    DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
+    FILES_MATCHING PATTERN "*.h"
+  )
+endfunction()
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -113,6 +113,7 @@ else()
  add_subdirectory(external/secp256k1)
  add_library(secp256k1::secp256k1 ALIAS secp256k1)
  add_subdirectory(external/ed25519-donna)
+  find_package(gRPC REQUIRED)
  find_package(lz4 REQUIRED)
  # Target names with :: are not allowed in a generator expression.
  # We need to pull the include directories and imported location properties
@@ -174,9 +175,6 @@ include(RippledCore)
 if (NOT USE_CONAN)
  include(deps/Protobuf)
  include(deps/gRPC)
-else()
-  include(conan/Protobuf)
-  include(conan/gRPC)
 endif()
 include(RippledInstall)
 include(RippledMultiConfig)
--- a/conanfile.py
+++ b/conanfile.py
@@ -40,6 +40,10 @@ class Xrpl(ConanFile):
        'wasmedge/0.11.2',
    ]

+    tool_requires = [
+        'protobuf/3.21.9',
+    ]
+
    default_options = {
        'assertions': False,
        'coverage': False,