Add patched recipe for SOCI: (#4510)

SOCI is the C++ database access library. The SOCI recipe was updated in
Conan Center Index (CCI), and it breaks for our choice of options. This
breakage occurs when you build with a fresh Conan cache (e.g. when you
submit a PR, or delete `~/.conan/data`).

* Add a custom Conan recipe for SOCI v4.0.3
* Update dependency building to handle exporting and installing Snappy
  and SOCI
  * Fix workflows to use custom SOCI recipe
* Update BUILD.md to include instruction for exporting the SOCI Conan
  recipe:
  * `conan export external/soci soci/4.0.3@`

This solution has been verified on Ubuntu 20.04 and macOS.

Context:

* There is a compiler error that the `sqlite3.h` header is not available
  when building soci.
* When package B depends on package A, it finds the pieces it needs by
  importing the Package Configuration File (PCF) that Conan generates
  for package A.
  * Read the CMake written by package B to check that it is importing
    the PCF correctly and linking its exports correctly.
  * Since this can be difficult, it is often more efficient to check
    https://github.com/conan-io/conan-center-index/issues for package B
    to see if anyone else has seen a similar problem.
  * One of the issues points to a problem area in soci's CMake. To
    confirm the diagnosis, review soci's CMake (after any patches are
    applied) in the Conan build directory `build/$buildId/src/`.
  * Review the Conan-generated PCF in
    `build/$buildId/build/$buildType/generators/`.
  * In this case, the problem was likely (re)introduced by
    https://github.com/conan-io/conan-center-index/pull/17026
* If there is a problem in the source or in the Conan recipe, the
  fastest fix is to copy the recipe and either:
  * Add a source patch to fix any problems in the source.
  * Change the recipe to fix any problems in the recipe.
* In this case, this can be done by finding soci's Conan recipe at
  https://github.com/conan-io/conan-center-index/tree/master/recipes/soci
  and then copying the `all` directory as `external/$packageName` in our
  project. Then, make any changes.
  * Test packages can be removed from the recipe folder as they are not
    needed.
  * If adding a patch in the `patches` directory, add a description for
    it to `conandata.yml`.
  * Since `conanfile.py` has no `version` property on the recipe class,
    builders need to pass a version on the command line (like they do
    for our `snappy` recipe).
* Add an example command to `BUILD.md`.

Future work: It may make sense to refer to recipes by revision, by
checking in a lockfile.

.github/actions/build/action.yml

@@ -10,19 +10,10 @@ inputs:
 runs:
   using: composite
   steps:
-    - name: export custom recipes
-      shell: bash
-      run: conan export external/snappy snappy/1.1.9@
-    - name: install dependencies
-      shell: bash
-      run: |
-        mkdir ${build_dir}
-        cd ${build_dir}
-        conan install \
-          --output-folder . \
-          --build missing \
-          --settings build_type=${{ inputs.configuration }} \
-          ..
+    - name: dependencies
+      uses: ./.github/actions/dependencies
+      with:
+        configuration: ${{ inputs.configuration }}
     - name: configure
       shell: bash
       run: |

.github/actions/dependencies/action.yml

@@ -0,0 +1,23 @@
+name: dependencies
+inputs:
+  configuration:
+    required: true
+# An implicit input is the environment variable `build_dir`.
+runs:
+  using: composite
+  steps:
+    - name: export custom recipes
+      shell: bash
+      run: |
+        conan export external/snappy snappy/1.1.9@
+        conan export external/soci soci/4.0.3@
+    - name: install dependencies
+      shell: bash
+      run: |
+        mkdir ${build_dir}
+        cd ${build_dir}
+        conan install \
+          --output-folder . \
+          --build missing \
+          --settings build_type=${{ inputs.configuration }} \
+          ..

.github/workflows/nix.yml

@@ -28,6 +28,8 @@ jobs:
               cxx: /usr/bin/clang++-14
     runs-on: [self-hosted, heavy]
     container: thejohnfreeman/rippled-build-ubuntu:12e19cd9034b
+    env:
+      build_dir: .build
     steps:
       - name: check environment
         run: |
@@ -47,15 +49,10 @@ jobs:
           conan profile update conf.tools.build:compiler_executables='{"c": "${{ matrix.profile.cc }}", "cpp": "${{ matrix.profile.cxx }}"}' default
       - name: checkout
         uses: actions/checkout@v3
-      - name: build dependencies
-        run: |
-          mkdir .build
-          cd .build
-          conan install \
-            --output-folder . \
-            --build missing \
-            --settings build_type=${{ matrix.configuration }} \
-            ..
+      - name: dependencies
+        uses: ./.github/actions/dependencies
+        with:
+          configuration: ${{ matrix.configuration }}
       - name: archive cache
         run: tar -czf conan.tar -C ~/.conan .
       - name: upload cache

.github/workflows/windows.yml

@@ -69,8 +69,10 @@ jobs:
         with:
             path: ${{ steps.conan-cache.outputs.dir }}
             key: ${{ hashFiles('~/.conan/profiles/default', 'conanfile.py', 'external/rocksdb/*', '.github/workflows/windows.yml') }}
-      - name: export Snappy
-        run: conan export external/snappy snappy/1.1.9@
+      - name: export custom recipes
+        run: |
+          conan export external/snappy snappy/1.1.9@
+          conan export external/soci soci/4.0.3@
       - name: install dependencies
         run: |
           mkdir $env:build_dir

BUILD.md

@@ -127,6 +127,13 @@ which allows you to statically link it with GCC, if you want.
 conan export external/snappy snappy/1.1.9@
 ```
 
+Export our [Conan recipe for SOCI](./external/soci).
+It patches their CMake to correctly import its dependencies.
+
+```
+conan export external/soci soci/4.0.3@
+```
+
 ## How to build and test
 
 Let's start with a couple of examples of common workflows.

external/soci/conandata.yml

@@ -0,0 +1,12 @@
+sources:
+  "4.0.3":
+    url: "https://github.com/SOCI/soci/archive/v4.0.3.tar.gz"
+    sha256: "4b1ff9c8545c5d802fbe06ee6cd2886630e5c03bf740e269bb625b45cf934928"
+patches:
+  "4.0.3":
+    - patch_file: "patches/0001-Remove-hardcoded-INSTALL_NAME_DIR-for-relocatable-li.patch"
+      patch_description: "Generate relocatable libraries on MacOS"
+      patch_type: "portability"
+    - patch_file: "patches/0002-Fix-soci_backend.patch"
+      patch_description: "Fix variable names for dependencies"
+      patch_type: "conan"

external/soci/conanfile.py

@@ -0,0 +1,212 @@
+from conan import ConanFile
+from conan.tools.build import check_min_cppstd
+from conan.tools.cmake import CMake, CMakeDeps, CMakeToolchain, cmake_layout
+from conan.tools.files import apply_conandata_patches, copy, export_conandata_patches, get, rmdir
+from conan.tools.microsoft import is_msvc
+from conan.tools.scm import Version
+from conan.errors import ConanInvalidConfiguration
+import os
+
+required_conan_version = ">=1.55.0"
+
+
+class SociConan(ConanFile):
+    name = "soci"
+    homepage = "https://github.com/SOCI/soci"
+    url = "https://github.com/conan-io/conan-center-index"
+    description = "The C++ Database Access Library "
+    topics = ("mysql", "odbc", "postgresql", "sqlite3")
+    license = "BSL-1.0"
+
+    settings = "os", "arch", "compiler", "build_type"
+    options = {
+        "shared":           [True, False],
+        "fPIC":             [True, False],
+        "empty":            [True, False],
+        "with_sqlite3":     [True, False],
+        "with_db2":         [True, False],
+        "with_odbc":        [True, False],
+        "with_oracle":      [True, False],
+        "with_firebird":    [True, False],
+        "with_mysql":       [True, False],
+        "with_postgresql":  [True, False],
+        "with_boost":       [True, False],
+    }
+    default_options = {
+        "shared":           False,
+        "fPIC":             True,
+        "empty":            False,
+        "with_sqlite3":     False,
+        "with_db2":         False,
+        "with_odbc":        False,
+        "with_oracle":      False,
+        "with_firebird":    False,
+        "with_mysql":       False,
+        "with_postgresql":  False,
+        "with_boost":       False,
+    }
+
+    def export_sources(self):
+        export_conandata_patches(self)
+
+    def layout(self):
+        cmake_layout(self, src_folder="src")
+
+    def config_options(self):
+        if self.settings.os == "Windows":
+            self.options.rm_safe("fPIC")
+
+    def configure(self):
+        if self.options.shared:
+            self.options.rm_safe("fPIC")
+
+    def requirements(self):
+        if self.options.with_sqlite3:
+            self.requires("sqlite3/3.41.1")
+        if self.options.with_odbc and self.settings.os != "Windows":
+            self.requires("odbc/2.3.11")
+        if self.options.with_mysql:
+            self.requires("libmysqlclient/8.0.31")
+        if self.options.with_postgresql:
+            self.requires("libpq/14.7")
+        if self.options.with_boost:
+            self.requires("boost/1.81.0")
+
+    @property
+    def _minimum_compilers_version(self):
+        return {
+            "Visual Studio": "14",
+            "gcc": "4.8",
+            "clang": "3.8",
+            "apple-clang": "8.0"
+        }
+
+    def validate(self):
+        if self.settings.compiler.get_safe("cppstd"):
+            check_min_cppstd(self, 11)
+
+        compiler = str(self.settings.compiler)
+        compiler_version = Version(self.settings.compiler.version.value)
+        if compiler not in self._minimum_compilers_version:
+            self.output.warning("{} recipe lacks information about the {} compiler support.".format(self.name, self.settings.compiler))
+        elif compiler_version < self._minimum_compilers_version[compiler]:
+            raise ConanInvalidConfiguration("{} requires a {} version >= {}".format(self.name, compiler, compiler_version))
+
+        prefix  = "Dependencies for"
+        message = "not configured in this conan package."
+        if self.options.with_db2:
+            # self.requires("db2/0.0.0") # TODO add support for db2
+            raise ConanInvalidConfiguration("{} DB2 {} ".format(prefix, message))
+        if self.options.with_oracle:
+            # self.requires("oracle_db/0.0.0") # TODO add support for oracle
+            raise ConanInvalidConfiguration("{} ORACLE {} ".format(prefix, message))
+        if self.options.with_firebird:
+            # self.requires("firebird/0.0.0") # TODO add support for firebird
+            raise ConanInvalidConfiguration("{} firebird {} ".format(prefix, message))
+
+    def source(self):
+        get(self, **self.conan_data["sources"][self.version], strip_root=True)
+
+    def generate(self):
+        tc = CMakeToolchain(self)
+
+        tc.variables["SOCI_SHARED"] = self.options.shared
+        tc.variables["SOCI_STATIC"] = not self.options.shared
+        tc.variables["SOCI_TESTS"] = False
+        tc.variables["SOCI_CXX11"] = True
+        tc.variables["SOCI_EMPTY"] = self.options.empty
+        tc.variables["WITH_SQLITE3"] = self.options.with_sqlite3
+        tc.variables["WITH_DB2"] = self.options.with_db2
+        tc.variables["WITH_ODBC"] = self.options.with_odbc
+        tc.variables["WITH_ORACLE"] = self.options.with_oracle
+        tc.variables["WITH_FIREBIRD"] = self.options.with_firebird
+        tc.variables["WITH_MYSQL"] = self.options.with_mysql
+        tc.variables["WITH_POSTGRESQL"] = self.options.with_postgresql
+        tc.variables["WITH_BOOST"] = self.options.with_boost
+        tc.generate()
+
+        deps = CMakeDeps(self)
+        deps.generate()
+
+    def build(self):
+        apply_conandata_patches(self)
+        cmake = CMake(self)
+        cmake.configure()
+        cmake.build()
+
+    def package(self):
+        copy(self, "LICENSE_1_0.txt", dst=os.path.join(self.package_folder, "licenses"), src=self.source_folder)
+
+        cmake = CMake(self)
+        cmake.install()
+
+        rmdir(self, os.path.join(self.package_folder, "lib", "cmake"))
+
+    def package_info(self):
+        self.cpp_info.set_property("cmake_file_name", "SOCI")
+
+        target_suffix = "" if self.options.shared else "_static"
+        lib_prefix = "lib" if is_msvc(self) and not self.options.shared else ""
+        version = Version(self.version)
+        lib_suffix = "_{}_{}".format(version.major, version.minor) if self.settings.os == "Windows" else ""
+
+        # soci_core
+        self.cpp_info.components["soci_core"].set_property("cmake_target_name", "SOCI::soci_core{}".format(target_suffix))
+        self.cpp_info.components["soci_core"].libs = ["{}soci_core{}".format(lib_prefix, lib_suffix)]
+        if self.options.with_boost:
+            self.cpp_info.components["soci_core"].requires.append("boost::boost")
+
+        # soci_empty
+        if self.options.empty:
+            self.cpp_info.components["soci_empty"].set_property("cmake_target_name", "SOCI::soci_empty{}".format(target_suffix))
+            self.cpp_info.components["soci_empty"].libs = ["{}soci_empty{}".format(lib_prefix, lib_suffix)]
+            self.cpp_info.components["soci_empty"].requires = ["soci_core"]
+
+        # soci_sqlite3
+        if self.options.with_sqlite3:
+            self.cpp_info.components["soci_sqlite3"].set_property("cmake_target_name", "SOCI::soci_sqlite3{}".format(target_suffix))
+            self.cpp_info.components["soci_sqlite3"].libs = ["{}soci_sqlite3{}".format(lib_prefix, lib_suffix)]
+            self.cpp_info.components["soci_sqlite3"].requires = ["soci_core", "sqlite3::sqlite3"]
+
+        # soci_odbc
+        if self.options.with_odbc:
+            self.cpp_info.components["soci_odbc"].set_property("cmake_target_name", "SOCI::soci_odbc{}".format(target_suffix))
+            self.cpp_info.components["soci_odbc"].libs = ["{}soci_odbc{}".format(lib_prefix, lib_suffix)]
+            self.cpp_info.components["soci_odbc"].requires = ["soci_core"]
+            if self.settings.os == "Windows":
+                self.cpp_info.components["soci_odbc"].system_libs.append("odbc32")
+            else:
+                self.cpp_info.components["soci_odbc"].requires.append("odbc::odbc")
+
+        # soci_mysql
+        if self.options.with_mysql:
+            self.cpp_info.components["soci_mysql"].set_property("cmake_target_name", "SOCI::soci_mysql{}".format(target_suffix))
+            self.cpp_info.components["soci_mysql"].libs = ["{}soci_mysql{}".format(lib_prefix, lib_suffix)]
+            self.cpp_info.components["soci_mysql"].requires = ["soci_core", "libmysqlclient::libmysqlclient"]
+
+        # soci_postgresql
+        if self.options.with_postgresql:
+            self.cpp_info.components["soci_postgresql"].set_property("cmake_target_name", "SOCI::soci_postgresql{}".format(target_suffix))
+            self.cpp_info.components["soci_postgresql"].libs = ["{}soci_postgresql{}".format(lib_prefix, lib_suffix)]
+            self.cpp_info.components["soci_postgresql"].requires = ["soci_core", "libpq::libpq"]
+
+        # TODO: to remove in conan v2 once cmake_find_package* generators removed
+        self.cpp_info.names["cmake_find_package"] = "SOCI"
+        self.cpp_info.names["cmake_find_package_multi"] = "SOCI"
+        self.cpp_info.components["soci_core"].names["cmake_find_package"] = "soci_core{}".format(target_suffix)
+        self.cpp_info.components["soci_core"].names["cmake_find_package_multi"] = "soci_core{}".format(target_suffix)
+        if self.options.empty:
+            self.cpp_info.components["soci_empty"].names["cmake_find_package"] = "soci_empty{}".format(target_suffix)
+            self.cpp_info.components["soci_empty"].names["cmake_find_package_multi"] = "soci_empty{}".format(target_suffix)
+        if self.options.with_sqlite3:
+            self.cpp_info.components["soci_sqlite3"].names["cmake_find_package"] = "soci_sqlite3{}".format(target_suffix)
+            self.cpp_info.components["soci_sqlite3"].names["cmake_find_package_multi"] = "soci_sqlite3{}".format(target_suffix)
+        if self.options.with_odbc:
+            self.cpp_info.components["soci_odbc"].names["cmake_find_package"] = "soci_odbc{}".format(target_suffix)
+            self.cpp_info.components["soci_odbc"].names["cmake_find_package_multi"] = "soci_odbc{}".format(target_suffix)
+        if self.options.with_mysql:
+            self.cpp_info.components["soci_mysql"].names["cmake_find_package"] = "soci_mysql{}".format(target_suffix)
+            self.cpp_info.components["soci_mysql"].names["cmake_find_package_multi"] = "soci_mysql{}".format(target_suffix)
+        if self.options.with_postgresql:
+            self.cpp_info.components["soci_postgresql"].names["cmake_find_package"] = "soci_postgresql{}".format(target_suffix)
+            self.cpp_info.components["soci_postgresql"].names["cmake_find_package_multi"] = "soci_postgresql{}".format(target_suffix)

external/soci/patches/0001-Remove-hardcoded-INSTALL_NAME_DIR-for-relocatable-li.patch

@@ -0,0 +1,39 @@
+From d491bf7b5040d314ffd0c6310ba01f78ff44c85e Mon Sep 17 00:00:00 2001
+From: Rasmus Thomsen <rasmus.thomsen@dampsoft.de>
+Date: Fri, 14 Apr 2023 09:16:29 +0200
+Subject: [PATCH] Remove hardcoded INSTALL_NAME_DIR for relocatable libraries
+ on MacOS
+
+---
+ cmake/SociBackend.cmake | 2 +-
+ src/core/CMakeLists.txt | 1 -
+ 2 files changed, 1 insertion(+), 2 deletions(-)
+
+diff --git a/cmake/SociBackend.cmake b/cmake/SociBackend.cmake
+index 5d4ef0df..39fe1f77 100644
+--- a/cmake/SociBackend.cmake
++++ b/cmake/SociBackend.cmake
+@@ -171,7 +171,7 @@ macro(soci_backend NAME)
+           set_target_properties(${THIS_BACKEND_TARGET}
+             PROPERTIES
+             SOVERSION ${${PROJECT_NAME}_SOVERSION}
+-            INSTALL_NAME_DIR ${CMAKE_INSTALL_PREFIX}/lib)
++          )
+ 
+           if(APPLE)
+             set_target_properties(${THIS_BACKEND_TARGET}
+diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt
+index 3e7deeae..f9eae564 100644
+--- a/src/core/CMakeLists.txt
++++ b/src/core/CMakeLists.txt
+@@ -59,7 +59,6 @@ if (SOCI_SHARED)
+       PROPERTIES
+       VERSION ${SOCI_VERSION}
+       SOVERSION ${SOCI_SOVERSION}
+-      INSTALL_NAME_DIR ${CMAKE_INSTALL_PREFIX}/lib
+       CLEAN_DIRECT_OUTPUT 1)
+   endif()
+ 
+-- 
+2.25.1
+

external/soci/patches/0002-Fix-soci_backend.patch

@@ -0,0 +1,24 @@
+diff --git a/cmake/SociBackend.cmake b/cmake/SociBackend.cmake
+index 0a664667..3fa2ed95 100644
+--- a/cmake/SociBackend.cmake
++++ b/cmake/SociBackend.cmake
+@@ -31,14 +31,13 @@ macro(soci_backend_deps_found NAME DEPS SUCCESS)
+     if(NOT DEPEND_FOUND)
+       list(APPEND DEPS_NOT_FOUND ${dep})
+     else()
+-      string(TOUPPER "${dep}" DEPU)
+-      if( ${DEPU}_INCLUDE_DIR )
+-        list(APPEND DEPS_INCLUDE_DIRS ${${DEPU}_INCLUDE_DIR})
++      if( ${dep}_INCLUDE_DIR )
++        list(APPEND DEPS_INCLUDE_DIRS ${${dep}_INCLUDE_DIR})
+       endif()
+-      if( ${DEPU}_INCLUDE_DIRS )
+-        list(APPEND DEPS_INCLUDE_DIRS ${${DEPU}_INCLUDE_DIRS})
++      if( ${dep}_INCLUDE_DIRS )
++        list(APPEND DEPS_INCLUDE_DIRS ${${dep}_INCLUDE_DIRS})
+       endif()
+-      list(APPEND DEPS_LIBRARIES ${${DEPU}_LIBRARIES})
++      list(APPEND DEPS_LIBRARIES ${${dep}_LIBRARIES})
+     endif()
+   endforeach()
+