fix: Handle logger exceptions (#2349)

src/app/ClioApplication.cpp

@@ -41,7 +41,6 @@
 #include "util/build/Build.hpp"
 #include "util/config/ConfigDefinition.hpp"
 #include "util/log/Logger.hpp"
-#include "util/prometheus/Prometheus.hpp"
 #include "web/AdminVerificationStrategy.hpp"
 #include "web/RPCServerHandler.hpp"
 #include "web/Server.hpp"
@@ -91,7 +90,6 @@ ClioApplication::ClioApplication(util::config::ClioConfigDefinition const& confi
     : config_(config), signalsHandler_{config_}
 {
     LOG(util::LogService::info()) << "Clio version: " << util::build::getClioFullVersionString();
-    PrometheusService::init(config);
     signalsHandler_.subscribeToStop([this]() { appStopper_.stop(); });
 }
 

src/main/Main.cpp

@@ -25,6 +25,7 @@
 #include "util/TerminationHandler.hpp"
 #include "util/config/ConfigDefinition.hpp"
 #include "util/log/Logger.hpp"
+#include "util/prometheus/Prometheus.hpp"
 
 #include <cstdlib>
 #include <exception>
@@ -52,6 +53,7 @@ try {
             if (not app::parseConfig(run.configPath))
                 return EXIT_FAILURE;
 
+            PrometheusService::init(gClioConfig);
             if (auto const initSuccess = util::LogService::init(gClioConfig); not initSuccess) {
                 std::cerr << initSuccess.error() << std::endl;
                 return EXIT_FAILURE;

src/util/log/Logger.cpp

@@ -25,6 +25,9 @@
 #include "util/config/ArrayView.hpp"
 #include "util/config/ConfigDefinition.hpp"
 #include "util/config/ObjectView.hpp"
+#include "util/prometheus/Counter.hpp"
+#include "util/prometheus/Label.hpp"
+#include "util/prometheus/Prometheus.hpp"
 
 #include <boost/algorithm/string/predicate.hpp>
 #include <boost/date_time/posix_time/posix_time_duration.hpp>
@@ -42,6 +45,7 @@
 #include <boost/log/keywords/target_file_name.hpp>
 #include <boost/log/keywords/time_based_rotation.hpp>
 #include <boost/log/sinks/text_file_backend.hpp>
+#include <boost/log/utility/exception_handler.hpp>
 #include <boost/log/utility/setup/common_attributes.hpp>
 #include <boost/log/utility/setup/console.hpp>
 #include <boost/log/utility/setup/file.hpp>
@@ -52,7 +56,9 @@
 #include <array>
 #include <cstddef>
 #include <cstdint>
+#include <exception>
 #include <filesystem>
+#include <functional>
 #include <ios>
 #include <iostream>
 #include <optional>
@@ -65,6 +71,30 @@
 
 namespace util {
 
+namespace {
+
+class LoggerExceptionHandler {
+    std::reference_wrapper<util::prometheus::CounterInt> exceptionCounter_ =
+        PrometheusService::counterInt("logger_exceptions_total_number", util::prometheus::Labels{});
+
+public:
+    using result_type = void;
+
+    LoggerExceptionHandler()
+    {
+        ASSERT(PrometheusService::isInitialised(), "Prometheus should be initialised before Logger");
+    }
+
+    void
+    operator()(std::exception const& e) const
+    {
+        std::cerr << fmt::format("Exception in logger: {}\n", e.what());
+        ++exceptionCounter_.get();
+    }
+};
+
+}  // namespace
+
 Logger LogService::generalLog = Logger{"General"};
 Logger LogService::alertLog = Logger{"Alert"};
 boost::log::filter LogService::filter{};
@@ -160,6 +190,10 @@ LogService::init(config::ClioConfigDefinition const& config)
             sinks::file::make_collector(keywords::target = dirPath, keywords::max_size = dirSize)
         );
         fileSink->locked_backend()->scan_for_files();
+
+        boost::log::core::get()->set_exception_handler(
+            boost::log::make_exception_handler<std::exception>(LoggerExceptionHandler())
+        );
     }
 
     // get default severity, can be overridden per channel using the `log_channels` array

src/util/prometheus/Prometheus.cpp

@@ -184,6 +184,12 @@ PrometheusService::init(util::config::ClioConfigDefinition const& config)
     impl = std::make_unique<util::prometheus::PrometheusImpl>(enabled, compressReply);
 }
 
+bool
+PrometheusService::isInitialised()
+{
+    return impl != nullptr;
+}
+
 util::prometheus::Bool
 PrometheusService::boolMetric(std::string name, util::prometheus::Labels labels, std::optional<std::string> description)
 {
@@ -271,7 +277,7 @@ PrometheusService::replaceInstance(std::unique_ptr<util::prometheus::PrometheusI
 util::prometheus::PrometheusInterface&
 PrometheusService::instance()
 {
-    ASSERT(impl != nullptr, "PrometheusService::instance() called before init()");
+    ASSERT(isInitialised(), "PrometheusService::instance() called before init()");
     return *impl;
 }
 

src/util/prometheus/Prometheus.hpp

@@ -260,6 +260,14 @@ public:
     static void
     init(util::config::ClioConfigDefinition const& config);
 
+    /**
+     * @brief Whether the singleton has been already initialised
+     *
+     * @return True if the singleton was already initialised and false otherwise
+     */
+    static bool
+    isInitialised();
+
     /**
      * @brief Get a bool based metric. It will be created if it doesn't exist
      * @note Prometheus does not have a native bool type, so we use a counter with a value of 0 or 1