Log detailed correlated consensus data together (#5302)

Combine multiple related debug log data points into a single
message. Allows quick correlation of events that
previously were either not logged or, if logged, strewn
across multiple lines, making correlation difficult.
The Heartbeat Timer and consensus ledger accept processing
each have this capability.

Also guarantees that log entries will be written if the
node is a validator, regardless of log severity level.
Otherwise, the level of these messages is at INFO severity.

src/test/beast/beast_Journal_test.cpp

@@ -53,6 +53,12 @@ public:
             if (level >= threshold())
                 ++m_count;
         }
+
+        void
+        writeAlways(severities::Severity level, std::string const&) override
+        {
+            ++m_count;
+        }
     };
 
     void

src/test/csf/Peer.h

@@ -116,6 +116,10 @@ struct Peer
         }
     };
 
+    class TestConsensusLogger
+    {
+    };
+
     /** Generic Validations adaptor that simply ignores recently stale
      * validations
      */
@@ -532,7 +536,8 @@ struct Peer
             closeResolution,
             rawCloseTimes,
             mode,
-            std::move(consensusJson));
+            std::move(consensusJson),
+            validating());
     }
 
     void
@@ -542,7 +547,8 @@ struct Peer
         NetClock::duration const& closeResolution,
         ConsensusCloseTimes const& rawCloseTimes,
         ConsensusMode const& mode,
-        Json::Value&& consensusJson)
+        Json::Value&& consensusJson,
+        const bool validating)
     {
         schedule(delays.ledgerAccept, [=, this]() {
             const bool proposing = mode == ConsensusMode::proposing;
@@ -877,6 +883,13 @@ struct Peer
     {
     }
 
+    bool
+    validating() const
+    {
+        // does not matter
+        return false;
+    }
+
     //--------------------------------------------------------------------------
     //  A locally submitted transaction
     void
@@ -917,7 +930,7 @@ struct Peer
         // Not yet modeling dynamic UNL.
         hash_set<PeerID> nowUntrusted;
         consensus.startRound(
-            now(), bestLCL, lastClosedLedger, nowUntrusted, runAsValidator);
+            now(), bestLCL, lastClosedLedger, nowUntrusted, runAsValidator, {});
     }
 
     // Start the consensus process assuming it is not yet running

src/test/csf/Sim.h

@@ -57,6 +57,14 @@ public:
         std::cout << clock_.now().time_since_epoch().count() << " " << text
                   << std::endl;
     }
+
+    void
+    writeAlways(beast::severities::Severity level, std::string const& text)
+        override
+    {
+        std::cout << clock_.now().time_since_epoch().count() << " " << text
+                  << std::endl;
+    }
 };
 
 class Sim

src/test/jtx/CaptureLogs.h

@@ -60,6 +60,14 @@ class CaptureLogs : public Logs
             std::lock_guard lock(strmMutex_);
             strm_ << text;
         }
+
+        void
+        writeAlways(beast::severities::Severity level, std::string const& text)
+            override
+        {
+            std::lock_guard lock(strmMutex_);
+            strm_ << text;
+        }
     };
 
 public:

src/test/jtx/CheckMessageLogs.h

@@ -48,6 +48,13 @@ class CheckMessageLogs : public Logs
             if (text.find(owner_.msg_) != std::string::npos)
                 *owner_.pFound_ = true;
         }
+
+        void
+        writeAlways(beast::severities::Severity level, std::string const& text)
+            override
+        {
+            write(level, text);
+        }
     };
 
 public:

src/test/server/ServerStatus_test.cpp

@@ -860,7 +860,7 @@ class ServerStatus_test : public beast::unit_test::suite,
 
         // mark the Network as having an Amendment Warning, but won't fail
         env.app().getOPs().setAmendmentWarned();
-        env.app().getOPs().beginConsensus(env.closed()->info().hash);
+        env.app().getOPs().beginConsensus(env.closed()->info().hash, {});
 
         // consensus doesn't change
         BEAST_EXPECT(
@@ -991,7 +991,7 @@ class ServerStatus_test : public beast::unit_test::suite,
         // mark the Network as Amendment Blocked, but still won't fail until
         // ELB is enabled (next step)
         env.app().getOPs().setAmendmentBlocked();
-        env.app().getOPs().beginConsensus(env.closed()->info().hash);
+        env.app().getOPs().beginConsensus(env.closed()->info().hash, {});
 
         // consensus now sees validation disabled
         BEAST_EXPECT(

src/test/server/Server_test.cpp

@@ -96,6 +96,13 @@ public:
 
             suite_.log << text << std::endl;
         }
+
+        void
+        writeAlways(beast::severities::Severity level, std::string const& text)
+            override
+        {
+            suite_.log << text << std::endl;
+        }
     };
 
     //--------------------------------------------------------------------------

src/test/unit_test/SuiteJournal.h

@@ -50,12 +50,26 @@ public:
 
     void
     write(beast::severities::Severity level, std::string const& text) override;
+
+    void
+    writeAlways(beast::severities::Severity level, std::string const& text)
+        override;
 };
 
 inline void
 SuiteJournalSink::write(
     beast::severities::Severity level,
     std::string const& text)
+{
+    // Only write the string if the level at least equals the threshold.
+    if (level >= threshold())
+        writeAlways(level, text);
+}
+
+inline void
+SuiteJournalSink::writeAlways(
+    beast::severities::Severity level,
+    std::string const& text)
 {
     using namespace beast::severities;
 
@@ -80,9 +94,7 @@ SuiteJournalSink::write(
         return "FTL:";
     }();
 
-    // Only write the string if the level at least equals the threshold.
-    if (level >= threshold())
-        suite_.log << s << partition_ << text << std::endl;
+    suite_.log << s << partition_ << text << std::endl;
 }
 
 class SuiteJournal
@@ -127,9 +139,16 @@ public:
     {
         if (level < threshold())
             return;
+        writeAlways(level, text);
+    }
 
+    inline void
+    writeAlways(beast::severities::Severity level, std::string const& text)
+        override
+    {
         strm_ << text << std::endl;
     }
+
     std::stringstream const&
     messages() const
     {