log parser

metrics.py

@@ -0,0 +1,69 @@
+#!/usr/bin/python3
+import argparse
+
+def parseLogs(filename):
+
+    with open(filename) as f:
+
+        totalTime = 0
+        totalTxns = 0
+        totalObjs = 0
+        
+        milTime = 0
+        milTxns = 0
+        milObjs = 0
+
+        for line in f:
+            if "Load phase" in line:
+                sequenceIdx = line.find("Sequence : ")
+                hashIdx =  line.find(" Hash :")
+                sequence = line[sequenceIdx + len("Sequence : "):hashIdx]
+                txnCountSubstr = "txn count = "
+                objCountSubstr = ". object count = "
+                loadTimeSubstr = ". load time = "
+                txnsSubstr = ". load txns per second = "
+                objsSubstr = ". load objs per second = "
+                txnCountIdx = line.find(txnCountSubstr)
+                objCountIdx = line.find(objCountSubstr)
+                loadTimeIdx = line.find(loadTimeSubstr)
+                txnsIdx = line.find(txnsSubstr)
+                objsIdx = line.find(objsSubstr)
+                txnCount = line[txnCountIdx + len(txnCountSubstr):objCountIdx]
+                objCount = line[objCountIdx + len(objCountSubstr):loadTimeIdx]
+                loadTime = line[loadTimeIdx + len(loadTimeSubstr):txnsIdx]
+                txnsPerSecond = line[txnsIdx + len(txnsSubstr):objsIdx]
+                objsPerSecond = line[objsIdx + len(objsSubstr):-1]
+                totalTime += float(loadTime);
+                totalTxns += float(txnCount)
+                totalObjs += float(objCount)
+                milTime += float(loadTime)
+                milTxns += float(txnCount)
+                milObjs += float(objCount)
+                if int(sequence) % 1000000 == 0:
+                    print("This million: ")
+                    print(str(milTxns/milTime) + " : " + str(milObjs/milTime))
+                    milTime = 0
+                    milTxns = 0
+                    milObjs - 0
+
+                print("Sequence = " + sequence + " : [time, txCount, objCount, txPerSec, objsPerSec]")
+                print(loadTime + " : " + txnCount + " : " + objCount + " : " + txnsPerSecond + " : " + objsPerSecond)
+                print("Aggregate: [txPerSec, objsPerSec]")
+                print(str(totalTxns/totalTime) + " : " + str(totalObjs/totalTime))
+
+
+        print("Last million: [txPerSec, objPerSec]")
+        print(str(milTxns/milTime) + " : " + str(milObjs/milTime))
+        print("Totals : [txnPerSec, objPerSec]")
+        print(str(totalTxns/totalTime) + " : " + str(totalObjs/totalTime))
+    
+
+parser = argparse.ArgumentParser(description='parses logs')
+parser.add_argument("--filename")
+
+args = parser.parse_args()
+
+def run(args):
+    parseLogs(args.filename)
+
+run(args)

reporting/CassandraBackend.cpp

@@ -2,7 +2,7 @@
 namespace Backend {
 template <class T, class F>
 void
-processAsyncWriteResponse(T&& requestParams, CassFuture* fut, F func)
+processAsyncWriteResponse(T& requestParams, CassFuture* fut, F func)
 {
     CassandraBackend const& backend = *requestParams.backend;
     auto rc = cass_future_error_code(fut);
@@ -40,11 +40,11 @@ flatMapWriteCallback(CassFuture* fut, void* cbData)
 {
     CassandraBackend::WriteCallbackData& requestParams =
         *static_cast<CassandraBackend::WriteCallbackData*>(cbData);
-    auto func = [&requestParams](auto& params, bool retry) {
+    auto func = [](auto& params, bool retry) {
-        requestParams.backend->write(params, retry);
+        params.backend->write(params, retry);
     };
 
-    processAsyncWriteResponse(std::move(requestParams), fut, func);
+    processAsyncWriteResponse(requestParams, fut, func);
 }
 
 void
@@ -52,10 +52,10 @@ flatMapWriteBookCallback(CassFuture* fut, void* cbData)
 {
     CassandraBackend::WriteCallbackData& requestParams =
         *static_cast<CassandraBackend::WriteCallbackData*>(cbData);
-    auto func = [&requestParams](auto& params, bool retry) {
+    auto func = [](auto& params, bool retry) {
-        requestParams.backend->writeBook(params, retry);
+        params.backend->writeBook(params, retry);
     };
-    processAsyncWriteResponse(std::move(requestParams), fut, func);
+    processAsyncWriteResponse(requestParams, fut, func);
 }
 /*
 
@@ -74,7 +74,7 @@ flatMapWriteKeyCallback(CassFuture* fut, void* cbData)
 {
     CassandraBackend::WriteCallbackData& requestParams =
         *static_cast<CassandraBackend::WriteCallbackData*>(cbData);
-    processAsyncWriteResponse(std::move(requestParams), fut, retryWriteKey);
+    processAsyncWriteResponse(requestParams, fut, retryWriteKey);
 }
 
 void
@@ -85,6 +85,7 @@ flatMapGetCreatedCallback(CassFuture* fut, void* cbData)
     CassandraBackend const& backend = *requestParams.backend;
     auto rc = cass_future_error_code(fut);
     if (rc != CASS_OK)
+        BOOST_LOG_TRIVIAL(info) << __func__;
     {
         BOOST_LOG_TRIVIAL(error)
             << "ERROR!!! Cassandra insert error: " << rc << ", "
@@ -129,30 +130,30 @@ flatMapWriteTransactionCallback(CassFuture* fut, void* cbData)
 {
     CassandraBackend::WriteTransactionCallbackData& requestParams =
         *static_cast<CassandraBackend::WriteTransactionCallbackData*>(cbData);
-    auto func = [&requestParams](auto& params, bool retry) {
+    auto func = [](auto& params, bool retry) {
-        requestParams.backend->writeTransaction(params, retry);
+        params.backend->writeTransaction(params, retry);
     };
-    processAsyncWriteResponse(std::move(requestParams), fut, func);
+    processAsyncWriteResponse(requestParams, fut, func);
 }
 void
 flatMapWriteAccountTxCallback(CassFuture* fut, void* cbData)
 {
     CassandraBackend::WriteAccountTxCallbackData& requestParams =
         *static_cast<CassandraBackend::WriteAccountTxCallbackData*>(cbData);
-    auto func = [&requestParams](auto& params, bool retry) {
+    auto func = [](auto& params, bool retry) {
-        requestParams.backend->writeAccountTx(params, retry);
+        params.backend->writeAccountTx(params, retry);
     };
-    processAsyncWriteResponse(std::move(requestParams), fut, func);
+    processAsyncWriteResponse(requestParams, fut, func);
 }
 void
 flatMapWriteLedgerHeaderCallback(CassFuture* fut, void* cbData)
 {
     CassandraBackend::WriteLedgerHeaderCallbackData& requestParams =
         *static_cast<CassandraBackend::WriteLedgerHeaderCallbackData*>(cbData);
-    auto func = [&requestParams](auto& params, bool retry) {
+    auto func = [](auto& params, bool retry) {
-        requestParams.backend->writeLedgerHeader(params, retry);
+        params.backend->writeLedgerHeader(params, retry);
     };
-    processAsyncWriteResponse(std::move(requestParams), fut, func);
+    processAsyncWriteResponse(requestParams, fut, func);
 }
 
 void
@@ -160,10 +161,10 @@ flatMapWriteLedgerHashCallback(CassFuture* fut, void* cbData)
 {
     CassandraBackend::WriteLedgerHashCallbackData& requestParams =
         *static_cast<CassandraBackend::WriteLedgerHashCallbackData*>(cbData);
-    auto func = [&requestParams](auto& params, bool retry) {
+    auto func = [](auto& params, bool retry) {
-        requestParams.backend->writeLedgerHash(params, retry);
+        params.backend->writeLedgerHash(params, retry);
     };
-    processAsyncWriteResponse(std::move(requestParams), fut, func);
+    processAsyncWriteResponse(requestParams, fut, func);
 }
 
 // Process the result of an asynchronous read. Retry on error
@@ -174,9 +175,7 @@ flatMapReadCallback(CassFuture* fut, void* cbData)
 {
     CassandraBackend::ReadCallbackData& requestParams =
         *static_cast<CassandraBackend::ReadCallbackData*>(cbData);
-    auto func = [&requestParams](auto& params) {
+    auto func = [](auto& params) { params.backend.read(params); };
-        requestParams.backend.read(params);
-    };
     CassandraAsyncResult asyncResult{requestParams, fut, func};
     CassandraResult& result = asyncResult.getResult();
 
@@ -195,9 +194,7 @@ flatMapReadObjectCallback(CassFuture* fut, void* cbData)
 {
     CassandraBackend::ReadObjectCallbackData& requestParams =
         *static_cast<CassandraBackend::ReadObjectCallbackData*>(cbData);
-    auto func = [&requestParams](auto& params) {
+    auto func = [](auto& params) { params.backend.readObject(params); };
-        requestParams.backend.readObject(params);
-    };
     CassandraAsyncResult asyncResult{requestParams, fut, func};
     CassandraResult& result = asyncResult.getResult();
 

reporting/CassandraBackend.h

@@ -108,6 +108,7 @@ public:
 
     ~CassandraPreparedStatement()
     {
+        BOOST_LOG_TRIVIAL(info) << __func__;
         if (prepared_)
         {
             cass_prepared_free(prepared_);
@@ -288,6 +289,7 @@ public:
     {
         if (statement_)
             cass_statement_free(statement_);
+        BOOST_LOG_TRIVIAL(info) << __func__;
     }
 };
 
@@ -620,6 +622,7 @@ public:
 
     ~CassandraBackend() override
     {
+        BOOST_LOG_TRIVIAL(info) << __func__;
         if (open_)
             close();
     }