diff --git a/metrics.py b/metrics.py new file mode 100644 index 00000000..413ab938 --- /dev/null +++ b/metrics.py @@ -0,0 +1,69 @@ +#!/usr/bin/python3 +import argparse + +def parseLogs(filename): + + with open(filename) as f: + + totalTime = 0 + totalTxns = 0 + totalObjs = 0 + + milTime = 0 + milTxns = 0 + milObjs = 0 + + for line in f: + if "Load phase" in line: + sequenceIdx = line.find("Sequence : ") + hashIdx = line.find(" Hash :") + sequence = line[sequenceIdx + len("Sequence : "):hashIdx] + txnCountSubstr = "txn count = " + objCountSubstr = ". object count = " + loadTimeSubstr = ". load time = " + txnsSubstr = ". load txns per second = " + objsSubstr = ". load objs per second = " + txnCountIdx = line.find(txnCountSubstr) + objCountIdx = line.find(objCountSubstr) + loadTimeIdx = line.find(loadTimeSubstr) + txnsIdx = line.find(txnsSubstr) + objsIdx = line.find(objsSubstr) + txnCount = line[txnCountIdx + len(txnCountSubstr):objCountIdx] + objCount = line[objCountIdx + len(objCountSubstr):loadTimeIdx] + loadTime = line[loadTimeIdx + len(loadTimeSubstr):txnsIdx] + txnsPerSecond = line[txnsIdx + len(txnsSubstr):objsIdx] + objsPerSecond = line[objsIdx + len(objsSubstr):-1] + totalTime += float(loadTime); + totalTxns += float(txnCount) + totalObjs += float(objCount) + milTime += float(loadTime) + milTxns += float(txnCount) + milObjs += float(objCount) + if int(sequence) % 1000000 == 0: + print("This million: ") + print(str(milTxns/milTime) + " : " + str(milObjs/milTime)) + milTime = 0 + milTxns = 0 + milObjs - 0 + + print("Sequence = " + sequence + " : [time, txCount, objCount, txPerSec, objsPerSec]") + print(loadTime + " : " + txnCount + " : " + objCount + " : " + txnsPerSecond + " : " + objsPerSecond) + print("Aggregate: [txPerSec, objsPerSec]") + print(str(totalTxns/totalTime) + " : " + str(totalObjs/totalTime)) + + + print("Last million: [txPerSec, objPerSec]") + print(str(milTxns/milTime) + " : " + str(milObjs/milTime)) + print("Totals : [txnPerSec, objPerSec]") + print(str(totalTxns/totalTime) + " : " + str(totalObjs/totalTime)) + + +parser = argparse.ArgumentParser(description='parses logs') +parser.add_argument("--filename") + +args = parser.parse_args() + +def run(args): + parseLogs(args.filename) + +run(args) diff --git a/reporting/CassandraBackend.cpp b/reporting/CassandraBackend.cpp index e925ae0d..056aeb04 100644 --- a/reporting/CassandraBackend.cpp +++ b/reporting/CassandraBackend.cpp @@ -2,7 +2,7 @@ namespace Backend { template void -processAsyncWriteResponse(T&& requestParams, CassFuture* fut, F func) +processAsyncWriteResponse(T& requestParams, CassFuture* fut, F func) { CassandraBackend const& backend = *requestParams.backend; auto rc = cass_future_error_code(fut); @@ -40,11 +40,11 @@ flatMapWriteCallback(CassFuture* fut, void* cbData) { CassandraBackend::WriteCallbackData& requestParams = *static_cast(cbData); - auto func = [&requestParams](auto& params, bool retry) { - requestParams.backend->write(params, retry); + auto func = [](auto& params, bool retry) { + params.backend->write(params, retry); }; - processAsyncWriteResponse(std::move(requestParams), fut, func); + processAsyncWriteResponse(requestParams, fut, func); } void @@ -52,10 +52,10 @@ flatMapWriteBookCallback(CassFuture* fut, void* cbData) { CassandraBackend::WriteCallbackData& requestParams = *static_cast(cbData); - auto func = [&requestParams](auto& params, bool retry) { - requestParams.backend->writeBook(params, retry); + auto func = [](auto& params, bool retry) { + params.backend->writeBook(params, retry); }; - processAsyncWriteResponse(std::move(requestParams), fut, func); + processAsyncWriteResponse(requestParams, fut, func); } /* @@ -74,7 +74,7 @@ flatMapWriteKeyCallback(CassFuture* fut, void* cbData) { CassandraBackend::WriteCallbackData& requestParams = *static_cast(cbData); - processAsyncWriteResponse(std::move(requestParams), fut, retryWriteKey); + processAsyncWriteResponse(requestParams, fut, retryWriteKey); } void @@ -85,6 +85,7 @@ flatMapGetCreatedCallback(CassFuture* fut, void* cbData) CassandraBackend const& backend = *requestParams.backend; auto rc = cass_future_error_code(fut); if (rc != CASS_OK) + BOOST_LOG_TRIVIAL(info) << __func__; { BOOST_LOG_TRIVIAL(error) << "ERROR!!! Cassandra insert error: " << rc << ", " @@ -129,30 +130,30 @@ flatMapWriteTransactionCallback(CassFuture* fut, void* cbData) { CassandraBackend::WriteTransactionCallbackData& requestParams = *static_cast(cbData); - auto func = [&requestParams](auto& params, bool retry) { - requestParams.backend->writeTransaction(params, retry); + auto func = [](auto& params, bool retry) { + params.backend->writeTransaction(params, retry); }; - processAsyncWriteResponse(std::move(requestParams), fut, func); + processAsyncWriteResponse(requestParams, fut, func); } void flatMapWriteAccountTxCallback(CassFuture* fut, void* cbData) { CassandraBackend::WriteAccountTxCallbackData& requestParams = *static_cast(cbData); - auto func = [&requestParams](auto& params, bool retry) { - requestParams.backend->writeAccountTx(params, retry); + auto func = [](auto& params, bool retry) { + params.backend->writeAccountTx(params, retry); }; - processAsyncWriteResponse(std::move(requestParams), fut, func); + processAsyncWriteResponse(requestParams, fut, func); } void flatMapWriteLedgerHeaderCallback(CassFuture* fut, void* cbData) { CassandraBackend::WriteLedgerHeaderCallbackData& requestParams = *static_cast(cbData); - auto func = [&requestParams](auto& params, bool retry) { - requestParams.backend->writeLedgerHeader(params, retry); + auto func = [](auto& params, bool retry) { + params.backend->writeLedgerHeader(params, retry); }; - processAsyncWriteResponse(std::move(requestParams), fut, func); + processAsyncWriteResponse(requestParams, fut, func); } void @@ -160,10 +161,10 @@ flatMapWriteLedgerHashCallback(CassFuture* fut, void* cbData) { CassandraBackend::WriteLedgerHashCallbackData& requestParams = *static_cast(cbData); - auto func = [&requestParams](auto& params, bool retry) { - requestParams.backend->writeLedgerHash(params, retry); + auto func = [](auto& params, bool retry) { + params.backend->writeLedgerHash(params, retry); }; - processAsyncWriteResponse(std::move(requestParams), fut, func); + processAsyncWriteResponse(requestParams, fut, func); } // Process the result of an asynchronous read. Retry on error @@ -174,9 +175,7 @@ flatMapReadCallback(CassFuture* fut, void* cbData) { CassandraBackend::ReadCallbackData& requestParams = *static_cast(cbData); - auto func = [&requestParams](auto& params) { - requestParams.backend.read(params); - }; + auto func = [](auto& params) { params.backend.read(params); }; CassandraAsyncResult asyncResult{requestParams, fut, func}; CassandraResult& result = asyncResult.getResult(); @@ -195,9 +194,7 @@ flatMapReadObjectCallback(CassFuture* fut, void* cbData) { CassandraBackend::ReadObjectCallbackData& requestParams = *static_cast(cbData); - auto func = [&requestParams](auto& params) { - requestParams.backend.readObject(params); - }; + auto func = [](auto& params) { params.backend.readObject(params); }; CassandraAsyncResult asyncResult{requestParams, fut, func}; CassandraResult& result = asyncResult.getResult(); diff --git a/reporting/CassandraBackend.h b/reporting/CassandraBackend.h index 9520d2a4..259a6401 100644 --- a/reporting/CassandraBackend.h +++ b/reporting/CassandraBackend.h @@ -108,6 +108,7 @@ public: ~CassandraPreparedStatement() { + BOOST_LOG_TRIVIAL(info) << __func__; if (prepared_) { cass_prepared_free(prepared_); @@ -288,6 +289,7 @@ public: { if (statement_) cass_statement_free(statement_); + BOOST_LOG_TRIVIAL(info) << __func__; } }; @@ -620,6 +622,7 @@ public: ~CassandraBackend() override { + BOOST_LOG_TRIVIAL(info) << __func__; if (open_) close(); }