ARCHIVE/clio
1
0
Fork 0
mirror of https://github.com/XRPLF/clio.git synced 2025-12-05 08:48:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

log parser

Browse Source
This commit is contained in:
CJ Cobb
2021-03-18 13:03:33 -04:00
parent 8d021f2019
commit 4fc44c5ac8
3 changed files with 95 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
metrics.py Normal file
View File

@@ -0,0 +1,69 @@
#!/usr/bin/python3
import argparse
def parseLogs(filename):
with open(filename) as f:
totalTime = 0
totalTxns = 0
totalObjs = 0
milTime = 0
milTxns = 0
milObjs = 0
for line in f:
if "Load phase" in line:
sequenceIdx = line.find("Sequence : ")
hashIdx = line.find(" Hash :")
sequence = line[sequenceIdx + len("Sequence : "):hashIdx]
txnCountSubstr = "txn count = "
objCountSubstr = ". object count = "
loadTimeSubstr = ". load time = "
txnsSubstr = ". load txns per second = "
objsSubstr = ". load objs per second = "
txnCountIdx = line.find(txnCountSubstr)
objCountIdx = line.find(objCountSubstr)
loadTimeIdx = line.find(loadTimeSubstr)
txnsIdx = line.find(txnsSubstr)
objsIdx = line.find(objsSubstr)
txnCount = line[txnCountIdx + len(txnCountSubstr):objCountIdx]
objCount = line[objCountIdx + len(objCountSubstr):loadTimeIdx]
loadTime = line[loadTimeIdx + len(loadTimeSubstr):txnsIdx]
txnsPerSecond = line[txnsIdx + len(txnsSubstr):objsIdx]
objsPerSecond = line[objsIdx + len(objsSubstr):-1]
totalTime += float(loadTime);
totalTxns += float(txnCount)
totalObjs += float(objCount)
milTime += float(loadTime)
milTxns += float(txnCount)
milObjs += float(objCount)
if int(sequence) % 1000000 == 0:
print("This million: ")
print(str(milTxns/milTime) + " : " + str(milObjs/milTime))
milTime = 0
milTxns = 0
milObjs - 0
print("Sequence = " + sequence + " : [time, txCount, objCount, txPerSec, objsPerSec]")
print(loadTime + " : " + txnCount + " : " + objCount + " : " + txnsPerSecond + " : " + objsPerSecond)
print("Aggregate: [txPerSec, objsPerSec]")
print(str(totalTxns/totalTime) + " : " + str(totalObjs/totalTime))
print("Last million: [txPerSec, objPerSec]")
print(str(milTxns/milTime) + " : " + str(milObjs/milTime))
print("Totals : [txnPerSec, objPerSec]")
print(str(totalTxns/totalTime) + " : " + str(totalObjs/totalTime))
parser = argparse.ArgumentParser(description='parses logs')
parser.add_argument("--filename")
args = parser.parse_args()
def run(args):
parseLogs(args.filename)
run(args)

49
reporting/CassandraBackend.cpp
View File

@@ -2,7 +2,7 @@
namespace Backend {
template <class T, class F>
void
processAsyncWriteResponse(T&& requestParams, CassFuture* fut, F func)
processAsyncWriteResponse(T& requestParams, CassFuture* fut, F func)
{
CassandraBackend const& backend = *requestParams.backend;
auto rc = cass_future_error_code(fut);
@@ -40,11 +40,11 @@ flatMapWriteCallback(CassFuture* fut, void* cbData)
{
CassandraBackend::WriteCallbackData& requestParams =
*static_cast<CassandraBackend::WriteCallbackData*>(cbData);
auto func = [&requestParams](auto& params, bool retry) {
requestParams.backend->write(params, retry);
auto func = [](auto& params, bool retry) {
params.backend->write(params, retry);
};
processAsyncWriteResponse(std::move(requestParams), fut, func);
processAsyncWriteResponse(requestParams, fut, func);
}
void
@@ -52,10 +52,10 @@ flatMapWriteBookCallback(CassFuture* fut, void* cbData)
{
CassandraBackend::WriteCallbackData& requestParams =
*static_cast<CassandraBackend::WriteCallbackData*>(cbData);
auto func = [&requestParams](auto& params, bool retry) {
requestParams.backend->writeBook(params, retry);
auto func = [](auto& params, bool retry) {
params.backend->writeBook(params, retry);
};
processAsyncWriteResponse(std::move(requestParams), fut, func);
processAsyncWriteResponse(requestParams, fut, func);
}
/*
@@ -74,7 +74,7 @@ flatMapWriteKeyCallback(CassFuture* fut, void* cbData)
{
CassandraBackend::WriteCallbackData& requestParams =
*static_cast<CassandraBackend::WriteCallbackData*>(cbData);
processAsyncWriteResponse(std::move(requestParams), fut, retryWriteKey);
processAsyncWriteResponse(requestParams, fut, retryWriteKey);
}
void
@@ -85,6 +85,7 @@ flatMapGetCreatedCallback(CassFuture* fut, void* cbData)
CassandraBackend const& backend = *requestParams.backend;
auto rc = cass_future_error_code(fut);
if (rc != CASS_OK)
BOOST_LOG_TRIVIAL(info) << __func__;
{
BOOST_LOG_TRIVIAL(error)
<< "ERROR!!! Cassandra insert error: " << rc << ", "
@@ -129,30 +130,30 @@ flatMapWriteTransactionCallback(CassFuture* fut, void* cbData)
{
CassandraBackend::WriteTransactionCallbackData& requestParams =
*static_cast<CassandraBackend::WriteTransactionCallbackData*>(cbData);
auto func = [&requestParams](auto& params, bool retry) {
requestParams.backend->writeTransaction(params, retry);
auto func = [](auto& params, bool retry) {
params.backend->writeTransaction(params, retry);
};
processAsyncWriteResponse(std::move(requestParams), fut, func);
processAsyncWriteResponse(requestParams, fut, func);
}
void
flatMapWriteAccountTxCallback(CassFuture* fut, void* cbData)
{
CassandraBackend::WriteAccountTxCallbackData& requestParams =
*static_cast<CassandraBackend::WriteAccountTxCallbackData*>(cbData);
auto func = [&requestParams](auto& params, bool retry) {
requestParams.backend->writeAccountTx(params, retry);
auto func = [](auto& params, bool retry) {
params.backend->writeAccountTx(params, retry);
};
processAsyncWriteResponse(std::move(requestParams), fut, func);
processAsyncWriteResponse(requestParams, fut, func);
}
void
flatMapWriteLedgerHeaderCallback(CassFuture* fut, void* cbData)
{
CassandraBackend::WriteLedgerHeaderCallbackData& requestParams =
*static_cast<CassandraBackend::WriteLedgerHeaderCallbackData*>(cbData);
auto func = [&requestParams](auto& params, bool retry) {
requestParams.backend->writeLedgerHeader(params, retry);
auto func = [](auto& params, bool retry) {
params.backend->writeLedgerHeader(params, retry);
};
processAsyncWriteResponse(std::move(requestParams), fut, func);
processAsyncWriteResponse(requestParams, fut, func);
}
void
@@ -160,10 +161,10 @@ flatMapWriteLedgerHashCallback(CassFuture* fut, void* cbData)
{
CassandraBackend::WriteLedgerHashCallbackData& requestParams =
*static_cast<CassandraBackend::WriteLedgerHashCallbackData*>(cbData);
auto func = [&requestParams](auto& params, bool retry) {
requestParams.backend->writeLedgerHash(params, retry);
auto func = [](auto& params, bool retry) {
params.backend->writeLedgerHash(params, retry);
};
processAsyncWriteResponse(std::move(requestParams), fut, func);
processAsyncWriteResponse(requestParams, fut, func);
}
// Process the result of an asynchronous read. Retry on error
@@ -174,9 +175,7 @@ flatMapReadCallback(CassFuture* fut, void* cbData)
{
CassandraBackend::ReadCallbackData& requestParams =
*static_cast<CassandraBackend::ReadCallbackData*>(cbData);
auto func = [&requestParams](auto& params) {
requestParams.backend.read(params);
};
auto func = [](auto& params) { params.backend.read(params); };
CassandraAsyncResult asyncResult{requestParams, fut, func};
CassandraResult& result = asyncResult.getResult();
@@ -195,9 +194,7 @@ flatMapReadObjectCallback(CassFuture* fut, void* cbData)
{
CassandraBackend::ReadObjectCallbackData& requestParams =
*static_cast<CassandraBackend::ReadObjectCallbackData*>(cbData);
auto func = [&requestParams](auto& params) {
requestParams.backend.readObject(params);
};
auto func = [](auto& params) { params.backend.readObject(params); };
CassandraAsyncResult asyncResult{requestParams, fut, func};
CassandraResult& result = asyncResult.getResult();

3
reporting/CassandraBackend.h
View File

@@ -108,6 +108,7 @@ public:
~CassandraPreparedStatement()
{
BOOST_LOG_TRIVIAL(info) << __func__;
if (prepared_)
{
cass_prepared_free(prepared_);
@@ -288,6 +289,7 @@ public:
{
if (statement_)
cass_statement_free(statement_);
BOOST_LOG_TRIVIAL(info) << __func__;
}
};
@@ -620,6 +622,7 @@ public:
~CassandraBackend() override
{
BOOST_LOG_TRIVIAL(info) << __func__;
if (open_)
close();
}