Script to generate reports from logs, and bug fixes:

* log_report.py is a script to generate debugging reports and combine the logs
of the locally run mainchain and sidechain servers.

* Log address book before pytest start

* Cleanup test utils

* Modify log_analyzer so joins all logs into a single file

* Organize "all" log as a dictionary

* Allow ConfigFile and Section classes to be pickled:
This caused a bug on mac platforms. Linux did not appear to use pickle.

* Add account history command to py scripts

* Add additional logging

* Add support to run sidechains under rr:

This is an undocumented feature to help debugging.
If the environment variable `RIPPLED_SIDECHAIN_RR` is set, it is assumed to
point to the rr executable. Sidechain 0 will then be run under rr.

bin/sidechain/python/app.py

@@ -1,5 +1,6 @@
 from contextlib import contextmanager
 import json
+import logging
 import os
 import pandas as pd
 from pathlib import Path
@@ -159,11 +160,16 @@ class App:
             raise ValueError('Cannot sign transaction without secret key')
         r = self(Sign(txn.account.secret_key, txn.to_cmd_obj()))
         raw_signed = r['tx_blob']
-        return self(Submit(raw_signed))
+        r = self(Submit(raw_signed))
+        logging.info(f'App.send_signed: {json.dumps(r, indent=1)}')
+        return r
 
     def send_command(self, cmd: Command) -> dict:
         '''Send the command to the rippled server'''
-        return self.client.send_command(cmd)
+        r = self.client.send_command(cmd)
+        logging.info(
+            f'App.send_command : {cmd.cmd_name()} : {json.dumps(r, indent=1)}')
+        return r
 
     # Need async version to close ledgers from async functions
     async def async_send_command(self, cmd: Command) -> dict:
@@ -589,12 +595,17 @@ def configs_for_testnet(config_file_prefix: str) -> List[ConfigFile]:
 
 
 # Start an app for a network with the config files matched by `config_file_prefix*/rippled.cfg`
+
+
+# Undocumented feature: if the environment variable RIPPLED_SIDECHAIN_RR is set, it is
+# assumed to point to the rr executable. Sidechain server 0 will then be run under rr.
 @contextmanager
 def testnet_app(*,
                 exe: str,
                 configs: List[ConfigFile],
                 command_logs: Optional[List[str]] = None,
                 run_server: Optional[List[bool]] = None,
+                sidechain_rr: Optional[str] = None,
                 extra_args: Optional[List[List[str]]] = None):
     '''Start a ripple testnet and return an app'''
     try:
@@ -603,6 +614,7 @@ def testnet_app(*,
                                   configs,
                                   command_logs=command_logs,
                                   run_server=run_server,
+                                  with_rr=sidechain_rr,
                                   extra_args=extra_args)
         network.wait_for_validated_ledger()
         app = App(network=network, standalone=False)

bin/sidechain/python/command.py

@@ -516,6 +516,7 @@ class Subscribe(SubscriptionCommand):
             streams: Optional[List[str]] = None,
             accounts: Optional[List[Account]] = None,
             accounts_proposed: Optional[List[Account]] = None,
+            account_history_account: Optional[Account] = None,
             books: Optional[
                 List[BookSubscription]] = None,  # taker_pays, taker_gets
             url: Optional[str] = None,
@@ -524,6 +525,7 @@ class Subscribe(SubscriptionCommand):
         super().__init__()
         self.streams = streams
         self.accounts = accounts
+        self.account_history_account = account_history_account
         self.accounts_proposed = accounts_proposed
         self.books = books
         self.url = url
@@ -542,6 +544,10 @@ class Subscribe(SubscriptionCommand):
             d['streams'] = self.streams
         if self.accounts is not None:
             d['accounts'] = [a.account_id for a in self.accounts]
+        if self.account_history_account is not None:
+            d['account_history_tx_stream'] = {
+                'account': self.account_history_account.account_id
+            }
         if self.accounts_proposed is not None:
             d['accounts_proposed'] = [
                 a.account_id for a in self.accounts_proposed

bin/sidechain/python/common.py

@@ -1,5 +1,6 @@
 import binascii
 import datetime
+import logging
 from typing import List, Optional, Union
 import pandas as pd
 import pytz
@@ -21,6 +22,7 @@ def enable_eprint():
 def eprint(*args, **kwargs):
     if not EPRINT_ENABLED:
         return
+    logging.error(*args)
     print(*args, file=sys.stderr, flush=True, **kwargs)
 
 
@@ -191,6 +193,10 @@ def XRP(v: Union[int, float]) -> Asset:
     return Asset(value=v * 1_000_000)
 
 
+def drops(v: int) -> Asset:
+    return Asset(value=v)
+
+
 class Path:
     '''Payment Path'''
     def __init__(self,

bin/sidechain/python/config_file.py

@@ -36,7 +36,10 @@ class Section:
         return None
 
     def __getattr__(self, name):
-        return self._kv_pairs[name]
+        try:
+            return self._kv_pairs[name]
+        except KeyError:
+            raise AttributeError(name)
 
     def __setattr__(self, name, value):
         if name in self.__dict__:
@@ -44,6 +47,12 @@ class Section:
         else:
             self._kv_pairs[name] = value
 
+    def __getstate__(self):
+        return vars(self)
+
+    def __setstate__(self, state):
+        vars(self).update(state)
+
 
 class ConfigFile:
     def __init__(self, *, file_name: Optional[str] = None):
@@ -79,5 +88,14 @@ class ConfigFile:
     def get_file_name(self):
         return self._file_name
 
+    def __getstate__(self):
+        return vars(self)
+
+    def __setstate__(self, state):
+        vars(self).update(state)
+
     def __getattr__(self, name):
-        return self._sections[name]
+        try:
+            return self._sections[name]
+        except KeyError:
+            raise AttributeError(name)

bin/sidechain/python/log_analyzer.py

@@ -2,9 +2,12 @@
 
 import argparse
 import json
+import os
 import re
+import sys
 
 from common import eprint
+from typing import IO, Optional
 
 
 class LogLine:
@@ -49,7 +52,7 @@ class LogLine:
         except Exception as e:
             eprint(f'init exception: {e} line: {line}')
 
-    def to_mixed_json(self) -> str:
+    def to_mixed_json_str(self) -> str:
         '''
         return a pretty printed string as mixed json
         '''
@@ -62,61 +65,105 @@ class LogLine:
             eprint(f'Using raw line: {self.raw_line}')
             return self.raw_line
 
-    def to_pure_json(self) -> str:
+    def to_pure_json(self) -> dict:
+        '''
+        return a json dict
+        '''
+        dict = {}
+        dict['t'] = self.timestamp
+        dict['m'] = self.module
+        dict['l'] = self.level
+        dict['msg'] = self.msg
+        if self.json_data:
+            dict['data'] = self.json_data
+        return dict
+
+    def to_pure_json_str(self, f_id: Optional[str] = None) -> str:
         '''
         return a pretty printed string as pure json
         '''
         try:
-            dict = {}
-            dict['t'] = self.timestamp
-            dict['m'] = self.module
-            dict['l'] = self.level
-            dict['msg'] = self.msg
-            if self.json_data:
-                dict['data'] = self.json_data
+            dict = self.to_pure_json(f_id)
             return json.dumps(dict, indent=1)
         except:
             return self.raw_line
 
 
-def convert_log(in_file_name: str, out_file_name: str, *, pure_json=False):
+def convert_log(in_file_name: str,
+                out: str,
+                *,
+                as_list=False,
+                pure_json=False,
+                module: Optional[str] = 'SidechainFederator') -> list:
+    result = []
     try:
         prev_lines = None
         with open(in_file_name) as input:
-            with open(out_file_name, "w") as out:
-                for l in input:
-                    l = l.strip()
-                    if not l:
-                        continue
-                    if LogLine.UNSTRUCTURED_RE.match(l):
-                        if prev_lines:
-                            log_line = LogLine(prev_lines)
-                            if log_line.module == 'SidechainFederator':
+            for l in input:
+                l = l.strip()
+                if not l:
+                    continue
+                if LogLine.UNSTRUCTURED_RE.match(l):
+                    if prev_lines:
+                        log_line = LogLine(prev_lines)
+                        if not module or log_line.module == module:
+                            if as_list:
+                                result.append(log_line.to_pure_json())
+                            else:
                                 if pure_json:
-                                    print(log_line.to_pure_json(), file=out)
+                                    print(log_line.to_pure_json_str(),
+                                          file=out)
                                 else:
-                                    print(log_line.to_mixed_json(), file=out)
-                        prev_lines = l
+                                    print(log_line.to_mixed_json_str(),
+                                          file=out)
+                    prev_lines = l
+                else:
+                    if not prev_lines:
+                        eprint(f'Error: Expected prev_lines. Cur line: {l}')
+                    assert prev_lines
+                    prev_lines += f' {l}'
+            if prev_lines:
+                log_line = LogLine(prev_lines)
+                if not module or log_line.module == module:
+                    if as_list:
+                        result.append(log_line.to_pure_json())
                     else:
-                        if not prev_lines:
-                            eprint(
-                                f'Error: Expected prev_lines. Cur line: {l}')
-                        assert prev_lines
-                        prev_lines += f' {l}'
-                if prev_lines:
-                    log_line = LogLine(prev_lines)
-                    if log_line.module == 'SidechainFederator':
                         if pure_json:
-                            print(log_line.to_pure_json(),
+                            print(log_line.to_pure_json_str(f_id),
                                   file=out,
                                   flush=True)
                         else:
-                            print(log_line.to_mixed_json(),
+                            print(log_line.to_mixed_json_str(),
                                   file=out,
                                   flush=True)
     except Exception as e:
         eprint(f'Excption: {e}')
         raise e
+    return result
+
+
+def convert_all(in_dir_name: str, out: IO, *, pure_json=False):
+    '''
+    Convert all the "debug.log" log files in one directory level below the in_dir_name into a single json file.
+    There will be a field called 'f' for the director name that the origional log file came from.
+    This is useful when analyzing networks that run on the local machine.
+    '''
+    if not os.path.isdir(in_dir_name):
+        print(f'Error: {in_dir_name} is not a directory')
+    files = []
+    f_ids = []
+    for subdir in os.listdir(in_dir_name):
+        file = f'{in_dir_name}/{subdir}/debug.log'
+        if not os.path.isfile(file):
+            continue
+        files.append(file)
+        f_ids.append(subdir)
+
+    result = {}
+    for f, f_id in zip(files, f_ids):
+        l = convert_log(f, out, as_list=True, pure_json=pure_json, module=None)
+        result[f_id] = l
+    print(json.dumps(result, indent=1), file=out, flush=True)
 
 
 def parse_args():
@@ -126,7 +173,7 @@ def parse_args():
     parser.add_argument(
         '--input',
         '-i',
-        help=('input log file'),
+        help=('input log file or sidechain config directory structure'),
     )
 
     parser.add_argument(
@@ -139,5 +186,13 @@ def parse_args():
 
 
 if __name__ == '__main__':
-    args = parse_args()
-    convert_log(args.input, args.output, pure_json=True)
+    try:
+        args = parse_args()
+        with open(args.output, "w") as out:
+            if os.path.isdir(args.input):
+                convert_all(args.input, out, pure_json=True)
+            else:
+                convert_log(args.input, out, pure_json=True)
+    except Exception as e:
+        eprint(f'Excption: {e}')
+        raise e

bin/sidechain/python/log_report.py

@@ -0,0 +1,285 @@
+#!/usr/bin/env python3
+
+import argparse
+from collections import defaultdict
+import datetime
+import json
+import numpy as np
+import os
+import pandas as pd
+import string
+import sys
+from typing import Dict, Set
+
+from common import eprint
+import log_analyzer
+
+
+def _has_256bit_hex_field_other(data, result: Set[str]):
+    return
+
+
+_has_256bit_hex_field_overloads = defaultdict(
+    lambda: _has_256bit_hex_field_other)
+
+
+def _has_256bit_hex_field_str(data: str, result: Set[str]):
+    if len(data) != 64:
+        return
+    for c in data:
+        o = ord(c.upper())
+        if ord('A') <= o <= ord('F'):
+            continue
+        if ord('0') <= o <= ord('9'):
+            continue
+        return
+    result.add(data)
+
+
+_has_256bit_hex_field_overloads[str] = _has_256bit_hex_field_str
+
+
+def _has_256bit_hex_field_dict(data: dict, result: Set[str]):
+    for k, v in data.items():
+        if k in [
+                "meta", "index", "LedgerIndex", "ledger_index", "ledger_hash",
+                "SigningPubKey", "suppression"
+        ]:
+            continue
+        _has_256bit_hex_field_overloads[type(v)](v, result)
+
+
+_has_256bit_hex_field_overloads[dict] = _has_256bit_hex_field_dict
+
+
+def _has_256bit_hex_field_list(data: list, result: Set[str]):
+    for v in data:
+        _has_256bit_hex_field_overloads[type(v)](v, result)
+
+
+_has_256bit_hex_field_overloads[list] = _has_256bit_hex_field_list
+
+
+def has_256bit_hex_field(data: dict) -> Set[str]:
+    '''
+    Find all the fields that are strings 64 chars long with only hex digits
+    This is useful when grouping transactions by hex
+    '''
+    result = set()
+    _has_256bit_hex_field_dict(data, result)
+    return result
+
+
+def group_by_txn(data: dict) -> dict:
+    '''
+    return a dictionary where the key is the transaction hash, the value is another dictionary.
+    The second dictionary the key is the server id, and the values are a list of log items
+    '''
+    def _make_default():
+        return defaultdict(lambda: list())
+
+    result = defaultdict(_make_default)
+    for server_id, log_list in data.items():
+        for log_item in log_list:
+            if txn_hashes := has_256bit_hex_field(log_item):
+                for h in txn_hashes:
+                    result[h][server_id].append(log_item)
+    return result
+
+
+def _rekey_dict_by_txn_date(hash_to_timestamp: dict,
+                            grouped_by_txn: dict) -> dict:
+    '''
+    hash_to_timestamp is a dictionary with a key of the txn hash and a value of the timestamp.
+    grouped_by_txn is a dictionary with a key of the txn and an unspecified value.
+    the keys in hash_to_timestamp are a superset of the keys in grouped_by_txn
+    This function returns a new grouped_by_txn dictionary with the transactions sorted by date.
+    '''
+    known_txns = [
+        k for k, v in sorted(hash_to_timestamp.items(), key=lambda x: x[1])
+    ]
+    result = {}
+    for k, v in grouped_by_txn.items():
+        if k not in known_txns:
+            result[k] = v
+    for h in known_txns:
+        result[h] = grouped_by_txn[h]
+    return result
+
+
+def _to_timestamp(str_time: str) -> datetime.datetime:
+    return datetime.datetime.strptime(
+        str_time.split('.')[0], "%Y-%b-%d %H:%M:%S")
+
+
+class Report:
+    def __init__(self, in_dir, out_dir):
+        self.in_dir = in_dir
+        self.out_dir = out_dir
+
+        self.combined_logs_file_name = f'{self.out_dir}/combined_logs.json'
+        self.grouped_by_txn_file_name = f'{self.out_dir}/grouped_by_txn.json'
+        self.counts_by_txn_and_server_file_name = f'{self.out_dir}/counts_by_txn_and_server.org'
+        self.data = None  # combined logs
+
+        # grouped_by_txn is a dictionary where the key is the server id. mainchain servers
+        # have a key of `mainchain_#` and sidechain servers have a key of
+        # `sidechain_#`, where `#` is a number.
+        self.grouped_by_txn = None
+
+        if not os.path.isdir(in_dir):
+            eprint(f'The input {self.in_dir} must be an existing directory')
+            sys.exit(1)
+
+        if os.path.exists(self.out_dir):
+            if not os.path.isdir(self.out_dir):
+                eprint(
+                    f'The output: {self.out_dir} exists and is not a directory'
+                )
+                sys.exit(1)
+        else:
+            os.makedirs(self.out_dir)
+
+        self.combine_logs()
+        with open(self.combined_logs_file_name) as f:
+            self.data = json.load(f)
+        self.grouped_by_txn = group_by_txn(self.data)
+
+        # counts_by_txn_and_server is a dictionary where the key is the txn_hash
+        # and the value is a pandas df with a row for every server and a column for every message
+        # the value is a count of how many times that message appears for that server.
+        counts_by_txn_and_server = {}
+        # dict where the key is a transaction hash and the value is the transaction
+        hash_to_txn = {}
+        # dict where the key is a transaction hash and the value is earliest timestamp in a log file
+        hash_to_timestamp = {}
+        for txn_hash, server_dict in self.grouped_by_txn.items():
+            message_set = set()
+            # message list is ordered by when it appears in the log
+            message_list = []
+            for server_id, messages in server_dict.items():
+                for m in messages:
+                    try:
+                        d = m['data']
+                        if 'msg' in d and 'transaction' in d['msg']:
+                            t = d['msg']['transaction']
+                        elif 'tx_json' in d:
+                            t = d['tx_json']
+                        if t['hash'] == txn_hash:
+                            hash_to_txn[txn_hash] = t
+                    except:
+                        pass
+                    msg = m['msg']
+                    t = _to_timestamp(m['t'])
+                    if txn_hash not in hash_to_timestamp:
+                        hash_to_timestamp[txn_hash] = t
+                    elif hash_to_timestamp[txn_hash] > t:
+                        hash_to_timestamp[txn_hash] = t
+                    if msg not in message_set:
+                        message_set.add(msg)
+                        message_list.append(msg)
+            df = pd.DataFrame(0,
+                              index=server_dict.keys(),
+                              columns=message_list)
+            for server_id, messages in server_dict.items():
+                for m in messages:
+                    df[m['msg']][server_id] += 1
+            counts_by_txn_and_server[txn_hash] = df
+
+        # sort the transactions by timestamp, but the txns with unknown timestamp at the beginning
+        self.grouped_by_txn = _rekey_dict_by_txn_date(hash_to_timestamp,
+                                                      self.grouped_by_txn)
+        counts_by_txn_and_server = _rekey_dict_by_txn_date(
+            hash_to_timestamp, counts_by_txn_and_server)
+
+        with open(self.grouped_by_txn_file_name, 'w') as out:
+            print(json.dumps(self.grouped_by_txn, indent=1), file=out)
+
+        with open(self.counts_by_txn_and_server_file_name, 'w') as out:
+            for txn_hash, df in counts_by_txn_and_server.items():
+                print(f'\n\n* Txn: {txn_hash}', file=out)
+                if txn_hash in hash_to_txn:
+                    print(json.dumps(hash_to_txn[txn_hash], indent=1),
+                          file=out)
+                rename_dict = {}
+                for column, renamed_column in zip(df.columns.array,
+                                                  string.ascii_uppercase):
+                    print(f'{renamed_column} = {column}', file=out)
+                    rename_dict[column] = renamed_column
+                df.rename(columns=rename_dict, inplace=True)
+                print(f'\n{df}', file=out)
+
+    def combine_logs(self):
+        try:
+            with open(self.combined_logs_file_name, "w") as out:
+                log_analyzer.convert_all(args.input, out, pure_json=True)
+        except Exception as e:
+            eprint(f'Excption: {e}')
+            raise e
+
+
+def main(input_dir_name: str, output_dir_name: str):
+    r = Report(input_dir_name, output_dir_name)
+
+    # Values are a list of log lines formatted as json. There are five fields:
+    # `t` is the timestamp.
+    # `m` is the module.
+    # `l` is the log level.
+    # `msg` is the message.
+    # `data` is the data.
+    # For example:
+    #
+    # {
+    #  "t": "2021-Oct-08 21:33:41.731371562 UTC",
+    #  "m": "SidechainFederator",
+    #  "l": "TRC",
+    #  "msg": "no last xchain txn with result",
+    #  "data": {
+    #   "needsOtherChainLastXChainTxn": true,
+    #   "isMainchain": false,
+    #   "jlogId": 121
+    #  }
+    # },
+
+
+# Lifecycle of a transaction
+# For each federator record:
+# Transaction detected: amount, seq, destination, chain, hash
+# Signature received: hash, seq
+# Signature sent: hash, seq, federator dst
+# Transaction submitted
+# Result received, and detect if error
+# Detect any field that doesn't match
+
+# Lifecycle of initialization
+
+# Chain listener messages
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description=(
+        'python script to generate a log report from a sidechain config directory structure containing the logs'
+    ))
+
+    parser.add_argument(
+        '--input',
+        '-i',
+        help=('directory with sidechain config directory structure'),
+    )
+
+    parser.add_argument(
+        '--output',
+        '-o',
+        help=('output directory for report files'),
+    )
+
+    return parser.parse_known_args()[0]
+
+
+if __name__ == '__main__':
+    try:
+        args = parse_args()
+        main(args.input, args.output)
+    except Exception as e:
+        eprint(f'Excption: {e}')
+        raise e

bin/sidechain/python/riplrepl.py

@@ -3,7 +3,9 @@
 Script to run an interactive shell to test sidechains.
 '''
 
-from common import disable_eprint
+import sys
+
+from common import disable_eprint, eprint
 import interactive
 import sidechain
 

bin/sidechain/python/sidechain.py

@@ -127,6 +127,12 @@ class Params:
         if args.debug_mainchain:
             self.debug_mainchain = arts.debug_mainchain
 
+        # Undocumented feature: if the environment variable RIPPLED_SIDECHAIN_RR is set, it is
+        # assumed to point to the rr executable. Sidechain server 0 will then be run under rr.
+        self.sidechain_rr = None
+        if 'RIPPLED_SIDECHAIN_RR' in os.environ:
+            self.sidechain_rr = os.environ['RIPPLED_SIDECHAIN_RR']
+
         self.standalone = args.standalone
         self.with_pauses = args.with_pauses
         self.interactive = args.interactive
@@ -471,7 +477,8 @@ def _multinode_with_callback(params: Params,
 
         with testnet_app(exe=params.sidechain_exe,
                          configs=testnet_configs,
-                         run_server=run_server_list) as n_app:
+                         run_server=run_server_list,
+                         sidechain_rr=params.sidechain_rr) as n_app:
 
             if params.with_pauses:
                 input("Pausing after testnet start (press enter to continue)")

bin/sidechain/python/test_utils.py

@@ -26,11 +26,13 @@ def _sc_subscribe_callback(v: dict):
 
 
 def mc_connect_subscription(app: App, door_account: Account):
-    app(Subscribe(accounts=[door_account]), _mc_subscribe_callback)
+    app(Subscribe(account_history_account=door_account),
+        _mc_subscribe_callback)
 
 
 def sc_connect_subscription(app: App, door_account: Account):
-    app(Subscribe(accounts=[door_account]), _sc_subscribe_callback)
+    app(Subscribe(account_history_account=door_account),
+        _sc_subscribe_callback)
 
 
 # This pops elements off the subscribe_queue until the transaction is found
@@ -111,18 +113,18 @@ def sc_wait_for_payment_detect(app: App, src: Account, dst: Account,
 def wait_for_balance_change(app: App,
                             acc: Account,
                             pre_balance: Asset,
-                            final_diff: Optional[Asset] = None):
+                            expected_diff: Optional[Asset] = None):
     logging.info(
-        f'waiting for balance change {acc.account_id = } {pre_balance = } {final_diff = }'
+        f'waiting for balance change {acc.account_id = } {pre_balance = } {expected_diff = }'
     )
     for i in range(30):
         new_bal = app.get_balance(acc, pre_balance(0))
         diff = new_bal - pre_balance
         if new_bal != pre_balance:
             logging.info(
-                f'Balance changed {acc.account_id = } {pre_balance = } {new_bal = } {diff = } {final_diff = }'
+                f'Balance changed {acc.account_id = } {pre_balance = } {new_bal = } {diff = } {expected_diff = }'
             )
-            if final_diff is None or diff == final_diff:
+            if expected_diff is None or diff == expected_diff:
                 return
         app.maybe_ledger_accept()
         time.sleep(2)
@@ -131,10 +133,10 @@ def wait_for_balance_change(app: App,
                 f'Waiting for balance to change {acc.account_id = } {pre_balance = }'
             )
     logging.error(
-        f'Expected balance to change {acc.account_id = } {pre_balance = } {new_bal = } {diff = } {final_diff = }'
+        f'Expected balance to change {acc.account_id = } {pre_balance = } {new_bal = } {diff = } {expected_diff = }'
     )
     raise ValueError(
-        f'Expected balance to change {acc.account_id = } {pre_balance = } {new_bal = } {diff = } {final_diff = }'
+        f'Expected balance to change {acc.account_id = } {pre_balance = } {new_bal = } {diff = } {expected_diff = }'
     )
 
 

bin/sidechain/python/testnet.py

@@ -17,13 +17,19 @@ from ripple_client import RippleClient
 class Network:
     # If run_server is None, run all the servers.
     # This is useful to help debugging
-    def __init__(self,
-                 exe: str,
-                 configs: List[ConfigFile],
-                 *,
-                 command_logs: Optional[List[str]] = None,
-                 run_server: Optional[List[bool]] = None,
-                 extra_args: Optional[List[List[str]]] = None):
+    def __init__(
+            self,
+            exe: str,
+            configs: List[ConfigFile],
+            *,
+            command_logs: Optional[List[str]] = None,
+            run_server: Optional[List[bool]] = None,
+            # undocumented feature. If with_rr is not None, assume it points to the rr debugger executable
+            # and run server 0 under rr
+            with_rr: Optional[str] = None,
+            extra_args: Optional[List[List[str]]] = None):
+
+        self.with_rr = with_rr
         if not configs:
             raise ValueError(f'Must specify at least one config')
 
@@ -164,7 +170,11 @@ class Network:
 
             client = self.clients[i]
             to_run = [client.exe, '--conf', client.config_file_name]
-            print(f'Starting server {client.config_file_name}')
+            if self.with_rr and i == 0:
+                to_run = [self.with_rr, 'record'] + to_run
+                print(f'Starting server with rr {client.config_file_name}')
+            else:
+                print(f'Starting server {client.config_file_name}')
             fout = open(os.devnull, 'w')
             p = subprocess.Popen(to_run + extra_args[i],
                                  stdout=fout,

bin/sidechain/python/tests/simple_xchain_transfer_test.py

@@ -6,45 +6,54 @@ from typing import Dict
 import sys
 
 from app import App
-from common import Asset, eprint, disable_eprint, XRP
+from common import Asset, eprint, disable_eprint, drops, XRP
 import interactive
 from sidechain import Params
 import sidechain
 import test_utils
 import time
 from transaction import Payment, Trust
+import tst_common
 
 
 def simple_xrp_test(mc_app: App, sc_app: App, params: Params):
     alice = mc_app.account_from_alias('alice')
     adam = sc_app.account_from_alias('adam')
+    mc_door = mc_app.account_from_alias('door')
+    sc_door = sc_app.account_from_alias('door')
 
     # main to side
     # First txn funds the side chain account
     with test_utils.test_context(mc_app, sc_app):
-        to_send_asset = XRP(1000)
-        pre_bal = sc_app.get_balance(adam, to_send_asset)
+        to_send_asset = XRP(9999)
+        mc_pre_bal = mc_app.get_balance(mc_door, to_send_asset)
+        sc_pre_bal = sc_app.get_balance(adam, to_send_asset)
         sidechain.main_to_side_transfer(mc_app, sc_app, alice, adam,
                                         to_send_asset, params)
-        test_utils.wait_for_balance_change(sc_app, adam, pre_bal,
+        test_utils.wait_for_balance_change(mc_app, mc_door, mc_pre_bal,
+                                           to_send_asset)
+        test_utils.wait_for_balance_change(sc_app, adam, sc_pre_bal,
                                            to_send_asset)
 
     for i in range(2):
         # even amounts for main to side
-        for value in range(10, 20, 2):
+        for value in range(20, 30, 2):
             with test_utils.test_context(mc_app, sc_app):
-                to_send_asset = XRP(value)
-                pre_bal = sc_app.get_balance(adam, to_send_asset)
+                to_send_asset = drops(value)
+                mc_pre_bal = mc_app.get_balance(mc_door, to_send_asset)
+                sc_pre_bal = sc_app.get_balance(adam, to_send_asset)
                 sidechain.main_to_side_transfer(mc_app, sc_app, alice, adam,
                                                 to_send_asset, params)
-                test_utils.wait_for_balance_change(sc_app, adam, pre_bal,
+                test_utils.wait_for_balance_change(mc_app, mc_door, mc_pre_bal,
+                                                   to_send_asset)
+                test_utils.wait_for_balance_change(sc_app, adam, sc_pre_bal,
                                                    to_send_asset)
 
         # side to main
         # odd amounts for side to main
-        for value in range(9, 19, 2):
+        for value in range(19, 29, 2):
             with test_utils.test_context(mc_app, sc_app):
-                to_send_asset = XRP(value)
+                to_send_asset = drops(value)
                 pre_bal = mc_app.get_balance(alice, to_send_asset)
                 sidechain.side_to_main_transfer(mc_app, sc_app, adam, alice,
                                                 to_send_asset, params)
@@ -108,37 +117,6 @@ def simple_iou_test(mc_app: App, sc_app: App, params: Params):
                                                    rcv_asset)
 
 
-def run(mc_app: App, sc_app: App, params: Params):
-    # process will run while stop token is non-zero
-    stop_token = Value('i', 1)
-    p = None
-    if mc_app.standalone:
-        p = Process(target=sidechain.close_mainchain_ledgers,
-                    args=(stop_token, params))
-        p.start()
-    try:
-        # TODO: Tests fail without this sleep. Fix this bug.
-        time.sleep(10)
-        setup_accounts(mc_app, sc_app, params)
-        simple_xrp_test(mc_app, sc_app, params)
-        simple_iou_test(mc_app, sc_app, params)
-    finally:
-        if p:
-            stop_token.value = 0
-            p.join()
-        sidechain._convert_log_files_to_json(
-            mc_app.get_configs() + sc_app.get_configs(), 'final.json')
-
-
-def standalone_test(params: Params):
-    def callback(mc_app: App, sc_app: App):
-        run(mc_app, sc_app, params)
-
-    sidechain._standalone_with_callback(params,
-                                        callback,
-                                        setup_user_accounts=False)
-
-
 def setup_accounts(mc_app: App, sc_app: App, params: Params):
     # Setup a funded user account on the main chain, and add an unfunded account.
     # Setup address book and add a funded account on the mainchain.
@@ -161,31 +139,13 @@ def setup_accounts(mc_app: App, sc_app: App, params: Params):
     ed = sc_app.create_account('ed')
 
 
-def multinode_test(params: Params):
-    def callback(mc_app: App, sc_app: App):
-        run(mc_app, sc_app, params)
-
-    sidechain._multinode_with_callback(params,
-                                       callback,
-                                       setup_user_accounts=False)
+def run_all(mc_app: App, sc_app: App, params: Params):
+    setup_accounts(mc_app, sc_app, params)
+    logging.info(f'mainchain:\n{mc_app.key_manager.to_string()}')
+    logging.info(f'sidechain:\n{sc_app.key_manager.to_string()}')
+    simple_xrp_test(mc_app, sc_app, params)
+    simple_iou_test(mc_app, sc_app, params)
 
 
 def test_simple_xchain(configs_dirs_dict: Dict[int, str]):
-    params = sidechain.Params(configs_dir=configs_dirs_dict[1])
-
-    if err_str := params.check_error():
-        eprint(err_str)
-        sys.exit(1)
-
-    if params.verbose:
-        print("eprint enabled")
-    else:
-        disable_eprint()
-
-    # Set to true to help debug tests
-    test_utils.test_context_verbose_logging = True
-
-    if params.standalone:
-        standalone_test(params)
-    else:
-        multinode_test(params)
+    tst_common.test_start(configs_dirs_dict, run_all)