From 24cf8ab8c7822944dd7691d8e768b53cd46578ec Mon Sep 17 00:00:00 2001 From: seelabs Date: Fri, 10 Sep 2021 11:25:17 -0400 Subject: [PATCH] Sidechain python test environment and repl --- bin/sidechain/python/app.py | 612 +++++++ bin/sidechain/python/command.py | 557 ++++++ bin/sidechain/python/common.py | 250 +++ bin/sidechain/python/config_file.py | 83 + bin/sidechain/python/create_config_files.py | 630 +++++++ bin/sidechain/python/interactive.py | 1496 +++++++++++++++++ bin/sidechain/python/log_analyzer.py | 143 ++ bin/sidechain/python/riplrepl.py | 33 + bin/sidechain/python/ripple_client.py | 193 +++ bin/sidechain/python/sidechain.py | 576 +++++++ bin/sidechain/python/test_utils.py | 174 ++ bin/sidechain/python/testnet.py | 206 +++ bin/sidechain/python/tests/conftest.py | 64 + .../tests/simple_xchain_transfer_test.py | 191 +++ bin/sidechain/python/transaction.py | 366 ++++ 15 files changed, 5574 insertions(+) create mode 100644 bin/sidechain/python/app.py create mode 100644 bin/sidechain/python/command.py create mode 100644 bin/sidechain/python/common.py create mode 100644 bin/sidechain/python/config_file.py create mode 100755 bin/sidechain/python/create_config_files.py create mode 100644 bin/sidechain/python/interactive.py create mode 100755 bin/sidechain/python/log_analyzer.py create mode 100755 bin/sidechain/python/riplrepl.py create mode 100644 bin/sidechain/python/ripple_client.py create mode 100755 bin/sidechain/python/sidechain.py create mode 100644 bin/sidechain/python/test_utils.py create mode 100644 bin/sidechain/python/testnet.py create mode 100644 bin/sidechain/python/tests/conftest.py create mode 100644 bin/sidechain/python/tests/simple_xchain_transfer_test.py create mode 100644 bin/sidechain/python/transaction.py diff --git a/bin/sidechain/python/app.py b/bin/sidechain/python/app.py new file mode 100644 index 0000000000..1fb7315b57 --- /dev/null +++ b/bin/sidechain/python/app.py @@ -0,0 +1,612 @@ +from contextlib import contextmanager +import json +import os +import pandas as pd +from pathlib import Path +import subprocess +import time +from typing import Callable, Dict, List, Optional, Set, Union + +from ripple_client import RippleClient +from common import Account, Asset, XRP +from command import AccountInfo, AccountLines, BookOffers, Command, FederatorInfo, LedgerAccept, Sign, Submit, SubscriptionCommand, WalletPropose +from config_file import ConfigFile +import testnet +from transaction import Payment, Transaction + + +class KeyManager: + def __init__(self): + self._aliases = {} # alias -> account + self._accounts = {} # account id -> account + + def add(self, account: Account) -> bool: + if account.nickname: + self._aliases[account.nickname] = account + self._accounts[account.account_id] = account + + def is_alias(self, name: str): + return name in self._aliases + + def account_from_alias(self, name: str) -> Account: + assert name in self._aliases + return self._aliases[name] + + def known_accounts(self) -> List[Account]: + return list(self._accounts.values()) + + def account_id_dict(self) -> Dict[str, Account]: + return self._accounts + + def alias_or_account_id(self, id: Union[Account, str]) -> str: + ''' + return the alias if it exists, otherwise return the id + ''' + if isinstance(id, Account): + return id.alias_or_account_id() + + if id in self._accounts: + return self._accounts[id].nickname + return id + + def alias_to_account_id(self, alias: str) -> Optional[str]: + if id in self._aliases: + return self._aliases[id].account_id + return None + + def to_string(self, nickname: Optional[str] = None): + names = [] + account_ids = [] + if nickname: + names = [nickname] + if nickname in self._aliases: + account_ids = [self._aliases[nickname].account_id] + else: + account_id = ['NA'] + else: + for (k, v) in self._aliases.items(): + names.append(k) + account_ids.append(v.account_id) + # use a dataframe to get a nice table output + df = pd.DataFrame(data={'name': names, 'id': account_ids}) + return f'{df.to_string(index=False)}' + + +class AssetAliases: + def __init__(self): + self._aliases = {} # alias -> asset + + def add(self, asset: Asset, name: str): + self._aliases[name] = asset + + def is_alias(self, name: str): + return name in self._aliases + + def asset_from_alias(self, name: str) -> Asset: + assert name in self._aliases + return self._aliases[name] + + def known_aliases(self) -> List[str]: + return list(self._aliases.keys()) + + def known_assets(self) -> List[Asset]: + return list(self._aliases.values()) + + def to_string(self, nickname: Optional[str] = None): + names = [] + currencies = [] + issuers = [] + if nickname: + names = [nickname] + if nickname in self._aliases: + v = self._aliases[nickname] + currencies = [v.currency] + iss = v.issuer if v.issuer else '' + issuers = [v.issuer if v.issuer else ''] + else: + currencies = ['NA'] + issuers = ['NA'] + else: + for (k, v) in self._aliases.items(): + names.append(k) + currencies.append(v.currency) + issuers.append(v.issuer if v.issuer else '') + # use a dataframe to get a nice table output + df = pd.DataFrame(data={ + 'name': names, + 'currency': currencies, + 'issuer': issuers + }) + return f'{df.to_string(index=False)}' + + +class App: + '''App to to interact with rippled servers''' + def __init__(self, + *, + standalone: bool, + network: Optional[testnet.Network] = None, + client: Optional[RippleClient] = None): + if network and client: + raise ValueError('Cannot specify both a testnet and client in App') + if not network and not client: + raise ValueError('Must specify a testnet or a client in App') + + self.standalone = standalone + self.network = network + + if client: + self.client = client + else: + self.client = self.network.get_client(0) + + self.key_manager = KeyManager() + self.asset_aliases = AssetAliases() + root_account = Account(nickname='root', + account_id='rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh', + secret_key='masterpassphrase') + self.key_manager.add(root_account) + + def shutdown(self): + if self.network: + self.network.shutdown() + else: + self.client.shutdown() + + def send_signed(self, txn: Transaction) -> dict: + '''Sign then send the given transaction''' + if not txn.account.secret_key: + raise ValueError('Cannot sign transaction without secret key') + r = self(Sign(txn.account.secret_key, txn.to_cmd_obj())) + raw_signed = r['tx_blob'] + return self(Submit(raw_signed)) + + def send_command(self, cmd: Command) -> dict: + '''Send the command to the rippled server''' + return self.client.send_command(cmd) + + # Need async version to close ledgers from async functions + async def async_send_command(self, cmd: Command) -> dict: + '''Send the command to the rippled server''' + return await self.client.async_send_command(cmd) + + def send_subscribe_command( + self, + cmd: SubscriptionCommand, + callback: Optional[Callable[[dict], None]] = None) -> dict: + '''Send the subscription command to the rippled server. If already subscribed, it will unsubscribe''' + return self.client.send_subscribe_command(cmd, callback) + + def get_pids(self) -> List[int]: + if self.network: + return self.network.get_pids() + if pid := self.client.get_pid(): + return [pid] + + def get_running_status(self) -> List[bool]: + if self.network: + return self.network.get_running_status() + if self.client.get_pid(): + return [True] + else: + return [False] + + # Get a dict of the server_state, validated_ledger_seq, and complete_ledgers + def get_brief_server_info(self) -> dict: + if self.network: + return self.network.get_brief_server_info() + else: + ret = {} + for (k, v) in self.client.get_brief_server_info().items(): + ret[k] = [v] + return ret + + def servers_start(self, + server_indexes: Optional[Union[Set[int], + List[int]]] = None, + *, + extra_args: Optional[List[List[str]]] = None): + if self.network: + return self.network.servers_start(server_indexes, + extra_args=extra_args) + else: + raise ValueError('Cannot start stand alone server') + + def servers_stop(self, + server_indexes: Optional[Union[Set[int], + List[int]]] = None): + if self.network: + return self.network.servers_stop(server_indexes) + else: + raise ValueError('Cannot stop stand alone server') + + def federator_info(self, + server_indexes: Optional[Union[Set[int], + List[int]]] = None): + # key is server index. value is federator_info result + result_dict = {} + if self.network: + if not server_indexes: + server_indexes = [ + i for i in range(self.network.num_clients()) + if self.network.is_running(i) + ] + for i in server_indexes: + if self.network.is_running(i): + result_dict[i] = self.network.get_client(i).send_command( + FederatorInfo()) + else: + if 0 in server_indexes: + result_dict[0] = self.client.send_command(FederatorInfo()) + return result_dict + + def __call__(self, + to_send: Union[Transaction, Command, SubscriptionCommand], + callback: Optional[Callable[[dict], None]] = None, + *, + insert_seq_and_fee=False) -> dict: + '''Call `send_signed` for transactions or `send_command` for commands''' + if isinstance(to_send, SubscriptionCommand): + return self.send_subscribe_command(to_send, callback) + assert callback is None + if isinstance(to_send, Transaction): + if insert_seq_and_fee: + self.insert_seq_and_fee(to_send) + return self.send_signed(to_send) + if isinstance(to_send, Command): + return self.send_command(to_send) + raise ValueError( + 'Expected `to_send` to be either a Transaction, Command, or SubscriptionCommand' + ) + + def get_configs(self) -> List[str]: + if self.network: + return self.network.get_configs() + return [self.client.config] + + def create_account(self, name: str) -> Account: + ''' Create an account. Use the name as the alias. ''' + if name == 'root': + return + assert not self.key_manager.is_alias(name) + + account = Account(nickname=name, result_dict=self(WalletPropose())) + self.key_manager.add(account) + return account + + def create_accounts(self, + names: List[str], + funding_account: Union[Account, str] = 'root', + amt: Union[int, Asset] = 1000000000) -> List[Account]: + '''Fund the accounts with nicknames 'names' by using the funding account and amt''' + accounts = [self.create_account(n) for n in names] + if not isinstance(funding_account, Account): + org_funding_account = funding_account + funding_account = self.key_manager.account_from_alias( + funding_account) + if not isinstance(funding_account, Account): + raise ValueError( + f'Could not find funding account {org_funding_account}') + if not isinstance(amt, Asset): + assert isinstance(amt, int) + amt = Asset(value=amt) + for a in accounts: + p = Payment(account=funding_account, dst=a, amt=amt) + self.send_signed(p) + return accounts + + def maybe_ledger_accept(self): + if not self.standalone: + return + self(LedgerAccept()) + + # Need async version to close ledgers from async functions + async def async_maybe_ledger_accept(self): + if not self.standalone: + return + await self.async_send_command(LedgerAccept()) + + def get_balances( + self, + account: Union[Account, List[Account], None] = None, + asset: Union[Asset, List[Asset]] = Asset() + ) -> pd.DataFrame: + '''Return a pandas dataframe of account balances. If account is None, treat as a wildcard (use address book)''' + if account is None: + account = self.key_manager.known_accounts() + if isinstance(account, list): + result = [self.get_balances(acc, asset) for acc in account] + return pd.concat(result, ignore_index=True) + if isinstance(asset, list): + result = [self.get_balances(account, ass) for ass in asset] + return pd.concat(result, ignore_index=True) + if asset.is_xrp(): + try: + df = self.get_account_info(account) + except: + # Most likely the account does not exist on the ledger. Give a balance of zero. + df = pd.DataFrame({ + 'account': [account], + 'balance': [0], + 'flags': [0], + 'owner_count': [0], + 'previous_txn_id': ['NA'], + 'previous_txn_lgr_seq': [-1], + 'sequence': [-1] + }) + df = df.assign(currency='XRP', peer='', limit='') + return df.loc[:, + ['account', 'balance', 'currency', 'peer', 'limit']] + else: + try: + df = self.get_trust_lines(account) + if df.empty: return df + df = df[(df['peer'] == asset.issuer.account_id) + & (df['currency'] == asset.currency)] + except: + # Most likely the account does not exist on the ledger. Return an empty data frame + df = pd.DataFrame({ + 'account': [], + 'balance': [], + 'currency': [], + 'peer': [], + 'limit': [], + }) + return df.loc[:, + ['account', 'balance', 'currency', 'peer', 'limit']] + + def get_balance(self, account: Account, asset: Asset) -> Asset: + '''Get a balance from a single account in a single asset''' + try: + df = self.get_balances(account, asset) + return asset(df.iloc[0]['balance']) + except: + return asset(0) + + def get_account_info(self, + account: Optional[Account] = None) -> pd.DataFrame: + '''Return a pandas dataframe of account info. If account is None, treat as a wildcard (use address book)''' + if account is None: + known_accounts = self.key_manager.known_accounts() + result = [self.get_account_info(acc) for acc in known_accounts] + return pd.concat(result, ignore_index=True) + try: + result = self.client.send_command(AccountInfo(account)) + except: + # Most likely the account does not exist on the ledger. Give a balance of zero. + return pd.DataFrame({ + 'account': [account], + 'balance': [0], + 'flags': [0], + 'owner_count': [0], + 'previous_txn_id': ['NA'], + 'previous_txn_lgr_seq': [-1], + 'sequence': [-1] + }) + if 'account_data' not in result: + raise ValueError('Bad result from account_info command') + info = result['account_data'] + for dk in ['LedgerEntryType', 'index']: + del info[dk] + df = pd.DataFrame([info]) + df.rename(columns={ + 'Account': 'account', + 'Balance': 'balance', + 'Flags': 'flags', + 'OwnerCount': 'owner_count', + 'PreviousTxnID': 'previous_txn_id', + 'PreviousTxnLgrSeq': 'previous_txn_lgr_seq', + 'Sequence': 'sequence' + }, + inplace=True) + df['balance'] = df['balance'].astype(int) + return df + + def get_trust_lines(self, + account: Account, + peer: Optional[Account] = None) -> pd.DataFrame: + '''Return a pandas dataframe account trust lines. If peer account is None, treat as a wildcard''' + result = self.send_command(AccountLines(account, peer=peer)) + if 'lines' not in result or 'account' not in result: + raise ValueError('Bad result from account_lines command') + account = result['account'] + lines = result['lines'] + for d in lines: + d['peer'] = d['account'] + d['account'] = account + return pd.DataFrame(lines) + + def get_offers(self, taker_pays: Asset, taker_gets: Asset) -> pd.DataFrame: + '''Return a pandas dataframe of offers''' + result = self.send_command(BookOffers(taker_pays, taker_gets)) + if 'offers' not in result: + raise ValueError('Bad result from book_offers command') + + offers = result['offers'] + delete_keys = [ + 'BookDirectory', 'BookNode', 'LedgerEntryType', 'OwnerNode', + 'PreviousTxnID', 'PreviousTxnLgrSeq', 'Sequence', 'index' + ] + for d in offers: + for dk in delete_keys: + del d[dk] + for t in ['TakerPays', 'TakerGets', 'owner_funds']: + if 'value' in d[t]: + d[t] = d[t]['value'] + df = pd.DataFrame(offers) + df.rename(columns={ + 'Account': 'account', + 'Flags': 'flags', + 'TakerGets': 'taker_gets', + 'TakerPays': 'taker_pays' + }, + inplace=True) + return df + + def account_balance(self, account: Account, asset: Asset) -> Asset: + '''get the account's balance of the asset''' + pass + + def substitute_nicknames( + self, + df: pd.DataFrame, + cols: List[str] = ['account', 'peer']) -> pd.DataFrame: + result = df.copy(deep=True) + for c in cols: + if c not in result: + continue + result[c] = result[c].map( + lambda x: self.key_manager.alias_or_account_id(x)) + return result + + def add_to_keymanager(self, account: Account): + self.key_manager.add(account) + + def is_alias(self, name: str) -> bool: + return self.key_manager.is_alias(name) + + def account_from_alias(self, name: str) -> Account: + return self.key_manager.account_from_alias(name) + + def known_accounts(self) -> List[Account]: + return self.key_manager.known_accounts() + + def known_asset_aliases(self) -> List[str]: + return self.asset_aliases.known_aliases() + + def known_iou_assets(self) -> List[Asset]: + return self.asset_aliases.known_assets() + + def is_asset_alias(self, name: str) -> bool: + return self.asset_aliases.is_alias(name) + + def add_asset_alias(self, asset: Asset, name: str): + self.asset_aliases.add(asset, name) + + def asset_from_alias(self, name: str) -> Asset: + return self.asset_aliases.asset_from_alias(name) + + def insert_seq_and_fee(self, txn: Transaction): + acc_info = self(AccountInfo(txn.account)) + # TODO: set better fee (Hard code a fee of 15 for now) + txn.set_seq_and_fee(acc_info['account_data']['Sequence'], 15) + + def get_client(self) -> RippleClient: + return self.client + + +def balances_dataframe(chains: List[App], + chain_names: List[str], + account_ids: Optional[List[Account]] = None, + assets: List[Asset] = None, + in_drops: bool = False): + def _removesuffix(self: str, suffix: str) -> str: + if suffix and self.endswith(suffix): + return self[:-len(suffix)] + else: + return self[:] + + def _balance_df(chain: App, acc: Optional[Account], + asset: Union[Asset, List[Asset]], in_drops: bool): + b = chain.get_balances(acc, asset) + if not in_drops: + b.loc[b['currency'] == 'XRP', 'balance'] /= 1_000_000 + b = chain.substitute_nicknames(b) + b = b.set_index('account') + return b + + if account_ids is None: + account_ids = [None] * len(chains) + + if assets is None: + # XRP and all assets in the assets alias list + assets = [[XRP(0)] + c.known_iou_assets() for c in chains] + + dfs = [] + keys = [] + for chain, chain_name, acc, asset in zip(chains, chain_names, account_ids, + assets): + dfs.append(_balance_df(chain, acc, asset, in_drops)) + keys.append(_removesuffix(chain_name, 'chain')) + df = pd.concat(dfs, keys=keys) + return df + + +# Start an app with a single client +@contextmanager +def single_client_app(*, + config: ConfigFile, + command_log: Optional[str] = None, + server_out=os.devnull, + run_server: bool = True, + exe: Optional[str] = None, + extra_args: Optional[List[str]] = None, + standalone=False): + '''Start a ripple server and return an app''' + try: + if extra_args is None: + extra_args = [] + to_run = None + app = None + client = RippleClient(config=config, command_log=command_log, exe=exe) + if run_server: + to_run = [client.exe, '--conf', client.config_file_name] + if standalone: + to_run.append('-a') + fout = open(server_out, 'w') + p = subprocess.Popen(to_run + extra_args, + stdout=fout, + stderr=subprocess.STDOUT) + client.set_pid(p.pid) + print( + f'started rippled: config: {client.config_file_name} PID: {p.pid}', + flush=True) + time.sleep(1.5) # give process time to startup + app = App(client=client, standalone=standalone) + yield app + finally: + if app: + app.shutdown() + if run_server and to_run: + subprocess.Popen(to_run + ['stop'], + stdout=fout, + stderr=subprocess.STDOUT) + p.wait() + + +def configs_for_testnet(config_file_prefix: str) -> List[ConfigFile]: + configs = [] + p = Path(config_file_prefix) + dir = p.parent + file = p.name + file_names = [] + for f in os.listdir(dir): + cfg = os.path.join(dir, f, 'rippled.cfg') + if f.startswith(file) and os.path.exists(cfg): + file_names.append(cfg) + file_names.sort() + return [ConfigFile(file_name=f) for f in file_names] + + +# Start an app for a network with the config files matched by `config_file_prefix*/rippled.cfg` +@contextmanager +def testnet_app(*, + exe: str, + configs: List[ConfigFile], + command_logs: Optional[List[str]] = None, + run_server: Optional[List[bool]] = None, + extra_args: Optional[List[List[str]]] = None): + '''Start a ripple testnet and return an app''' + try: + app = None + network = testnet.Network(exe, + configs, + command_logs=command_logs, + run_server=run_server, + extra_args=extra_args) + network.wait_for_validated_ledger() + app = App(network=network, standalone=False) + yield app + finally: + if app: + app.shutdown() diff --git a/bin/sidechain/python/command.py b/bin/sidechain/python/command.py new file mode 100644 index 0000000000..545da1aea8 --- /dev/null +++ b/bin/sidechain/python/command.py @@ -0,0 +1,557 @@ +import json +from typing import List, Optional, Union + +from common import Account, Asset + + +class Command: + '''Interface for all commands sent to the server''' + + # command id useful for websocket messages + next_cmd_id = 1 + + def __init__(self): + self.cmd_id = Command.next_cmd_id + Command.next_cmd_id += 1 + + def cmd_name(self) -> str: + '''Return the command name for use in a command line''' + assert False + return '' + + def get_command_line_list(self) -> List[str]: + '''Return a list of strings suitable for a command line command for a rippled server''' + return [self.cmd_name, json.dumps(self.to_cmd_obj())] + + def get_websocket_dict(self) -> dict: + '''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket''' + result = self.to_cmd_obj() + return self.add_websocket_fields(result) + + def to_cmd_obj(self) -> dict: + '''Return an object suitalbe for use in a command (input to json.dumps or similar)''' + assert False + return {} + + def add_websocket_fields(self, cmd_dict: dict) -> dict: + cmd_dict['id'] = self.cmd_id + cmd_dict['command'] = self.cmd_name() + return cmd_dict + + def _set_flag(self, flag_bit: int, value: bool = True): + '''Set or clear the flag bit''' + if value: + self.flags |= flag_bit + else: + self.flags &= ~flag_bit + return self + + +class SubscriptionCommand(Command): + def __init__(self): + super().__init__() + + +class PathFind(Command): + '''Rippled ripple_path_find command''' + def __init__(self, + *, + src: Account, + dst: Account, + amt: Asset, + send_max: Optional[Asset] = None, + src_currencies: Optional[List[Asset]] = None, + ledger_hash: Optional[str] = None, + ledger_index: Optional[Union[int, str]] = None): + super().__init__() + self.src = src + self.dst = dst + self.amt = amt + self.send_max = send_max + self.src_currencies = src_currencies + self.ledger_hash = ledger_hash + self.ledger_index = ledger_index + + def cmd_name(self) -> str: + return 'ripple_path_find' + + def add_websocket_fields(self, cmd_dict: dict) -> dict: + cmd_dict = super().add_websocket_fields(cmd_dict) + cmd_dict['subcommand'] = 'create' + return cmd_dict + + def to_cmd_obj(self) -> dict: + '''convert to transaction form (suitable for using json.dumps or similar)''' + cmd = { + 'source_account': self.src.account_id, + 'destination_account': self.dst.account_id, + 'destination_amount': self.amt.to_cmd_obj() + } + if self.send_max is not None: + cmd['send_max'] = self.send_max.to_cmd_obj() + if self.ledger_hash is not None: + cmd['ledger_hash'] = self.ledger_hash + if self.ledger_index is not None: + cmd['ledger_index'] = self.ledger_index + if self.src_currencies: + c = [] + for sc in self.src_currencies: + d = {'currency': sc.currency, 'issuer': sc.issuer.account_id} + c.append(d) + cmd['source_currencies'] = c + return cmd + + +class Sign(Command): + '''Rippled sign command''' + def __init__(self, secret: str, tx: dict): + super().__init__() + self.tx = tx + self.secret = secret + + def cmd_name(self) -> str: + return 'sign' + + def get_command_line_list(self) -> List[str]: + '''Return a list of strings suitable for a command line command for a rippled server''' + return [self.cmd_name(), self.secret, f'{json.dumps(self.tx)}'] + + def get_websocket_dict(self) -> dict: + '''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket''' + result = {'secret': self.secret, 'tx_json': self.tx} + return self.add_websocket_fields(result) + + +class Submit(Command): + '''Rippled submit command''' + def __init__(self, tx_blob: str): + super().__init__() + self.tx_blob = tx_blob + + def cmd_name(self) -> str: + return 'submit' + + def get_command_line_list(self) -> List[str]: + '''Return a list of strings suitable for a command line command for a rippled server''' + return [self.cmd_name(), self.tx_blob] + + def get_websocket_dict(self) -> dict: + '''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket''' + result = {'tx_blob': self.tx_blob} + return self.add_websocket_fields(result) + + +class LedgerAccept(Command): + '''Rippled ledger_accept command''' + def __init__(self): + super().__init__() + + def cmd_name(self) -> str: + return 'ledger_accept' + + def get_command_line_list(self) -> List[str]: + '''Return a list of strings suitable for a command line command for a rippled server''' + return [self.cmd_name()] + + def get_websocket_dict(self) -> dict: + '''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket''' + result = {} + return self.add_websocket_fields(result) + + +class Stop(Command): + '''Rippled stop command''' + def __init__(self): + super().__init__() + + def cmd_name(self) -> str: + return 'stop' + + def get_command_line_list(self) -> List[str]: + '''Return a list of strings suitable for a command line command for a rippled server''' + return [self.cmd_name()] + + def get_websocket_dict(self) -> dict: + '''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket''' + result = {} + return self.add_websocket_fields(result) + + +class LogLevel(Command): + '''Rippled log_level command''' + def __init__(self, severity: str, *, partition: Optional[str] = None): + super().__init__() + self.severity = severity + self.partition = partition + + def cmd_name(self) -> str: + return 'log_level' + + def get_command_line_list(self) -> List[str]: + '''Return a list of strings suitable for a command line command for a rippled server''' + if self.partition is not None: + return [self.cmd_name(), self.partition, self.severity] + return [self.cmd_name(), self.severity] + + def get_websocket_dict(self) -> dict: + '''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket''' + result = {'severity': self.severity} + if self.partition is not None: + result['partition'] = self.partition + return self.add_websocket_fields(result) + + +class WalletPropose(Command): + '''Rippled wallet_propose command''' + def __init__(self, + *, + passphrase: Optional[str] = None, + seed: Optional[str] = None, + seed_hex: Optional[str] = None, + key_type: Optional[str] = None): + super().__init__() + self.passphrase = passphrase + self.seed = seed + self.seed_hex = seed_hex + self.key_type = key_type + + def cmd_name(self) -> str: + return 'wallet_propose' + + def get_command_line_list(self) -> List[str]: + '''Return a list of strings suitable for a command line command for a rippled server''' + assert not self.seed and not self.seed_hex and ( + not self.key_type or self.key_type == 'secp256k1') + if self.passphrase: + return [self.cmd_name(), self.passphrase] + return [self.cmd_name()] + + def get_websocket_dict(self) -> dict: + '''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket''' + result = {} + if self.seed is not None: + result['seed'] = self.seed + if self.seed_hex is not None: + result['seed_hex'] = self.seed_hex + if self.passphrase is not None: + result['passphrase'] = self.passphrase + if self.key_type is not None: + result['key_type'] = self.key_type + return self.add_websocket_fields(result) + + +class ValidationCreate(Command): + '''Rippled validation_create command''' + def __init__(self, *, secret: Optional[str] = None): + super().__init__() + self.secret = secret + + def cmd_name(self) -> str: + return 'validation_create' + + def get_command_line_list(self) -> List[str]: + '''Return a list of strings suitable for a command line command for a rippled server''' + if self.secret: + return [self.cmd_name(), self.secret] + return [self.cmd_name()] + + def get_websocket_dict(self) -> dict: + '''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket''' + result = {} + if self.secret is not None: + result['secret'] = self.secret + return self.add_websocket_fields(result) + + +class AccountInfo(Command): + '''Rippled account_info command''' + def __init__(self, + account: Account, + *, + strict: Optional[bool] = None, + ledger_hash: Optional[str] = None, + ledger_index: Optional[Union[str, int]] = None, + queue: Optional[bool] = None, + signers_list: Optional[bool] = None): + super().__init__() + self.account = account + self.strict = strict + self.ledger_hash = ledger_hash + self.ledger_index = ledger_index + self.queue = queue + self.signers_list = signers_list + assert not ((ledger_hash is not None) and (ledger_index is not None)) + + def cmd_name(self) -> str: + return 'account_info' + + def get_command_line_list(self) -> List[str]: + '''Return a list of strings suitable for a command line command for a rippled server''' + result = [self.cmd_name(), self.account.account_id] + if self.ledger_index is not None: + result.append(self.ledger_index) + if self.ledger_hash is not None: + result.append(self.ledger_hash) + if self.strict is not None: + result.append(self.strict) + return result + + def get_websocket_dict(self) -> dict: + '''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket''' + result = {'account': self.account.account_id} + if self.ledger_index is not None: + result['ledger_index'] = self.ledger_index + if self.ledger_hash is not None: + result['ledger_hash'] = self.ledger_hash + if self.strict is not None: + result['strict'] = self.strict + if self.queue is not None: + result['queue'] = self.queue + return self.add_websocket_fields(result) + + +class AccountLines(Command): + '''Rippled account_lines command''' + def __init__(self, + account: Account, + *, + peer: Optional[Account] = None, + ledger_hash: Optional[str] = None, + ledger_index: Optional[Union[str, int]] = None, + limit: Optional[int] = None, + marker=None): + super().__init__() + self.account = account + self.peer = peer + self.ledger_hash = ledger_hash + self.ledger_index = ledger_index + self.limit = limit + self.marker = marker + assert not ((ledger_hash is not None) and (ledger_index is not None)) + + def cmd_name(self) -> str: + return 'account_lines' + + def get_command_line_list(self) -> List[str]: + '''Return a list of strings suitable for a command line command for a rippled server''' + assert sum(x is None for x in [ + self.ledger_index, self.ledger_hash, self.limit, self.marker + ]) == 4 + result = [self.cmd_name(), self.account.account_id] + if self.peer is not None: + result.append(self.peer) + return result + + def get_websocket_dict(self) -> dict: + '''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket''' + result = {'account': self.account.account_id} + if self.peer is not None: + result['peer'] = self.peer + if self.ledger_index is not None: + result['ledger_index'] = self.ledger_index + if self.ledger_hash is not None: + result['ledger_hash'] = self.ledger_hash + if self.limit is not None: + result['limit'] = self.limit + if self.marker is not None: + result['marker'] = self.marker + return self.add_websocket_fields(result) + + +class AccountTx(Command): + def __init__(self, + account: Account, + *, + limit: Optional[int] = None, + marker=None): + super().__init__() + self.account = account + self.limit = limit + self.marker = marker + + def cmd_name(self) -> str: + return 'account_tx' + + def get_command_line_list(self) -> List[str]: + '''Return a list of strings suitable for a command line command for a rippled server''' + result = [self.cmd_name(), self.account.account_id] + return result + + def get_websocket_dict(self) -> dict: + '''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket''' + result = {'account': self.account.account_id} + if self.limit is not None: + result['limit'] = self.limit + if self.marker is not None: + result['marker'] = self.marker + return self.add_websocket_fields(result) + + +class BookOffers(Command): + '''Rippled book_offers command''' + def __init__(self, + taker_pays: Asset, + taker_gets: Asset, + *, + taker: Optional[Account] = None, + ledger_hash: Optional[str] = None, + ledger_index: Optional[Union[str, int]] = None, + limit: Optional[int] = None, + marker=None): + super().__init__() + self.taker_pays = taker_pays + self.taker_gets = taker_gets + self.taker = taker + self.ledger_hash = ledger_hash + self.ledger_index = ledger_index + self.limit = limit + self.marker = marker + assert not ((ledger_hash is not None) and (ledger_index is not None)) + + def cmd_name(self) -> str: + return 'book_offers' + + def get_command_line_list(self) -> List[str]: + '''Return a list of strings suitable for a command line command for a rippled server''' + assert sum(x is None for x in [ + self.ledger_index, self.ledger_hash, self.limit, self.marker + ]) == 4 + return [ + self.cmd_name(), + self.taker_pays.cmd_str(), + self.taker_gets.cmd_str() + ] + + def get_websocket_dict(self) -> dict: + '''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket''' + result = { + 'taker_pays': self.taker_pays.to_cmd_obj(), + 'taker_gets': self.taker_gets.to_cmd_obj() + } + if self.taker is not None: + result['taker'] = self.taker.account_id + if self.ledger_index is not None: + result['ledger_index'] = self.ledger_index + if self.ledger_hash is not None: + result['ledger_hash'] = self.ledger_hash + if self.limit is not None: + result['limit'] = self.limit + if self.marker is not None: + result['marker'] = self.marker + return self.add_websocket_fields(result) + + +class BookSubscription: + '''Spec for a book in a subscribe command''' + def __init__(self, + taker_pays: Asset, + taker_gets: Asset, + *, + taker: Optional[Account] = None, + snapshot: Optional[bool] = None, + both: Optional[bool] = None): + self.taker_pays = taker_pays + self.taker_gets = taker_gets + self.taker = taker + self.snapshot = snapshot + self.both = both + + def to_cmd_obj(self) -> dict: + '''Return an object suitalbe for use in a command''' + result = { + 'taker_pays': self.taker_pays.to_cmd_obj(), + 'taker_gets': self.taker_gets.to_cmd_obj() + } + if self.taker is not None: + result['taker'] = self.taker.account_id + if self.snapshot is not None: + result['snapshot'] = self.snapshot + if self.both is not None: + result['both'] = self.both + return result + + +class ServerInfo(Command): + '''Rippled server_info command''' + def __init__(self): + super().__init__() + + def cmd_name(self) -> str: + return 'server_info' + + def get_command_line_list(self) -> List[str]: + '''Return a list of strings suitable for a command line command for a rippled server''' + return [self.cmd_name()] + + def get_websocket_dict(self) -> dict: + '''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket''' + result = {} + return self.add_websocket_fields(result) + + +class FederatorInfo(Command): + '''Rippled federator_info command''' + def __init__(self): + super().__init__() + + def cmd_name(self) -> str: + return 'federator_info' + + def get_command_line_list(self) -> List[str]: + '''Return a list of strings suitable for a command line command for a rippled server''' + return [self.cmd_name()] + + def get_websocket_dict(self) -> dict: + '''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket''' + result = {} + return self.add_websocket_fields(result) + + +class Subscribe(SubscriptionCommand): + '''The subscribe method requests periodic notifications from the server + when certain events happen. See: https://developers.ripple.com/subscribe.html''' + def __init__( + self, + *, + streams: Optional[List[str]] = None, + accounts: Optional[List[Account]] = None, + accounts_proposed: Optional[List[Account]] = None, + books: Optional[ + List[BookSubscription]] = None, # taker_pays, taker_gets + url: Optional[str] = None, + url_username: Optional[str] = None, + url_password: Optional[str] = None): + super().__init__() + self.streams = streams + self.accounts = accounts + self.accounts_proposed = accounts_proposed + self.books = books + self.url = url + self.url_username = url_username + self.url_password = url_password + self.websocket = None + + def cmd_name(self) -> str: + if self.websocket: + return 'unsubscribe' + return 'subscribe' + + def to_cmd_obj(self) -> dict: + d = {} + if self.streams is not None: + d['streams'] = self.streams + if self.accounts is not None: + d['accounts'] = [a.account_id for a in self.accounts] + if self.accounts_proposed is not None: + d['accounts_proposed'] = [ + a.account_id for a in self.accounts_proposed + ] + if self.books is not None: + d['books'] = [b.to_cmd_obj() for b in self.books] + if self.url is not None: + d['url'] = self.url + if self.url_username is not None: + d['url_username'] = self.url_username + if self.url_password is not None: + d['url_password'] = self.url_password + return d diff --git a/bin/sidechain/python/common.py b/bin/sidechain/python/common.py new file mode 100644 index 0000000000..a1676781bb --- /dev/null +++ b/bin/sidechain/python/common.py @@ -0,0 +1,250 @@ +import binascii +import datetime +from typing import List, Optional, Union +import pandas as pd +import pytz +import sys + +EPRINT_ENABLED = True + + +def disable_eprint(): + global EPRINT_ENABLED + EPRINT_ENABLED = False + + +def enable_eprint(): + global EPRINT_ENABLED + EPRINT_ENABLED = True + + +def eprint(*args, **kwargs): + if not EPRINT_ENABLED: + return + print(*args, file=sys.stderr, flush=True, **kwargs) + + +def to_rippled_epoch(d: datetime.datetime) -> int: + '''Convert from a datetime to the number of seconds since Jan 1, 2000 (rippled epoch)''' + start = datetime.datetime(2000, 1, 1, tzinfo=pytz.utc) + return int((d - start).total_seconds()) + + +class Account: # pylint: disable=too-few-public-methods + ''' + Account in the ripple ledger + ''' + def __init__(self, + *, + account_id: Optional[str] = None, + nickname: Optional[str] = None, + public_key: Optional[str] = None, + public_key_hex: Optional[str] = None, + secret_key: Optional[str] = None, + result_dict: Optional[dict] = None): + self.account_id = account_id + self.nickname = nickname + self.public_key = public_key + self.public_key_hex = public_key_hex + self.secret_key = secret_key + + if result_dict is not None: + self.account_id = result_dict['account_id'] + self.public_key = result_dict['public_key'] + self.public_key_hex = result_dict['public_key_hex'] + self.secret_key = result_dict['master_key'] + + # Accounts are equal if they represent the same account on the ledger + # I.e. only check the account_id field for equality. + def __eq__(self, lhs): + if not isinstance(lhs, self.__class__): + return False + return self.account_id == lhs.account_id + + def __ne__(self, lhs): + return not self.__eq__(lhs) + + def __str__(self) -> str: + if self.nickname is not None: + return self.nickname + return self.account_id + + def alias_or_account_id(self) -> str: + ''' + return the alias if it exists, otherwise return the id + ''' + if self.nickname is not None: + return self.nickname + return self.account_id + + def account_id_str_as_hex(self) -> str: + return binascii.hexlify(self.account_id.encode()).decode('utf-8') + + def to_cmd_obj(self) -> dict: + return { + 'account_id': self.account_id, + 'nickname': self.nickname, + 'public_key': self.public_key, + 'public_key_hex': self.public_key_hex, + 'secret_key': self.secret_key + } + + +class Asset: + '''An XRP or IOU value''' + def __init__( + self, + *, + value: Union[int, float, None] = None, + currency: Optional[ + str] = None, # Will default to 'XRP' if not specified + issuer: Optional[Account] = None, + from_asset=None, # asset is of type Optional[Asset] + # from_rpc_result is a python object resulting from an rpc command + from_rpc_result: Optional[Union[dict, str]] = None): + + assert from_asset is None or from_rpc_result is None + + self.value = value + self.issuer = issuer + self.currency = currency + if from_asset is not None: + if self.value is None: + self.value = from_asset.value + if self.issuer is None: + self.issuer = from_asset.issuer + if self.currency is None: + self.currency = from_asset.currency + if from_rpc_result is not None: + if isinstance(from_rpc_result, str): + self.value = int(from_rpc_result) + self.currency = 'XRP' + else: + self.value = from_rpc_result['value'] + self.currency = float(from_rpc_result['currency']) + self.issuer = Account(account_id=from_rpc_result['issuer']) + + if self.currency is None: + self.currency = 'XRP' + + if isinstance(self.value, str): + if self.is_xrp(): + self.value = int(value) + else: + self.value = float(value) + + def __call__(self, value: Union[int, float]): + '''Call operator useful for a terse syntax for assets in tests. I.e. USD(100)''' + return Asset(value=value, from_asset=self) + + def __add__(self, lhs): + assert (self.issuer == lhs.issuer and self.currency == lhs.currency) + return Asset(value=self.value + lhs.value, + currency=self.currency, + issuer=self.issuer) + + def __sub__(self, lhs): + assert (self.issuer == lhs.issuer and self.currency == lhs.currency) + return Asset(value=self.value - lhs.value, + currency=self.currency, + issuer=self.issuer) + + def __eq__(self, lhs): + if not isinstance(lhs, self.__class__): + return False + return (self.value == lhs.value and self.currency == lhs.currency + and self.issuer == lhs.issuer) + + def __ne__(self, lhs): + return not self.__eq__(lhs) + + def __str__(self) -> str: + value_part = '' if self.value is None else f'{self.value}/' + issuer_part = '' if self.issuer is None else f'/{self.issuer}' + return f'{value_part}{self.currency}{issuer_part}' + + def __repr__(self) -> str: + return self.__str__() + + def is_xrp(self) -> bool: + ''' return true if the asset represents XRP''' + return self.currency == 'XRP' + + def cmd_str(self) -> str: + value_part = '' if self.value is None else f'{self.value}/' + issuer_part = '' if self.issuer is None else f'/{self.issuer.account_id}' + return f'{value_part}{self.currency}{issuer_part}' + + def to_cmd_obj(self) -> dict: + '''Return an object suitalbe for use in a command''' + if self.currency == 'XRP': + if self.value is not None: + return f'{self.value}' # must be a string + return {'currency': self.currency} + result = {'currency': self.currency, 'issuer': self.issuer.account_id} + if self.value is not None: + result['value'] = f'{self.value}' # must be a string + return result + + +def XRP(v: Union[int, float]) -> Asset: + return Asset(value=v * 1_000_000) + + +class Path: + '''Payment Path''' + def __init__(self, + nodes: Optional[List[Union[Account, Asset]]] = None, + *, + result_list: Optional[List[dict]] = None): + assert not (nodes and result_list) + if result_list is not None: + self.result_list = result_list + return + if nodes is None: + self.result_list = [] + return + self.result_list = [ + self._create_account_path_node(n) + if isinstance(n, Account) else self._create_currency_path_node(n) + for n in nodes + ] + + def _create_account_path_node(self, account: Account) -> dict: + return { + 'account': account.account_id, + 'type': 1, + 'type_hex': '0000000000000001' + } + + def _create_currency_path_node(self, asset: Asset) -> dict: + result = { + 'currency': asset.currency, + 'type': 48, + 'type_hex': '0000000000000030' + } + if not asset.is_xrp(): + result['issuer'] = asset.issuer.account_id + return result + + def to_cmd_obj(self) -> list: + '''Return an object suitalbe for use in a command''' + return self.result_list + + +class PathList: + '''Collection of paths for use in payments''' + def __init__(self, + path_list: Optional[List[Path]] = None, + *, + result_list: Optional[List[List[dict]]] = None): + # result_list can be the response from the rippled server + assert not (path_list and result_list) + if result_list is not None: + self.paths = [Path(result_list=l) for l in result_list] + return + self.paths = path_list + + def to_cmd_obj(self) -> list: + '''Return an object suitalbe for use in a command''' + return [p.to_cmd_obj() for p in self.paths] diff --git a/bin/sidechain/python/config_file.py b/bin/sidechain/python/config_file.py new file mode 100644 index 0000000000..d8c1419266 --- /dev/null +++ b/bin/sidechain/python/config_file.py @@ -0,0 +1,83 @@ +from typing import List, Optional, Tuple + + +class Section: + def section_header(l: str) -> Optional[str]: + ''' + If the line is a section header, return the section name + otherwise return None + ''' + if l.startswith('[') and l.endswith(']'): + return l[1:-1] + return None + + def __init__(self, name: str): + super().__setattr__('_name', name) + # lines contains all non key-value pairs + super().__setattr__('_lines', []) + super().__setattr__('_kv_pairs', {}) + + def get_name(self): + return self._name + + def add_line(self, l): + s = l.split('=') + if len(s) == 2: + self._kv_pairs[s[0].strip()] = s[1].strip() + else: + self._lines.append(l) + + def get_lines(self): + return self._lines + + def get_line(self) -> Optional[str]: + if len(self._lines) > 0: + return self._lines[0] + return None + + def __getattr__(self, name): + return self._kv_pairs[name] + + def __setattr__(self, name, value): + if name in self.__dict__: + super().__setattr__(name, value) + else: + self._kv_pairs[name] = value + + +class ConfigFile: + def __init__(self, *, file_name: Optional[str] = None): + # parse the file + self._file_name = file_name + self._sections = {} + if not file_name: + return + + cur_section = None + with open(file_name) as f: + for n, l in enumerate(f): + l = l.strip() + if l.startswith('#') or not l: + continue + if section_name := Section.section_header(l): + if cur_section: + self.add_section(cur_section) + cur_section = Section(section_name) + continue + if not cur_section: + raise ValueError( + f'Error parsing config file: {file_name} line_num: {n} line: {l}' + ) + cur_section.add_line(l) + + if cur_section: + self.add_section(cur_section) + + def add_section(self, s: Section): + self._sections[s.get_name()] = s + + def get_file_name(self): + return self._file_name + + def __getattr__(self, name): + return self._sections[name] diff --git a/bin/sidechain/python/create_config_files.py b/bin/sidechain/python/create_config_files.py new file mode 100755 index 0000000000..e789ffbaa0 --- /dev/null +++ b/bin/sidechain/python/create_config_files.py @@ -0,0 +1,630 @@ +#!/usr/bin/env python3 + +# Generate rippled config files, each with their own ports, database paths, and validation_seeds. +# There will be configs for shards/no_shards, main/test nets, two config files for each combination +# (so one can run in a dogfood mode while another is tested). To avoid confusion,The directory path +# will be $data_dir/{main | test}.{shard | no_shard}.{dog | test} +# The config file will reside in that directory with the name rippled.cfg +# The validators file will reside in that directory with the name validators.txt +''' +Script to test and debug sidechains. + +The rippled exe location can be set through the command line or +the environment variable RIPPLED_MAINCHAIN_EXE + +The configs_dir (where the config files will reside) can be set through the command line +or the environment variable RIPPLED_SIDECHAIN_CFG_DIR +''' + +import argparse +from dataclasses import dataclass +import json +import os +from pathlib import Path +import sys +from typing import Dict, List, Optional, Tuple, Union + +from config_file import ConfigFile +from command import ValidationCreate, WalletPropose +from common import Account, Asset, eprint, XRP +from app import App, single_client_app + +mainnet_validators = """ +[validator_list_sites] +https://vl.ripple.com + +[validator_list_keys] +ED2677ABFFD1B33AC6FBC3062B71F1E8397C1505E1C42C64D11AD1B28FF73F4734 +""" + +altnet_validators = """ +[validator_list_sites] +https://vl.altnet.rippletest.net + +[validator_list_keys] +ED264807102805220DA0F312E71FC2C69E1552C9C5790F6C25E3729DEB573D5860 +""" + +node_size = 'medium' +default_data_dir = '/home/swd/data/rippled' + + +@dataclass +class Keypair: + public_key: str + secret_key: str + account_id: Optional[str] + + +def generate_node_keypairs(n: int, rip: App) -> List[Keypair]: + ''' + generate keypairs suitable for validator keys + ''' + result = [] + for i in range(n): + keys = rip(ValidationCreate()) + result.append( + Keypair(public_key=keys["validation_public_key"], + secret_key=keys["validation_seed"], + account_id=None)) + return result + + +def generate_federator_keypairs(n: int, rip: App) -> List[Keypair]: + ''' + generate keypairs suitable for federator keys + ''' + result = [] + for i in range(n): + keys = rip(WalletPropose(key_type='ed25519')) + result.append( + Keypair(public_key=keys["public_key"], + secret_key=keys["master_seed"], + account_id=keys["account_id"])) + return result + + +class Ports: + ''' + Port numbers for various services. + Port numbers differ by cfg_index so different configs can run + at the same time without interfering with each other. + ''' + peer_port_base = 51235 + http_admin_port_base = 5005 + ws_public_port_base = 6005 + + def __init__(self, cfg_index: int): + self.peer_port = Ports.peer_port_base + cfg_index + self.http_admin_port = Ports.http_admin_port_base + cfg_index + self.ws_public_port = Ports.ws_public_port_base + (2 * cfg_index) + # note admin port uses public port base + self.ws_admin_port = Ports.ws_public_port_base + (2 * cfg_index) + 1 + + +class Network: + def __init__(self, num_nodes: int, num_validators: int, + start_cfg_index: int, rip: App): + self.validator_keypairs = generate_node_keypairs(num_validators, rip) + self.ports = [Ports(start_cfg_index + i) for i in range(num_nodes)] + + +class SidechainNetwork(Network): + def __init__(self, num_nodes: int, num_federators: int, + num_validators: int, start_cfg_index: int, rip: App): + super().__init__(num_nodes, num_validators, start_cfg_index, rip) + self.federator_keypairs = generate_federator_keypairs( + num_federators, rip) + self.main_account = rip(WalletPropose(key_type='secp256k1')) + + +class XChainAsset: + def __init__(self, main_asset: Asset, side_asset: Asset, + main_value: Union[int, float], side_value: Union[int, float], + main_refund_penalty: Union[int, float], + side_refund_penalty: Union[int, float]): + self.main_asset = main_asset(main_value) + self.side_asset = side_asset(side_value) + self.main_refund_penalty = main_asset(main_refund_penalty) + self.side_refund_penalty = side_asset(side_refund_penalty) + + +def generate_asset_stanzas( + assets: Optional[Dict[str, XChainAsset]] = None) -> str: + if assets is None: + # default to xrp only at a 1:1 value + assets = {} + assets['xrp_xrp_sidechain_asset'] = XChainAsset( + XRP(0), XRP(0), 1, 1, 400, 400) + + index_stanza = """ +[sidechain_assets]""" + + asset_stanzas = [] + + for name, xchainasset in assets.items(): + index_stanza += '\n' + name + new_stanza = f""" +[{name}] +mainchain_asset={json.dumps(xchainasset.main_asset.to_cmd_obj())} +sidechain_asset={json.dumps(xchainasset.side_asset.to_cmd_obj())} +mainchain_refund_penalty={json.dumps(xchainasset.main_refund_penalty.to_cmd_obj())} +sidechain_refund_penalty={json.dumps(xchainasset.side_refund_penalty.to_cmd_obj())}""" + asset_stanzas.append(new_stanza) + + return index_stanza + '\n' + '\n'.join(asset_stanzas) + + +# First element of the returned tuple is the sidechain stanzas +# second element is the bootstrap stanzas +def generate_sidechain_stanza( + mainchain_ports: Ports, + main_account: dict, + federators: List[Keypair], + signing_key: str, + mainchain_cfg_file: str, + xchain_assets: Optional[Dict[str, + XChainAsset]] = None) -> Tuple[str, str]: + mainchain_ip = "127.0.0.1" + + federators_stanza = """ +# federator signing public keys +[sidechain_federators] +""" + federators_secrets_stanza = """ +# federator signing secret keys (for standalone-mode testing only; Normally won't be in a config file) +[sidechain_federators_secrets] +""" + bootstrap_federators_stanza = """ +# first value is federator signing public key, second is the signing pk account +[sidechain_federators] +""" + + assets_stanzas = generate_asset_stanzas(xchain_assets) + + for fed in federators: + federators_stanza += f'{fed.public_key}\n' + federators_secrets_stanza += f'{fed.secret_key}\n' + bootstrap_federators_stanza += f'{fed.public_key} {fed.account_id}\n' + + sidechain_stanzas = f""" +[sidechain] +signing_key={signing_key} +mainchain_account={main_account["account_id"]} +mainchain_ip={mainchain_ip} +mainchain_port_ws={mainchain_ports.ws_public_port} +# mainchain config file is: {mainchain_cfg_file} + +{assets_stanzas} + +{federators_stanza} + +{federators_secrets_stanza} +""" + bootstrap_stanzas = f""" +[sidechain] +mainchain_secret={main_account["master_seed"]} + +{bootstrap_federators_stanza} +""" + return (sidechain_stanzas, bootstrap_stanzas) + + +# cfg_type will typically be either 'dog' or 'test', but can be any string. It is only used +# to create the data directories. +def generate_cfg_dir(*, + ports: Ports, + with_shards: bool, + main_net: bool, + cfg_type: str, + sidechain_stanza: str, + sidechain_bootstrap_stanza: str, + validation_seed: Optional[str] = None, + validators: Optional[List[str]] = None, + fixed_ips: Optional[List[Ports]] = None, + data_dir: str, + full_history: bool = False, + with_hooks: bool = False) -> str: + ips_stanza = '' + this_ip = '127.0.0.1' + if fixed_ips: + ips_stanza = '# Fixed ips for a testnet.\n' + ips_stanza += '[ips_fixed]\n' + for i, p in enumerate(fixed_ips): + if p.peer_port == ports.peer_port: + continue + # rippled limits the number of connects per ip. So use the other loopback devices + ips_stanza += f'127.0.0.{i+1} {p.peer_port}\n' + else: + ips_stanza = '# Where to find some other servers speaking the Ripple protocol.\n' + ips_stanza += '[ips]\n' + if main_net: + ips_stanza += 'r.ripple.com 51235\n' + else: + ips_stanza += 'r.altnet.rippletest.net 51235\n' + disable_shards = '' if with_shards else '# ' + disable_delete = '#' if full_history else '' + history_line = 'full' if full_history else '256' + earliest_seq_line = '' + if sidechain_stanza: + earliest_seq_line = 'earliest_seq=1' + hooks_line = 'Hooks' if with_hooks else '' + validation_seed_stanza = '' + if validation_seed: + validation_seed_stanza = f''' +[validation_seed] +{validation_seed} + ''' + node_size = 'medium' + shard_str = 'shards' if with_shards else 'no_shards' + net_str = 'main' if main_net else 'test' + if not fixed_ips: + sub_dir = data_dir + f'/{net_str}.{shard_str}.{cfg_type}' + if sidechain_stanza: + sub_dir += '.sidechain' + else: + sub_dir = data_dir + f'/{cfg_type}' + db_path = sub_dir + '/db' + debug_logfile = sub_dir + '/debug.log' + shard_db_path = sub_dir + '/shards' + node_db_path = db_path + '/nudb' + + cfg_str = f""" +[server] +port_rpc_admin_local +port_peer +port_ws_admin_local +port_ws_public +#ssl_key = /etc/ssl/private/server.key +#ssl_cert = /etc/ssl/certs/server.crt + +[port_rpc_admin_local] +port = {ports.http_admin_port} +ip = {this_ip} +admin = {this_ip} +protocol = http + +[port_peer] +port = {ports.peer_port} +ip = 0.0.0.0 +protocol = peer + +[port_ws_admin_local] +port = {ports.ws_admin_port} +ip = {this_ip} +admin = {this_ip} +protocol = ws + +[port_ws_public] +port = {ports.ws_public_port} +ip = {this_ip} +protocol = ws +# protocol = wss + +[node_size] +{node_size} + +[ledger_history] +{history_line} + +[node_db] +type=NuDB +path={node_db_path} +open_files=2000 +filter_bits=12 +cache_mb=256 +file_size_mb=8 +file_size_mult=2 +{earliest_seq_line} +{disable_delete}online_delete=256 +{disable_delete}advisory_delete=0 + +[database_path] +{db_path} + +# This needs to be an absolute directory reference, not a relative one. +# Modify this value as required. +[debug_logfile] +{debug_logfile} + +[sntp_servers] +time.windows.com +time.apple.com +time.nist.gov +pool.ntp.org + +{ips_stanza} + +[validators_file] +validators.txt + +[rpc_startup] +{{ "command": "log_level", "severity": "fatal" }} +{{ "command": "log_level", "partition": "SidechainFederator", "severity": "trace" }} + +[ssl_verify] +1 + +{validation_seed_stanza} + +{disable_shards}[shard_db] +{disable_shards}type=NuDB +{disable_shards}path={shard_db_path} +{disable_shards}max_historical_shards=6 + +{sidechain_stanza} + +[features] +{hooks_line} +PayChan +Flow +FlowCross +TickSize +fix1368 +Escrow +fix1373 +EnforceInvariants +SortedDirectories +fix1201 +fix1512 +fix1513 +fix1523 +fix1528 +DepositAuth +Checks +fix1571 +fix1543 +fix1623 +DepositPreauth +fix1515 +fix1578 +MultiSignReserve +fixTakerDryOfferRemoval +fixMasterKeyAsRegularKey +fixCheckThreading +fixPayChanRecipientOwnerDir +DeletableAccounts +fixQualityUpperBound +RequireFullyCanonicalSig +fix1781 +HardenedValidations +fixAmendmentMajorityCalc +NegativeUNL +TicketBatch +FlowSortStrands +fixSTAmountCanonicalize +fixRmSmallIncreasedQOffers +CheckCashMakesTrustLine +""" + + validators_str = '' + for p in [sub_dir, db_path, shard_db_path]: + Path(p).mkdir(parents=True, exist_ok=True) + # Add the validators.txt file + if validators: + validators_str = '[validators]\n' + for k in validators: + validators_str += f'{k}\n' + else: + validators_str = mainnet_validators if main_net else altnet_validators + with open(sub_dir + "/validators.txt", "w") as f: + f.write(validators_str) + + # add the rippled.cfg file + with open(sub_dir + "/rippled.cfg", "w") as f: + f.write(cfg_str) + + if sidechain_bootstrap_stanza: + # add the bootstrap file + with open(sub_dir + "/sidechain_bootstrap.cfg", "w") as f: + f.write(sidechain_bootstrap_stanza) + + return sub_dir + "/rippled.cfg" + + +def generate_multinode_net(out_dir: str, + mainnet: Network, + sidenet: SidechainNetwork, + xchain_assets: Optional[Dict[str, + XChainAsset]] = None): + mainnet_cfgs = [] + for i in range(len(mainnet.ports)): + validator_kp = mainnet.validator_keypairs[i] + ports = mainnet.ports[i] + mainchain_cfg_file = generate_cfg_dir( + ports=ports, + with_shards=False, + main_net=True, + cfg_type=f'mainchain_{i}', + sidechain_stanza='', + sidechain_bootstrap_stanza='', + validation_seed=validator_kp.secret_key, + data_dir=out_dir) + mainnet_cfgs.append(mainchain_cfg_file) + + for i in range(len(sidenet.ports)): + validator_kp = sidenet.validator_keypairs[i] + ports = sidenet.ports[i] + + mainnet_i = i % len(mainnet.ports) + sidechain_stanza, sidechain_bootstrap_stanza = generate_sidechain_stanza( + mainnet.ports[mainnet_i], sidenet.main_account, + sidenet.federator_keypairs, + sidenet.federator_keypairs[i].secret_key, mainnet_cfgs[mainnet_i], + xchain_assets) + + generate_cfg_dir( + ports=ports, + with_shards=False, + main_net=True, + cfg_type=f'sidechain_{i}', + sidechain_stanza=sidechain_stanza, + sidechain_bootstrap_stanza=sidechain_bootstrap_stanza, + validation_seed=validator_kp.secret_key, + validators=[kp.public_key for kp in sidenet.validator_keypairs], + fixed_ips=sidenet.ports, + data_dir=out_dir, + full_history=True, + with_hooks=False) + + +def parse_args(): + parser = argparse.ArgumentParser( + description=('Create config files for testing sidechains')) + + parser.add_argument( + '--exe', + '-e', + help=('path to rippled executable'), + ) + + parser.add_argument( + '--usd', + '-u', + action='store_true', + help=('include a USD/root IOU asset for cross chain transfers'), + ) + + parser.add_argument( + '--cfgs_dir', + '-c', + help= + ('path to configuration file dir (where the output config files will be located)' + ), + ) + + return parser.parse_known_args()[0] + + +class Params: + def __init__(self): + args = parse_args() + + self.exe = None + if 'RIPPLED_MAINCHAIN_EXE' in os.environ: + self.exe = os.environ['RIPPLED_MAINCHAIN_EXE'] + if args.exe: + self.exe = args.exe + + self.configs_dir = None + if 'RIPPLED_SIDECHAIN_CFG_DIR' in os.environ: + self.configs_dir = os.environ['RIPPLED_SIDECHAIN_CFG_DIR'] + if args.cfgs_dir: + self.configs_dir = args.cfgs_dir + + self.usd = False + if args.usd: + self.usd = args.usd + + def check_error(self) -> str: + ''' + Check for errors. Return `None` if no errors, + otherwise return a string describing the error + ''' + if not self.exe: + return 'Missing exe location. Either set the env variable RIPPLED_MAINCHAIN_EXE or use the --exe_mainchain command line switch' + if not self.configs_dir: + return 'Missing configs directory location. Either set the env variable RIPPLED_SIDECHAIN_CFG_DIR or use the --cfgs_dir command line switch' + + +def main(params: Params, + xchain_assets: Optional[Dict[str, XChainAsset]] = None): + + if err_str := params.check_error(): + eprint(err_str) + sys.exit(1) + index = 0 + nonvalidator_cfg_file_name = generate_cfg_dir( + ports=Ports(index), + with_shards=False, + main_net=True, + cfg_type='non_validator', + sidechain_stanza='', + sidechain_bootstrap_stanza='', + validation_seed=None, + data_dir=params.configs_dir) + index = index + 1 + + nonvalidator_config = ConfigFile(file_name=nonvalidator_cfg_file_name) + with single_client_app(exe=params.exe, + config=nonvalidator_config, + standalone=True) as rip: + mainnet = Network(num_nodes=1, + num_validators=1, + start_cfg_index=index, + rip=rip) + sidenet = SidechainNetwork(num_nodes=5, + num_federators=5, + num_validators=5, + start_cfg_index=index + 1, + rip=rip) + generate_multinode_net( + out_dir=f'{params.configs_dir}/sidechain_testnet', + mainnet=mainnet, + sidenet=sidenet, + xchain_assets=xchain_assets) + index = index + 2 + + (Path(params.configs_dir) / 'logs').mkdir(parents=True, exist_ok=True) + + for with_shards in [True, False]: + for is_main_net in [True, False]: + for cfg_type in ['dog', 'test', 'one', 'two']: + if not is_main_net and cfg_type not in ['dog', 'test']: + continue + + mainnet = Network(num_nodes=1, + num_validators=1, + start_cfg_index=index, + rip=rip) + mainchain_cfg_file = generate_cfg_dir( + data_dir=params.configs_dir, + ports=mainnet.ports[0], + with_shards=with_shards, + main_net=is_main_net, + cfg_type=cfg_type, + sidechain_stanza='', + sidechain_bootstrap_stanza='', + validation_seed=mainnet.validator_keypairs[0]. + secret_key) + + sidenet = SidechainNetwork(num_nodes=1, + num_federators=5, + num_validators=1, + start_cfg_index=index + 1, + rip=rip) + signing_key = sidenet.federator_keypairs[0].secret_key + + sidechain_stanza, sizechain_bootstrap_stanza = generate_sidechain_stanza( + mainnet.ports[0], sidenet.main_account, + sidenet.federator_keypairs, signing_key, + mainchain_cfg_file, xchain_assets) + + generate_cfg_dir( + data_dir=params.configs_dir, + ports=sidenet.ports[0], + with_shards=with_shards, + main_net=is_main_net, + cfg_type=cfg_type, + sidechain_stanza=sidechain_stanza, + sidechain_bootstrap_stanza=sizechain_bootstrap_stanza, + validation_seed=sidenet.validator_keypairs[0]. + secret_key) + index = index + 2 + + +if __name__ == '__main__': + params = Params() + + xchain_assets = None + if params.usd: + xchain_assets = {} + xchain_assets['xrp_xrp_sidechain_asset'] = XChainAsset( + XRP(0), XRP(0), 1, 1, 200, 200) + root_account = Account(account_id="rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh") + main_iou_asset = Asset(value=0, currency='USD', issuer=root_account) + side_iou_asset = Asset(value=0, currency='USD', issuer=root_account) + xchain_assets['iou_iou_sidechain_asset'] = XChainAsset( + main_iou_asset, side_iou_asset, 1, 1, 0.02, 0.02) + + main(params, xchain_assets) diff --git a/bin/sidechain/python/interactive.py b/bin/sidechain/python/interactive.py new file mode 100644 index 0000000000..b60acce9c3 --- /dev/null +++ b/bin/sidechain/python/interactive.py @@ -0,0 +1,1496 @@ +import binascii +import cmd +import json +import os +import pandas as pd +from pathlib import Path +import pprint +import time +from typing import Callable, Dict, List, Optional, Union + +from app import App, balances_dataframe +from command import AccountTx, Subscribe +from common import Account, Asset, XRP +from transaction import SetHook, Payment, Trust + + +def clear_screen(): + if os.name == 'nt': + _ = os.system('cls') + else: + _ = os.system('clear') + + +# Directory to find hooks. The hook should be in a directory call "hook_name" +# in a file called "hook_name.wasm" +HOOKS_DIR = Path() + + +def set_hooks_dir(n: str): + global HOOKS_DIR + if n: + HOOKS_DIR = Path(n) + + +_valid_hook_names = ['doubler', 'notascam'] + + +def _file_to_hex(filename: Path) -> str: + with open(filename, 'rb') as f: + content = f.read() + return binascii.hexlify(content).decode('utf8') + + +def _removesuffix(self: str, suffix: str) -> str: + if suffix and self.endswith(suffix): + return self[:-len(suffix)] + else: + return self[:] + + +class SidechainRepl(cmd.Cmd): + ''' + Simple repl for interacting with side chains + ''' + intro = '\n\nWelcome to the sidechain test shell. Type help or ? to list commands.\n' + prompt = 'RiplRepl> ' + + def preloop(self): + clear_screen() + + def __init__(self, mc_app: App, sc_app: App): + super().__init__() + assert mc_app.is_alias('door') and sc_app.is_alias('door') + self.mc_app = mc_app + self.sc_app = sc_app + + def _complete_chain(self, text, line): + if not text: + return ['mainchain', 'sidechain'] + else: + return [ + c for c in ['mainchain', 'sidechain'] if c.startswith(text) + ] + + def _complete_unit(self, text, line): + if not text: + return ['drops', 'xrp'] + else: + return [c for c in ['drops', 'xrp'] if c.startswith(text)] + + def _complete_account(self, text, line, chain_name=None): + known_accounts = set() + chains = [self.mc_app, self.sc_app] + if chain_name == 'mainchain': + chains = [self.mc_app] + elif chain_name == 'sidechain': + chains = [self.sc_app] + for chain in chains: + known_accounts = known_accounts | set( + [a.nickname for a in chain.known_accounts()]) + if not text: + return list(known_accounts) + else: + return [c for c in known_accounts if c.startswith(text)] + + def _complete_asset(self, text, line, chain_name=None): + known_assets = set() + chains = [self.mc_app, self.sc_app] + if chain_name == 'mainchain': + chains = [self.mc_app] + elif chain_name == 'sidechain': + chains = [self.sc_app] + for chain in chains: + known_assets = known_assets | set(chain.known_asset_aliases()) + if not text: + return list(known_assets) + else: + return [c for c in known_assets if c.startswith(text)] + + ################## + # addressbook + def do_addressbook(self, line): + def print_addressbook(chain: App, chain_name: str, + nickname: Optional[str]): + if nickname and not chain.is_alias(nickname): + print( + f"{nickname} is not part of {chain_name}'s address book.") + print(f'{chain_name}:\n{chain.key_manager.to_string(nickname)}') + + args = line.split() + if len(args) > 2: + print( + f'Error: Too many arguments to addressbook command. Type "help" for help.' + ) + return + + chains = [self.mc_app, self.sc_app] + chain_names = ['mainchain', 'sidechain'] + nickname = None + + if args and args[0] in ['mainchain', 'sidechain']: + chain_names = [args[0]] + if args[0] == 'mainchain': + chains = [self.mc_app] + else: + chains = [self.sc_app] + args.pop(0) + + if args: + nickname = args[0] + + for chain, name in zip(chains, chain_names): + print_addressbook(chain, name, nickname) + print('\n') + + def complete_addressbook(self, text, line, begidx, endidx): + args = line.split() + arg_num = len(args) + if arg_num == 2: # chain + return self._complete_chain(text, line) + self._complete_account( + text, line) + if arg_num == 3: # account + return self._complete_account(text, line, chain_name=args[1]) + return [] + + def help_addressbook(self): + print('\n'.join([ + 'addressbook [mainchain | sidechain] [account]', + 'Show the address book for the specified chain and account.', + 'If a chain is not specified, show both address books.', + 'If the account is not specified, show all addresses.', '' + ])) + + # addressbook + ################## + + ################## + # balance + def do_balance(self, line): + args = line.split() + if len(args) > 3: + print( + f'Error: Too many arguments to balance command. Type "help" for help.' + ) + return + + in_drops = False + if args and args[-1] in ['xrp', 'drops']: + unit = args[-1] + args.pop() + if unit == 'xrp': + in_drops = False + elif unit == 'drops': + in_drops = True + + chains = [self.mc_app, self.sc_app] + chain_names = ['mainchain', 'sidechain'] + if args and args[0] in ['mainchain', 'sidechain']: + chain_names = [args[0]] + args.pop(0) + if chain_names[0] == 'mainchain': + chains = [self.mc_app] + else: + chains = [self.sc_app] + + account_ids = [None] * len(chains) + if args: + nickname = args[0] + args.pop() + account_ids = [] + for c in chains: + if not c.is_alias(nickname): + print(f'Error: {nickname} is not in the address book') + return + account_ids.append(c.account_from_alias(nickname)) + + assets = [[XRP(0)]] * len(chains) + if args: + asset_alias = args[0] + args.pop() + if len(chains) != 1: + print( + f'Error: iou assets can only be shown for a single chain at a time' + ) + return + if not chains[0].is_asset_alias(asset_alias): + print(f'Error: {asset_alias} is not a valid asset alias') + return + assets = [[chains[0].asset_from_alias(asset_alias)]] + else: + # XRP and all assets in the assets alias list + assets = [[XRP(0)] + c.known_iou_assets() for c in chains] + + assert not args + + df = balances_dataframe(chains, chain_names, account_ids, assets, + in_drops) + df_as_str = df.to_string(float_format=lambda x: f'{x:,.6f}') + print(f'{df_as_str}\n') + + def complete_balance(self, text, line, begidx, endidx): + args = line.split() + arg_num = len(args) + if arg_num == 2: # chain or account + return self._complete_chain(text, line) + self._complete_account( + text, line) + elif arg_num == 3: # account or unit or asset_alias + return self._complete_account(text, line) + self._complete_unit( + text, line, chain_name=args[1]) + self._complete_asset( + text, line, chain_name=args[1]) + elif arg_num == 4: # unit + return self._complete_unit(text, line) + self._complete_asset( + text, line, chain_name=args[1]) + return [] + + def help_balance(self): + print('\n'.join([ + f'balance [sidechain | mainchain] [account_name] [xrp | drops | asset_alias]', + 'Show the balance the specified account.' + 'If no account is specified, show the balance for all accounts in the addressbook.', + 'If no chain is specified, show the balances for both chains.', '' + 'If no asset alias is specified, show balances for all known asset aliases.' + ])) + + # balance + ################## + + ################## + # account_info + def _account_info_df(self, chain: App, acc: Optional[Account]): + b = chain.get_account_info(acc) + b = chain.substitute_nicknames(b) + b = b.set_index('account') + return b + + def do_account_info(self, line): + args = line.split() + if len(args) > 2: + print( + f'Error: Too many arguments to account_info command. Type "help" for help.' + ) + return + chains = [self.mc_app, self.sc_app] + chain_names = ['mainchain', 'sidechain'] + if args and args[0] in ['mainchain', 'sidechain']: + chain_names = [args[0]] + args.pop(0) + if chain_names[0] == 'mainchain': + chains = [self.mc_app] + else: + chains = [self.sc_app] + + account_ids = [None] * len(chains) + if args: + nickname = args[0] + args.pop() + account_ids = [] + for c in chains: + if not c.is_alias(nickname): + print(f'Error: {nickname} is not in the address book') + return + account_ids.append(c.account_from_alias(nickname)) + + assert not args + + dfs = [] + keys = [] + for chain, chain_name, acc in zip(chains, chain_names, account_ids): + dfs.append(self._account_info_df(chain, acc)) + keys.append(_removesuffix(chain_name, 'chain')) + df = pd.concat(dfs, keys=keys) + df_as_str = df.to_string(float_format=lambda x: f'{x:,.6f}') + print(f'{df_as_str}\n') + + def complete_account_info(self, text, line, begidx, endidx): + args = line.split() + arg_num = len(args) + if arg_num == 2: # chain or account + return self._complete_chain(text, line) + self._complete_account( + text, line) + elif arg_num == 3: # account + return self._complete_account(text, line) + return [] + + def help_account_info(self): + print('\n'.join([ + f'account_info [sidechain | mainchain] [account_name]', + 'Show the account_info the specified account.' + 'If no account is specified, show the account_info for all accounts in the addressbook.', + 'If no chain is specified, show the account_info for both chains.', + ])) + + # account_info + ################## + + ################## + # pay + def do_pay(self, line): + args = line.split() + if len(args) < 4: + print( + f'Error: Too few arguments to pay command. Type "help" for help.' + ) + return + + if len(args) > 5: + print( + f'Error: Too many arguments to pay command. Type "help" for help.' + ) + return + + in_drops = False + if args and args[-1] in ['xrp', 'drops']: + unit = args[-1] + if unit == 'xrp': + in_drops = False + elif unit == 'drops': + in_drops = True + args.pop() + + chain = None + if args[0] not in ['mainchain', 'sidechain']: + print( + f'Error: First argument must specify the chain. Type "help" for help.' + ) + return + + if args[0] == 'mainchain': + chain = self.mc_app + else: + chain = self.sc_app + args.pop(0) + + nickname = args[0] + if nickname == 'door': + print( + f'Error: The "door" account should never be used as a source of payments.' + ) + return + if not chain.is_alias(nickname): + print(f'Error: {nickname} is not in the address book') + return + src_account = chain.account_from_alias(nickname) + args.pop(0) + + nickname = args[0] + if nickname == 'door': + print( + f'Error: "pay" cannot be used for cross chain transactions. Use the "xchain" command instead.' + ) + return + if not chain.is_alias(nickname): + print(f'Error: {nickname} is not in the address book') + return + dst_account = chain.account_from_alias(nickname) + args.pop(0) + + amt_value = None + try: + amt_value = int(args[0]) + except: + try: + if not in_drops: + amt_value = float(args[0]) + except: + pass + + if amt_value is None: + print(f'Error: {args[0]} is an invalid amount.') + return + args.pop(0) + + asset = Asset(value=0) + + if args: + asset_alias = args[0] + args.pop(0) + if not chain.is_asset_alias(asset_alias): + print(f'Error: {args[0]} is an invalid asset alias.') + return + asset = chain.asset_from_alias(asset_alias) + + assert not args + + if asset.is_xrp() and not in_drops: + amt_value *= 1_000_000 + + amt = asset(value=amt_value) + + chain(Payment(account=src_account, dst=dst_account, amt=amt)) + chain.maybe_ledger_accept() + + def complete_pay(self, text, line, begidx, endidx): + args = line.split() + arg_num = len(args) + if not text: + arg_num += 1 + if arg_num == 2: # chain + return self._complete_chain(text, line) + elif arg_num == 3: # account + return self._complete_account(text, line, chain_name=args[1]) + elif arg_num == 4: # account + return self._complete_account(text, line, chain_name=args[1]) + elif arg_num == 5: # amount + completions = [] + elif arg_num == 6: # drops or xrp or asset + return self._complete_unit(text, line) + self._complete_asset( + text, line, chain_name=args[1]) + return [] + + def help_pay(self): + print('\n'.join([ + f'pay (sidechain | mainchain) src_account dst_account amount [xrp | drops | iou_alias]', + 'Send xrp from the src account to the dst account.' + 'Note: the door account can not be used as the src or dst.', + 'Cross chain transactions should use the xchain command instead of this.', + '' + ])) + + # pay + ################## + + ################## + # xchain + def do_xchain(self, line): + args = line.split() + if len(args) < 4: + print( + f'Error: Too few arguments to pay command. Type "help" for help.' + ) + return + + if len(args) > 5: + print( + f'Error: Too many arguments to pay command. Type "help" for help.' + ) + return + + in_drops = False + if args and args[-1] in ['xrp', 'drops']: + unit = args[-1] + if unit == 'xrp': + in_drops = False + elif unit == 'drops': + in_drops = True + args.pop() + + chain = None + if args[0] not in ['mainchain', 'sidechain']: + print( + f'Error: First argument must specify the chain. Type "help" for help.' + ) + return + + if args[0] == 'mainchain': + chain = self.mc_app + other_chain = self.sc_app + else: + chain = self.sc_app + other_chain = self.mc_app + args.pop(0) + + nickname = args[0] + if nickname == 'door': + print( + f'Error: The "door" account can not be used as the source of cross chain funds.' + ) + return + if not chain.is_alias(nickname): + print(f'Error: {nickname} is not in the address book') + return + src_account = chain.account_from_alias(nickname) + args.pop(0) + + nickname = args[0] + if nickname == 'door': + print( + f'Error: The "door" account can not be used as the destination of cross chain funds.' + ) + return + if not other_chain.is_alias(nickname): + print(f'Error: {nickname} is not in the address book') + return + dst_account = other_chain.account_from_alias(nickname) + args.pop(0) + + amt_value = None + try: + amt_value = int(args[0]) + except: + try: + if not in_drops: + amt_value = float(args[0]) + except: + pass + + if amt_value is None: + print(f'Error: {args[0]} is an invalid amount.') + return + args.pop(0) + + asset = Asset(value=0) + + if args: + asset_alias = args[0] + args.pop(0) + if not chain.is_asset_alias(asset_alias): + print(f'Error: {asset_alias} is an invalid asset alias.') + return + asset = chain.asset_from_alias(asset_alias) + + assert not args + + if asset.is_xrp() and not in_drops: + amt_value *= 1_000_000 + + amt = asset(value=amt_value) + + assert not args + memos = [{'Memo': {'MemoData': dst_account.account_id_str_as_hex()}}] + door_account = chain.account_from_alias('door') + chain( + Payment(account=src_account, + dst=door_account, + amt=amt, + memos=memos)) + chain.maybe_ledger_accept() + if other_chain.standalone: + # from_chain (side chain) sends a txn, but won't close the to_chain (main chain) ledger + time.sleep(2) + other_chain.maybe_ledger_accept() + + def complete_xchain(self, text, line, begidx, endidx): + args = line.split() + arg_num = len(args) + if not text: + arg_num += 1 + if arg_num == 2: # chain + return self._complete_chain(text, line) + elif arg_num == 3: # this chain account + return self._complete_account(text, line, chain_name=args[1]) + elif arg_num == 4: # other chain account + other_chain_name = None + if args[1] == 'mainchain': + other_chain_name = 'sidechain' + if args[1] == 'sidechain': + other_chain_name = 'mainchain' + return self._complete_account(text, + line, + chain_name=other_chain_name) + elif arg_num == 5: # amount + completions = [] + elif arg_num == 6: # drops or xrp or asset + return self._complete_unit(text, line) + self._complete_asset( + test, line, chain_name=args[1]) + return [] + + def help_xchain(self): + print('\n'.join([ + f'xchain (sidechain | mainchain) this_chain_account other_chain_account amount [xrp | drops | iou_alias]', + 'Send xrp from the specified chain to the other chain.' + 'Note: the door account can not be used as the account.', '' + ])) + + # xchain + ################## + + ################## + # server_info + def do_server_info(self, line): + def data_dict(chain: App, chain_name: str): + file_names = [c.get_file_name() for c in chain.get_configs()] + data = { + 'pid': chain.get_pids(), + 'config': file_names, + 'running': chain.get_running_status() + } + bsi = chain.get_brief_server_info() + data.update(bsi) + df = pd.DataFrame(data=data) + indexes = [[], []] + for i in range(len(file_names)): + indexes[0].append(chain_name) + indexes[1].append(i) + data['indexes'] = indexes + return data + + def df_from_dicts(d1: dict, d2: Optional[dict] = None) -> pd.DataFrame: + indexes = [[], []] + for i in range(2): + if d2: + indexes[i] = d1['indexes'][i] + d2['indexes'][i] + else: + indexes[i] = d1['indexes'][i] + data = {} + for k in d1.keys(): + if k == 'indexes': continue + if d2: + data[k] = d1[k] + d2[k] + else: + data[k] = d1[k] + if k == 'config': + # save space by omitting the common prefix on the configs + cp = os.path.commonprefix(data[k]) + data[k] = [os.path.relpath(f, cp) for f in data[k]] + return pd.DataFrame(data=data, index=indexes) + + args = line.split() + if len(args) > 1: + print( + f'Error: Too many arguments to server_info command. Type "help" for help.' + ) + return + + chains = [self.mc_app, self.sc_app] + chain_names = ['mainchain', 'sidechain'] + + if args and args[0] in ['mainchain', 'sidechain']: + chain_names = [args[0]] + if args[0] == 'mainchain': + chains = [self.mc_app] + else: + chains = [self.sc_app] + args.pop(0) + + data_dicts = [ + data_dict(chain, _removesuffix(name, 'chain')) + for chain, name in zip(chains, chain_names) + ] + df = df_from_dicts(*data_dicts) + print(f'{df.to_string(index=True)}') + + def complete_server_info(self, text, line, begidx, endidx): + arg_num = len(line.split()) + if arg_num == 2: # chain + return self._complete_chain(text, line) + return [] + + def help_server_info(self): + print('\n'.join([ + 'server_info [mainchain | sidechain]', + 'Show the process ids and config files for the rippled servers running for the specified chain.', + 'If a chain is not specified, show info for both chains.', + ])) + + # server_info + ################## + + ################## + # federator_info + + def do_federator_info(self, line): + args = line.split() + indexes = set() + verbose = False + raw = False + while args and (args[-1] == 'verbose' or args[-1] == 'raw'): + if args[-1] == 'verbose': + verbose = True + if args[-1] == 'raw': + raw = True + args.pop() + + try: + for i in args: + indexes.add(int(i)) + except: + f'Error: federator_info bad arguments: {args}. Type "help" for help.' + + def global_df(info_dict: dict) -> pd.DataFrame: + indexes = [] + keys = [] + mc_last_sent_seq = [] + mc_seq = [] + mc_num_pending = [] + mc_sync_state = [] + sc_last_sent_seq = [] + sc_seq = [] + sc_num_pending = [] + sc_sync_state = [] + for (k, v) in info_dict.items(): + indexes.append(k) + info = v['info'] + keys.append(info['public_key']) + mc = info['mainchain'] + sc = info['sidechain'] + mc_last_sent_seq.append(mc['last_transaction_sent_seq']) + sc_last_sent_seq.append(sc['last_transaction_sent_seq']) + mc_seq.append(mc['sequence']) + sc_seq.append(sc['sequence']) + mc_num_pending.append(len(mc['pending_transactions'])) + sc_num_pending.append(len(sc['pending_transactions'])) + if 'state' in mc['listener_info']: + mc_sync_state.append(mc['listener_info']['state']) + else: + mc_sync_state.append(None) + if 'state' in sc['listener_info']: + sc_sync_state.append(sc['listener_info']['state']) + else: + sc_sync_state.append(None) + + data = { + ('key', ''): keys, + ('mainchain', 'last_sent_seq'): mc_last_sent_seq, + ('mainchain', 'seq'): mc_seq, + ('mainchain', 'num_pending'): mc_num_pending, + ('mainchain', 'sync_state'): mc_sync_state, + ('sidechain', 'last_sent_seq'): sc_last_sent_seq, + ('sidechain', 'seq'): sc_seq, + ('sidechain', 'num_pending'): sc_num_pending, + ('sidechain', 'sync_state'): sc_sync_state + } + return pd.DataFrame(data=data, index=indexes) + + def pending_df(info_dict: dict, verbose=False) -> pd.DataFrame: + indexes = [[], []] + amounts = [] + dsts = [] + num_sigs = [] + hashes = [] + signatures = [] + for (k, v) in info_dict.items(): + for chain in ['mainchain', 'sidechain']: + info = v['info'][chain] + pending = info['pending_transactions'] + idx = (k, chain) + for t in pending: + amt = t['amount'] + try: + amt = int(amt) / 1_000_000.0 + except: + pass + dst = t['destination_account'] + h = t['hash'] + ns = len(t['signatures']) + if not verbose: + indexes[0].append(idx[0]) + indexes[1].append(idx[1]) + amounts.append(amt) + dsts.append(dst) + hashes.append(h) + num_sigs.append(ns) + else: + for sig in t['signatures']: + indexes[0].append(idx[0]) + indexes[1].append(idx[1]) + amounts.append(amt) + dsts.append(dst) + hashes.append(h) + num_sigs.append(ns) + signatures.append(sig['public_key']) + + data = { + 'amount': amounts, + 'dest_account': dsts, + 'num_sigs': num_sigs, + 'hash': hashes + } + if verbose: + data['sigs'] = signatures + return pd.DataFrame(data=data, index=indexes) + + info_dict = self.sc_app.federator_info(indexes) + if raw: + pprint.pprint(info_dict) + return + + gdf = global_df(info_dict) + print(gdf) + # pending + print() + pdf = pending_df(info_dict, verbose) + print(pdf) + + def complete_federator_info(self, text, line, begidx, endidx): + args = line.split() + if 'verbose'.startswith(args[-1]): + return ['verbose'] + if 'raw'.startswith(args[-1]): + return ['raw'] + running_status = sc_app.get_running_status() + return [ + str(i) for i in range(0, len(sc_app.get_running_status())) + if running_status[i] + ] + + def help_federator_info(self): + print('\n'.join([ + 'federator_info [server_index...] [verbose | raw]', + 'Show the state of the federators queues and startup synchronization.', + 'If a server index is not specified, show info for all running federators.', + ])) + + # federator_info + ################## + + ################## + # new_account + def do_new_account(self, line): + args = line.split() + if len(args) < 2: + print( + f'Error: new_account command takes at least two arguments. Type "help" for help.' + ) + return + + chain = None + + if args[0] not in ['mainchain', 'sidechain']: + print( + f'Error: The first argument must be "mainchain" or "sidechain".' + ) + return + + if args[0] == 'mainchain': + chain = self.mc_app + else: + chain = self.sc_app + args.pop(0) + + for alias in args: + if chain.is_alias(alias): + print(f'Warning: The alias {alias} already exists.') + else: + chain.create_account(alias) + + def complete_new_account(self, text, line, begidx, endidx): + arg_num = len(line.split()) + if arg_num == 2: # chain + return self._complete_chain(text, line) + return [] + + def help_new_account(self): + print('\n'.join([ + 'new_account (mainchain | sidechain) alias [alias...]', + 'Add a new account to the address book', + ])) + + # new_account + ################## + + ################## + # new_iou + def do_new_iou(self, line): + args = line.split() + if len(args) != 4: + print( + f'Error: new_iou command takes exactly four arguments. Type "help" for help.' + ) + return + + chain = None + + if args[0] not in ['mainchain', 'sidechain']: + print( + f'Error: The first argument must be "mainchain" or "sidechain".' + ) + return + + if args[0] == 'mainchain': + chain = self.mc_app + else: + chain = self.sc_app + args.pop(0) + + (alias, currency, issuer) = args + + if chain.is_asset_alias(alias): + print(f'Error: The alias {alias} already exists.') + return + + if not chain.is_alias(issuer): + print( + f'Error: The issuer {issuer} is not part of the address book.') + return + + asset = Asset(value=0, + currency=currency, + issuer=chain.account_from_alias(issuer)) + chain.add_asset_alias(asset, alias) + + def complete_new_iou(self, text, line, begidx, endidx): + arg_num = len(line.split()) + if arg_num == 2: # chain + return self._complete_chain(text, line) + if arg_num == 5: # issuer + return self._complete_account(text, line) + return [] + + def help_new_iou(self): + print('\n'.join([ + 'new_iou (mainchain | sidechain) alias currency issuer', + 'Add a new iou alias', + ])) + + # new_iou + ################## + + ################## + # ious + def do_ious(self, line): + def print_ious(chain: App, chain_name: str, nickname: Optional[str]): + if nickname and not chain.is_asset_alias(nickname): + print( + f"{nickname} is not part of {chain_name}'s asset aliases.") + print(f'{chain_name}:\n{chain.asset_aliases.to_string(nickname)}') + + args = line.split() + if len(args) > 2: + print( + f'Error: Too many arguments to ious command. Type "help" for help.' + ) + return + + chains = [self.mc_app, self.sc_app] + chain_names = ['mainchain', 'sidechain'] + nickname = None + + if args and args[0] in ['mainchain', 'sidechain']: + chain_names = [args[0]] + if args[0] == 'mainchain': + chains = [self.mc_app] + else: + chains = [self.sc_app] + args.pop(0) + + if args: + nickname = args[0] + + for chain, name in zip(chains, chain_names): + print_ious(chain, name, nickname) + print('\n') + + def complete_ious(self, text, line, begidx, endidx): + args = line.split() + arg_num = len(args) + if arg_num == 2: # chain or iou + return self._complete_chain(text, line) + self._complete_asset( + text, line) + if arg_num == 3: # iou + return self._complete_asset(text, line, chain_name=args[1]) + return [] + + def help_ious(self): + print('\n'.join([ + 'ious [mainchain | sidechain] [alias]', + 'Show the iou aliases for the specified chain and alias.', + 'If a chain is not specified, show aliases for both chains.', + 'If the alias is not specified, show all aliases.', '' + ])) + + # ious + ################## + + ################## + # set_trust + def do_set_trust(self, line): + args = line.split() + if len(args) != 4: + print( + f'Error: set_trust command takes exactly four arguments. Type "help" for help.' + ) + return + + chain = None + + if args[0] not in ['mainchain', 'sidechain']: + print( + f'Error: The first argument must be "mainchain" or "sidechain".' + ) + return + + if args[0] == 'mainchain': + chain = self.mc_app + else: + chain = self.sc_app + args.pop(0) + + (alias, accountStr, amountStr) = args + + if not chain.is_asset_alias(alias): + print(f'Error: The alias {alias} does not exists.') + return + + if not chain.is_alias(accountStr): + print( + f'Error: The issuer {issuer} is not part of the address book.') + return + + account = chain.account_from_alias(accountStr) + + amount = None + try: + amount = int(amountStr) + except: + try: + amount = float(amountStr) + except: + pass + + if amount is None: + print(f'Error: Invalid amount {amountStr}') + return + + asset = chain.asset_from_alias(alias)(amount) + chain(Trust(account=account, limit_amt=asset)) + chain.maybe_ledger_accept() + + def complete_set_trust(self, text, line, begidx, endidx): + args = line.split() + arg_num = len(args) + if arg_num == 2: # chain + return self._complete_chain(text, line) + if arg_num == 3: # iou + return self._complete_asset(text, line, chain_name=args[1]) + if arg_num == 4: # account + return self._complete_account(text, line, chain_name=args[1]) + return [] + + def help_set_trust(self): + print('\n'.join([ + 'set_trust (mainchain | sidechain) iou_alias account amount', + "Set trust amount for account's side of the iou trust line to amount", + ])) + + # set_trust + ################## + + ################## + # ledger_accept + def do_ledger_accept(self, line): + args = line.split() + if len(args) != 1: + print( + f'Error: ledger_accept command takes exactly one argument. Type "help" for help.' + ) + return + + chain = None + + if args[0] not in ['mainchain', 'sidechain']: + print( + f'Error: The first argument must be "mainchain" or "sidechain".' + ) + return + + if args[0] == 'mainchain': + chain = self.mc_app + else: + chain = self.sc_app + args.pop(0) + + assert not args + + chain.maybe_ledger_accept() + + def complete_ledger_accept(self, text, line, begidx, endidx): + arg_num = len(line.split()) + if arg_num == 2: # chain + return self._complete_chain(text, line) + return [] + + def help_ledger_accept(self): + print('\n'.join([ + 'ledger_accept (mainchain | sidechain)', + 'Force a ledger_accept if the chain is running in stand alone mode.', + ])) + + # ledger_accept + ################## + + ################## + # server_start + + def do_server_start(self, line): + args = line.split() + if len(args) == 0: + print( + f'Error: server_start command takes one or more arguments. Type "help" for help.' + ) + return + indexes = set() + if len(args) == 1 and args[0] == 'all': + # re-start all stopped servers + running_status = self.sc_app.get_running_status() + for (i, running) in enumerate(running_status): + if not running: + indexes.add(i) + else: + try: + for i in args: + indexes.add(int(i)) + except: + f'Error: server_start bad arguments: {args}. Type "help" for help.' + self.sc_app.servers_start(indexes) + + def complete_server_start(self, text, line, begidx, endidx): + running_status = sc_app.get_running_status() + if 'all'.startswith(text): + return ['all'] + return [ + str(i) for (i, running) in enumerate(running_status) + if not running and str(i).startswith(text) + ] + + def help_server_start(self): + print('\n'.join([ + 'server_start index [index...] | all', + 'Start a running server', + ])) + + # server_start + ################## + + ################## + # server_stop + + def do_server_stop(self, line): + args = line.split() + if len(args) == 0: + print( + f'Error: server_stop command takes one or more arguments. Type "help" for help.' + ) + return + indexes = set() + if len(args) == 1 and args[0] == 'all': + # stop all running servers + running_status = self.sc_app.get_running_status() + for (i, running) in enumerate(running_status): + if running: + indexes.add(i) + else: + try: + for i in args: + indexes.add(int(i)) + except: + f'Error: server_stop bad arguments: {args}. Type "help" for help.' + self.sc_app.servers_stop(indexes) + + def complete_server_stop(self, text, line, begidx, endidx): + running_status = sc_app.get_running_status() + if 'all'.startswith(text): + return ['all'] + return [ + str(i) for (i, running) in enumerate(running_status) + if running and str(i).startswith(text) + ] + + def help_server_stop(self): + print('\n'.join([ + 'server_stop index [index...] | all', + 'Stop a running server', + ])) + + # server_stop + ################## + + ################## + # hook + + def do_hook(self, line): + args = line.split() + if len(args) != 2: + print( + f'Error: hook command takes two arguments. Type "help" for help.' + ) + return + nickname = args[0] + args.pop(0) + hook_name = args[0] + args.pop(0) + assert not args + + if nickname == 'door': + print(f'Error: Cannot set hooks on the "door" account.') + return + + if not self.sc_app.is_alias(nickname): + print(f'Error: {nickname} is not in the address book') + return + + src_account = self.sc_app.account_from_alias(nickname) + + if hook_name not in _valid_hook_names: + print( + f'{hook_name} is not a valid hook. Valid hooks are: {_valid_hook_names}' + ) + return + + hook_file = HOOKS_DIR / hook_name / f'{hook_name}.wasm' + if not os.path.isfile(hook_file): + print(f'Error: The hook file {hook_file} does not exist.') + return + create_code = _file_to_hex(hook_file) + self.sc_app(SetHook(account=src_account, create_code=create_code)) + self.sc_app.maybe_ledger_accept() + + def complete_hook(self, text, line, begidx, endidx): + args = line.split() + arg_num = len(args) + if not text: + arg_num += 1 + if arg_num == 2: # account + return self._complete_account(text, line, chain_name='sidechain') + elif arg_num == 3: # hook + if not text: + return _valid_hook_names + return [c for c in _valid_hook_names if c.startswith(text)] + return [] + + def help_hook(self): + print('\n'.join([ + 'hook account hook_name', + 'Set a hook on a sidechain account', + ])) + + # hook + ################## + + ################## + # quit + def do_quit(self, arg): + print('Thank you for using RiplRepl. Goodbye.\n\n') + return True + + def help_quit(self): + print('Exit the program.') + + # quit + ################## + + ################## + # setup_accounts + + def do_setup_accounts(self, arg): + for a in ['alice', 'bob']: + self.mc_app.create_account(a) + for a in ['brad', 'carol']: + self.sc_app.create_account(a) + amt = Asset(value=5000 * 1_000_000) + src = self.mc_app.account_from_alias('root') + dst = self.mc_app.account_from_alias('alice') + self.mc_app(Payment(account=src, dst=dst, amt=amt)) + self.mc_app.maybe_ledger_accept() + + # setup_accounts + ################## + + ################## + # setup_ious + + def do_setup_ious(self, arg): + mc_app = self.mc_app + sc_app = self.sc_app + mc_asset = Asset(value=0, + currency='USD', + issuer=mc_app.account_from_alias('root')) + sc_asset = Asset(value=0, + currency='USD', + issuer=sc_app.account_from_alias('door')) + mc_app.add_asset_alias(mc_asset, 'rrr') + sc_app.add_asset_alias(sc_asset, 'ddd') + mc_app( + Trust(account=mc_app.account_from_alias('alice'), + limit_amt=mc_asset(1_000_000))) + + ## create brad account on the side chain and set the trust line + memos = [{ + 'Memo': { + 'MemoData': + sc_app.account_from_alias('brad').account_id_str_as_hex() + } + }] + mc_app( + Payment(account=mc_app.account_from_alias('alice'), + dst=mc_app.account_from_alias('door'), + amt=Asset(value=3000 * 1_000_000), + memos=memos)) + mc_app.maybe_ledger_accept() + + # create a trust line to alice and pay her USD/rrr + mc_app( + Trust(account=mc_app.account_from_alias('alice'), + limit_amt=mc_asset(1_000_000))) + mc_app.maybe_ledger_accept() + mc_app( + Payment(account=mc_app.account_from_alias('root'), + dst=mc_app.account_from_alias('alice'), + amt=mc_asset(10_000))) + mc_app.maybe_ledger_accept() + + time.sleep(2) + + # create a trust line for brad + sc_app( + Trust(account=sc_app.account_from_alias('brad'), + limit_amt=sc_asset(1_000_000))) + + # setup_ious + ################## + + ################## + # q + + def do_q(self, arg): + return self.do_quit(arg) + + def help_q(self): + return self.help_quit() + + # q + ################## + + ################## + # account_tx + + def do_account_tx(self, line): + args = line.split() + if len(args) < 2: + print( + f'Error: account_tx command takes two or three arguments. Type "help" for help.' + ) + return + + chain = None + + if args[0] not in ['mainchain', 'sidechain']: + print( + f'Error: The first argument must be "mainchain" or "sidechain".' + ) + return + + if args[0] == 'mainchain': + chain = self.mc_app + else: + chain = self.sc_app + args.pop(0) + + accountStr = args[0] + args.pop(0) + + out_file = None + if args: + out_file = args[0] + args.pop(0) + + assert not args + + if not chain.is_alias(accountStr): + print( + f'Error: The issuer {issuer} is not part of the address book.') + return + + account = chain.account_from_alias(accountStr) + + result = json.dumps(chain(AccountTx(account=account)), indent=1) + print(f'{result}') + if out_file: + with open(out_file, 'a') as f: + f.write(f'{result}\n') + + def complete_account_tx(self, text, line, begidx, endidx): + args = line.split() + arg_num = len(args) + if not text: + arg_num += 1 + if arg_num == 2: # chain + return self._complete_chain(text, line) + if arg_num == 3: # account + return self._complete_account(text, line, chain_name=args[1]) + return [] + + def help_account_tx(self): + print('\n'.join([ + 'account_tx (mainchain | sidechain) account [filename]', + 'Return the account transactions', + ])) + + # account_tx + ################## + + ################## + # subscribe + + # Note: The callback isn't called until the user types a new command. + # TODO: Make subscribe asynchronous so the callback is called without requiring the user to type + # a new command. + def do_subscribe(self, line): + args = line.split() + if len(args) != 3: + print( + f'Error: subscribe command takes exactly three arguments. Type "help" for help.' + ) + return + + chain = None + + if args[0] not in ['mainchain', 'sidechain']: + print( + f'Error: The first argument must be "mainchain" or "sidechain".' + ) + return + + if args[0] == 'mainchain': + chain = self.mc_app + else: + chain = self.sc_app + args.pop(0) + + accountStr = args[0] + args.pop(0) + + out_file = args[0] + args.pop(0) + + assert not args + + if not chain.is_alias(accountStr): + print( + f'Error: The issuer {issuer} is not part of the address book.') + return + + account = chain.account_from_alias(accountStr) + + def _subscribe_callback(v: dict): + with open(out_file, 'a') as f: + f.write(f'{json.dumps(v, indent=1)}\n') + + chain(Subscribe(accounts=[account]), _subscribe_callback) + + def complete_subscribe(self, text, line, begidx, endidx): + args = line.split() + arg_num = len(args) + if not text: + arg_num += 1 + if arg_num == 2: # chain + return self._complete_chain(text, line) + if arg_num == 3: # account + return self._complete_account(text, line, chain_name=args[1]) + return [] + + def help_subscribe(self): + print('\n'.join([ + 'subscribe (mainchain | sidechain) account filename', + 'Subscribe to the stream and write the results to filename', + 'Note: The file is not updated until the user types a new command' + ])) + + # subscribe + ################## + + ################## + # EOF + def do_EOF(self, line): + print('Thank you for using RiplRepl. Goodbye.\n\n') + return True + + def help_EOF(self): + print('Exit the program by typing control-d.') + + # EOF + ################## + + +def repl(mc_app: App, sc_app: App): + SidechainRepl(mc_app, sc_app).cmdloop() diff --git a/bin/sidechain/python/log_analyzer.py b/bin/sidechain/python/log_analyzer.py new file mode 100755 index 0000000000..1809e7d8cc --- /dev/null +++ b/bin/sidechain/python/log_analyzer.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 + +import argparse +import json +import re + +from common import eprint + + +class LogLine: + UNSTRUCTURED_RE = re.compile(r'''(?x) + # The x flag enables insignificant whitespace mode (allowing comments) + ^(?P.*UTC) + [\ ] + (?P[^:]*):(?P[^\ ]*) + [\ ] + (?P.*$) + ''') + + STRUCTURED_RE = re.compile(r'''(?x) + # The x flag enables insignificant whitespace mode (allowing comments) + ^(?P.*UTC) + [\ ] + (?P[^:]*):(?P[^\ ]*) + [\ ] + (?P[^{]*) + [\ ] + (?P.*$) + ''') + + def __init__(self, line: str): + self.raw_line = line + self.json_data = None + + try: + if line.endswith('}'): + m = self.STRUCTURED_RE.match(line) + try: + self.json_data = json.loads(m.group('json_data')) + except: + m = self.UNSTRUCTURED_RE.match(line) + else: + m = self.UNSTRUCTURED_RE.match(line) + + self.timestamp = m.group('timestamp') + self.level = m.group('level') + self.module = m.group('module') + self.msg = m.group('msg') + except Exception as e: + eprint(f'init exception: {e} line: {line}') + + def to_mixed_json(self) -> str: + ''' + return a pretty printed string as mixed json + ''' + try: + r = f'{self.timestamp} {self.module}:{self.level} {self.msg}' + if self.json_data: + r += '\n' + json.dumps(self.json_data, indent=1) + return r + except: + eprint(f'Using raw line: {self.raw_line}') + return self.raw_line + + def to_pure_json(self) -> str: + ''' + return a pretty printed string as pure json + ''' + try: + dict = {} + dict['t'] = self.timestamp + dict['m'] = self.module + dict['l'] = self.level + dict['msg'] = self.msg + if self.json_data: + dict['data'] = self.json_data + return json.dumps(dict, indent=1) + except: + return self.raw_line + + +def convert_log(in_file_name: str, out_file_name: str, *, pure_json=False): + try: + prev_lines = None + with open(in_file_name) as input: + with open(out_file_name, "w") as out: + for l in input: + l = l.strip() + if not l: + continue + if LogLine.UNSTRUCTURED_RE.match(l): + if prev_lines: + log_line = LogLine(prev_lines) + if log_line.module == 'SidechainFederator': + if pure_json: + print(log_line.to_pure_json(), file=out) + else: + print(log_line.to_mixed_json(), file=out) + prev_lines = l + else: + if not prev_lines: + eprint( + f'Error: Expected prev_lines. Cur line: {l}') + assert prev_lines + prev_lines += f' {l}' + if prev_lines: + log_line = LogLine(prev_lines) + if log_line.module == 'SidechainFederator': + if pure_json: + print(log_line.to_pure_json(), + file=out, + flush=True) + else: + print(log_line.to_mixed_json(), + file=out, + flush=True) + except Exception as e: + eprint(f'Excption: {e}') + raise e + + +def parse_args(): + parser = argparse.ArgumentParser( + description=('python script to convert log files to json')) + + parser.add_argument( + '--input', + '-i', + help=('input log file'), + ) + + parser.add_argument( + '--output', + '-o', + help=('output log file'), + ) + + return parser.parse_known_args()[0] + + +if __name__ == '__main__': + args = parse_args() + convert_log(args.input, args.output, pure_json=True) diff --git a/bin/sidechain/python/riplrepl.py b/bin/sidechain/python/riplrepl.py new file mode 100755 index 0000000000..0b4b91ac33 --- /dev/null +++ b/bin/sidechain/python/riplrepl.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +''' +Script to run an interactive shell to test sidechains. +''' + +from common import disable_eprint +import interactive +import sidechain + + +def main(): + params = sidechain.Params() + params.interactive = True + + interactive.set_hooks_dir(params.hooks_dir) + + if err_str := params.check_error(): + eprint(err_str) + sys.exit(1) + + if params.verbose: + print("eprint enabled") + else: + disable_eprint() + + if params.standalone: + sidechain.standalone_interactive_repl(params) + else: + sidechain.multinode_interactive_repl(params) + + +if __name__ == '__main__': + main() diff --git a/bin/sidechain/python/ripple_client.py b/bin/sidechain/python/ripple_client.py new file mode 100644 index 0000000000..de88d12042 --- /dev/null +++ b/bin/sidechain/python/ripple_client.py @@ -0,0 +1,193 @@ +import asyncio +import datetime +import json +import os +from os.path import expanduser +import subprocess +import sys +from typing import Callable, List, Optional, Union +import time +import websockets + +from command import Command, ServerInfo, SubscriptionCommand +from common import eprint +from config_file import ConfigFile + + +class RippleClient: + '''Client to send commands to the rippled server''' + def __init__(self, + *, + config: ConfigFile, + exe: str, + command_log: Optional[str] = None): + self.config = config + self.exe = exe + self.command_log = command_log + section = config.port_ws_admin_local + self.websocket_uri = f'{section.protocol}://{section.ip}:{section.port}' + self.subscription_websockets = [] + self.tasks = [] + self.pid = None + if command_log: + with open(self.command_log, 'w') as f: + f.write(f'# Start \n') + + @property + def config_file_name(self): + return self.config.get_file_name() + + def shutdown(self): + try: + group = asyncio.gather(*self.tasks, return_exceptions=True) + group.cancel() + asyncio.get_event_loop().run_until_complete(group) + for ws in self.subscription_websockets: + asyncio.get_event_loop().run_until_complete(ws.close()) + except asyncio.CancelledError: + pass + + def set_pid(self, pid: int): + self.pid = pid + + def get_pid(self) -> Optional[int]: + return self.pid + + def get_config(self) -> ConfigFile: + return self.config + + # Get a dict of the server_state, validated_ledger_seq, and complete_ledgers + def get_brief_server_info(self) -> dict: + ret = { + 'server_state': 'NA', + 'ledger_seq': 'NA', + 'complete_ledgers': 'NA' + } + if not self.pid or self.pid == -1: + return ret + r = self.send_command(ServerInfo()) + if 'info' not in r: + return ret + r = r['info'] + for f in ['server_state', 'complete_ledgers']: + if f in r: + ret[f] = r[f] + if 'validated_ledger' in r: + ret['ledger_seq'] = r['validated_ledger']['seq'] + return ret + + def _write_command_log_command(self, cmd: str, cmd_index: int) -> None: + if not self.command_log: + return + with open(self.command_log, 'a') as f: + f.write(f'\n\n# command {cmd_index}\n') + f.write(f'{cmd}') + + def _write_command_log_result(self, result: str, cmd_index: int) -> None: + if not self.command_log: + return + with open(self.command_log, 'a') as f: + f.write(f'\n\n# result {cmd_index}\n') + f.write(f'{result}') + + def _send_command_line_command(self, cmd_id: int, *args) -> dict: + '''Send the command to the rippled server using the command line interface''' + to_run = [self.exe, '-q', '--conf', self.config_file_name, '--'] + to_run.extend(args) + self._write_command_log_command(to_run, cmd_id) + max_retries = 4 + for retry_count in range(0, max_retries + 1): + try: + r = subprocess.check_output(to_run) + self._write_command_log_result(r, cmd_id) + return json.loads(r.decode('utf-8'))['result'] + except Exception as e: + if retry_count == max_retries: + raise + eprint( + f'Got exception: {str(e)}\nretrying..{retry_count+1} of {max_retries}' + ) + time.sleep(1) # give process time to startup + + async def _send_websock_command( + self, + cmd: Command, + conn: Optional[websockets.client.Connect] = None) -> dict: + assert self.websocket_uri + if conn is None: + async with websockets.connect(self.websocket_uri) as ws: + return await self._send_websock_command(cmd, ws) + + to_send = json.dumps(cmd.get_websocket_dict()) + self._write_command_log_command(to_send, cmd.cmd_id) + await conn.send(to_send) + r = await conn.recv() + self._write_command_log_result(r, cmd.cmd_id) + j = json.loads(r) + if not 'result' in j: + eprint( + f'Error sending websocket command: {json.dumps(cmd.get_websocket_dict(), indent=1)}' + ) + eprint(f'Result: {json.dumps(j, indent=1)}') + raise ValueError('Error sending websocket command') + return j['result'] + + def send_command(self, cmd: Command) -> dict: + '''Send the command to the rippled server''' + if self.websocket_uri: + return asyncio.get_event_loop().run_until_complete( + self._send_websock_command(cmd)) + return self._send_command_line_command(cmd.cmd_id, + *cmd.get_command_line_list()) + + # Need async version to close ledgers from async functions + async def async_send_command(self, cmd: Command) -> dict: + '''Send the command to the rippled server''' + if self.websocket_uri: + return await self._send_websock_command(cmd) + return self._send_command_line_command(cmd.cmd_id, + *cmd.get_command_line_list()) + + def send_subscribe_command( + self, + cmd: SubscriptionCommand, + callback: Optional[Callable[[dict], None]] = None) -> dict: + '''Send the command to the rippled server''' + assert self.websocket_uri + ws = cmd.websocket + if ws is None: + # subscribe + assert callback + ws = asyncio.get_event_loop().run_until_complete( + websockets.connect(self.websocket_uri)) + self.subscription_websockets.append(ws) + result = asyncio.get_event_loop().run_until_complete( + self._send_websock_command(cmd, ws)) + if cmd.websocket is not None: + # unsubscribed. close the websocket + self.subscription_websockets.remove(cmd.websocket) + cmd.websocket.close() + cmd.websocket = None + else: + # setup a task to read the websocket + cmd.websocket = ws # must be set after the _send_websock_command or will unsubscribe + + async def subscribe_callback(ws: websockets.client.Connect, + cb: Callable[[dict], None]): + while True: + r = await ws.recv() + d = json.loads(r) + cb(d) + + task = asyncio.get_event_loop().create_task( + subscribe_callback(cmd.websocket, callback)) + self.tasks.append(task) + return result + + def stop(self): + '''Stop the server''' + return self.send_command(Stop()) + + def set_log_level(self, severity: str, *, partition: Optional[str] = None): + '''Set the server log level''' + return self.send_command(LogLevel(severity, parition=parition)) diff --git a/bin/sidechain/python/sidechain.py b/bin/sidechain/python/sidechain.py new file mode 100755 index 0000000000..c36683890b --- /dev/null +++ b/bin/sidechain/python/sidechain.py @@ -0,0 +1,576 @@ +#!/usr/bin/env python3 +''' +Script to test and debug sidechains. + +The mainchain exe location can be set through the command line or +the environment variable RIPPLED_MAINCHAIN_EXE + +The sidechain exe location can be set through the command line or +the environment variable RIPPLED_SIDECHAIN_EXE + +The configs_dir (generated with create_config_files.py) can be set through the command line +or the environment variable RIPPLED_SIDECHAIN_CFG_DIR +''' + +import argparse +import json +from multiprocessing import Process, Value +import os +import sys +import time +from typing import Callable, Dict, List, Optional + +from app import App, single_client_app, testnet_app, configs_for_testnet +from command import AccountInfo, AccountTx, LedgerAccept, LogLevel, Subscribe +from common import Account, Asset, eprint, disable_eprint, XRP +from config_file import ConfigFile +import interactive +from log_analyzer import convert_log +from test_utils import mc_wait_for_payment_detect, sc_wait_for_payment_detect, mc_connect_subscription, sc_connect_subscription +from transaction import AccountSet, Payment, SignerListSet, SetRegularKey, Ticket, Trust + + +def parse_args_helper(parser: argparse.ArgumentParser): + + parser.add_argument( + '--debug_sidechain', + '-ds', + action='store_true', + help=('Mode to debug sidechain (prompt to run sidechain in gdb)'), + ) + + parser.add_argument( + '--debug_mainchain', + '-dm', + action='store_true', + help=('Mode to debug mainchain (prompt to run sidechain in gdb)'), + ) + + parser.add_argument( + '--exe_mainchain', + '-em', + help=('path to mainchain rippled executable'), + ) + + parser.add_argument( + '--exe_sidechain', + '-es', + help=('path to mainchain rippled executable'), + ) + + parser.add_argument( + '--cfgs_dir', + '-c', + help= + ('path to configuration file dir (generated with create_config_files.py)' + ), + ) + + parser.add_argument( + '--standalone', + '-a', + action='store_true', + help=('run standalone tests'), + ) + + parser.add_argument( + '--interactive', + '-i', + action='store_true', + help=('run interactive repl'), + ) + + parser.add_argument( + '--quiet', + '-q', + action='store_true', + help=('Disable printing errors (eprint disabled)'), + ) + + parser.add_argument( + '--verbose', + '-v', + action='store_true', + help=('Enable printing errors (eprint enabled)'), + ) + + # Pauses are use for attaching debuggers and looking at logs are know checkpoints + parser.add_argument( + '--with_pauses', + '-p', + action='store_true', + help= + ('Add pauses at certain checkpoints in tests until "enter" key is hit' + ), + ) + + parser.add_argument( + '--hooks_dir', + help=('path to hooks dir'), + ) + + +def parse_args(): + parser = argparse.ArgumentParser(description=('Test and debug sidechains')) + parse_args_helper(parser) + return parser.parse_known_args()[0] + + +class Params: + def __init__(self, *, configs_dir: Optional[str] = None): + args = parse_args() + + self.debug_sidechain = False + if args.debug_sidechain: + self.debug_sidechain = args.debug_sidechain + self.debug_mainchain = False + if args.debug_mainchain: + self.debug_mainchain = arts.debug_mainchain + + self.standalone = args.standalone + self.with_pauses = args.with_pauses + self.interactive = args.interactive + self.quiet = args.quiet + self.verbose = args.verbose + + self.mainchain_exe = None + if 'RIPPLED_MAINCHAIN_EXE' in os.environ: + self.mainchain_exe = os.environ['RIPPLED_MAINCHAIN_EXE'] + if args.exe_mainchain: + self.mainchain_exe = args.exe_mainchain + + self.sidechain_exe = None + if 'RIPPLED_SIDECHAIN_EXE' in os.environ: + self.sidechain_exe = os.environ['RIPPLED_SIDECHAIN_EXE'] + if args.exe_sidechain: + self.sidechain_exe = args.exe_sidechain + + self.configs_dir = None + if 'RIPPLED_SIDECHAIN_CFG_DIR' in os.environ: + self.configs_dir = os.environ['RIPPLED_SIDECHAIN_CFG_DIR'] + if args.cfgs_dir: + self.configs_dir = args.cfgs_dir + if configs_dir is not None: + self.configs_dir = configs_dir + + self.hooks_dir = None + if 'RIPPLED_SIDECHAIN_HOOKS_DIR' in os.environ: + self.hooks_dir = os.environ['RIPPLED_SIDECHAIN_HOOKS_DIR'] + if args.hooks_dir: + self.hooks_dir = args.hooks_dir + + if not self.configs_dir: + self.mainchain_config = None + self.sidechain_config = None + self.sidechain_bootstrap_config = None + self.genesis_account = None + self.mc_door_account = None + self.user_account = None + self.sc_door_account = None + self.federators = None + return + + if self.standalone: + self.mainchain_config = ConfigFile( + file_name=f'{self.configs_dir}/main.no_shards.dog/rippled.cfg') + self.sidechain_config = ConfigFile( + file_name= + f'{self.configs_dir}/main.no_shards.dog.sidechain/rippled.cfg') + self.sidechain_bootstrap_config = ConfigFile( + file_name= + f'{self.configs_dir}/main.no_shards.dog.sidechain/sidechain_bootstrap.cfg' + ) + else: + self.mainchain_config = ConfigFile( + file_name= + f'{self.configs_dir}/sidechain_testnet/main.no_shards.mainchain_0/rippled.cfg' + ) + self.sidechain_config = ConfigFile( + file_name= + f'{self.configs_dir}/sidechain_testnet/sidechain_0/rippled.cfg' + ) + self.sidechain_bootstrap_config = ConfigFile( + file_name= + f'{self.configs_dir}/sidechain_testnet/sidechain_0/sidechain_bootstrap.cfg' + ) + + self.genesis_account = Account( + account_id='rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh', + secret_key='masterpassphrase', + nickname='genesis') + self.mc_door_account = Account( + account_id=self.sidechain_config.sidechain.mainchain_account, + secret_key=self.sidechain_bootstrap_config.sidechain. + mainchain_secret, + nickname='door') + self.user_account = Account( + account_id='rJynXY96Vuq6B58pST9K5Ak5KgJ2JcRsQy', + secret_key='snVsJfrr2MbVpniNiUU6EDMGBbtzN', + nickname='alice') + + self.sc_door_account = Account( + account_id='rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh', + secret_key='masterpassphrase', + nickname='door') + self.federators = [ + l.split()[1].strip() for l in + self.sidechain_bootstrap_config.sidechain_federators.get_lines() + ] + + def check_error(self) -> str: + ''' + Check for errors. Return `None` if no errors, + otherwise return a string describing the error + ''' + if not self.mainchain_exe: + return 'Missing mainchain_exe location. Either set the env variable RIPPLED_MAINCHAIN_EXE or use the --exe_mainchain command line switch' + if not self.sidechain_exe: + return 'Missing sidechain_exe location. Either set the env variable RIPPLED_SIDECHAIN_EXE or use the --exe_sidechain command line switch' + if not self.configs_dir: + return 'Missing configs directory location. Either set the env variable RIPPLED_SIDECHAIN_CFG_DIR or use the --cfgs_dir command line switch' + if self.verbose and self.quiet: + return 'Cannot specify both verbose and quiet options at the same time' + + +mainDoorKeeper = 0 +sideDoorKeeper = 1 +updateSignerList = 2 + + +def setup_mainchain(mc_app: App, + params: Params, + setup_user_accounts: bool = True): + mc_app.add_to_keymanager(params.mc_door_account) + if setup_user_accounts: + mc_app.add_to_keymanager(params.user_account) + + mc_app(LogLevel('fatal')) + + # Allow rippling through the genesis account + mc_app(AccountSet(account=params.genesis_account).set_default_ripple(True)) + mc_app.maybe_ledger_accept() + + # Create and fund the mc door account + mc_app( + Payment(account=params.genesis_account, + dst=params.mc_door_account, + amt=XRP(10_000))) + mc_app.maybe_ledger_accept() + + # Create a trust line so USD/root account ious can be sent cross chain + mc_app( + Trust(account=params.mc_door_account, + limit_amt=Asset(value=1_000_000, + currency='USD', + issuer=params.genesis_account))) + + # set the chain's signer list and disable the master key + divide = 4 * len(params.federators) + by = 5 + quorum = (divide + by - 1) // by + mc_app( + SignerListSet(account=params.mc_door_account, + quorum=quorum, + keys=params.federators)) + mc_app.maybe_ledger_accept() + r = mc_app(Ticket(account=params.mc_door_account, src_tag=mainDoorKeeper)) + mc_app.maybe_ledger_accept() + mc_app(Ticket(account=params.mc_door_account, src_tag=sideDoorKeeper)) + mc_app.maybe_ledger_accept() + mc_app(Ticket(account=params.mc_door_account, src_tag=updateSignerList)) + mc_app.maybe_ledger_accept() + mc_app(AccountSet(account=params.mc_door_account).set_disable_master()) + mc_app.maybe_ledger_accept() + + if setup_user_accounts: + # Create and fund a regular user account + mc_app( + Payment(account=params.genesis_account, + dst=params.user_account, + amt=XRP(2_000))) + mc_app.maybe_ledger_accept() + + +def setup_sidechain(sc_app: App, + params: Params, + setup_user_accounts: bool = True): + sc_app.add_to_keymanager(params.sc_door_account) + if setup_user_accounts: + sc_app.add_to_keymanager(params.user_account) + + sc_app(LogLevel('fatal')) + sc_app(LogLevel('trace', partition='SidechainFederator')) + + # set the chain's signer list and disable the master key + divide = 4 * len(params.federators) + by = 5 + quorum = (divide + by - 1) // by + sc_app( + SignerListSet(account=params.genesis_account, + quorum=quorum, + keys=params.federators)) + sc_app.maybe_ledger_accept() + sc_app(Ticket(account=params.genesis_account, src_tag=mainDoorKeeper)) + sc_app.maybe_ledger_accept() + sc_app(Ticket(account=params.genesis_account, src_tag=sideDoorKeeper)) + sc_app.maybe_ledger_accept() + sc_app(Ticket(account=params.genesis_account, src_tag=updateSignerList)) + sc_app.maybe_ledger_accept() + sc_app(AccountSet(account=params.genesis_account).set_disable_master()) + sc_app.maybe_ledger_accept() + + +def _xchain_transfer(from_chain: App, to_chain: App, src: Account, + dst: Account, amt: Asset, from_chain_door: Account, + to_chain_door: Account): + memos = [{'Memo': {'MemoData': dst.account_id_str_as_hex()}}] + from_chain(Payment(account=src, dst=from_chain_door, amt=amt, memos=memos)) + from_chain.maybe_ledger_accept() + if to_chain.standalone: + # from_chain (side chain) sends a txn, but won't close the to_chain (main chain) ledger + time.sleep(1) + to_chain.maybe_ledger_accept() + + +def main_to_side_transfer(mc_app: App, sc_app: App, src: Account, dst: Account, + amt: Asset, params: Params): + _xchain_transfer(mc_app, sc_app, src, dst, amt, params.mc_door_account, + params.sc_door_account) + + +def side_to_main_transfer(mc_app: App, sc_app: App, src: Account, dst: Account, + amt: Asset, params: Params): + _xchain_transfer(sc_app, mc_app, src, dst, amt, params.sc_door_account, + params.mc_door_account) + + +def simple_test(mc_app: App, sc_app: App, params: Params): + try: + bob = sc_app.create_account('bob') + main_to_side_transfer(mc_app, sc_app, params.user_account, bob, + XRP(200), params) + main_to_side_transfer(mc_app, sc_app, params.user_account, bob, + XRP(60), params) + + if params.with_pauses: + _convert_log_files_to_json( + mc_app.get_configs() + sc_app.get_configs(), + 'checkpoint1.json') + input( + "Pausing to check for main -> side txns (press enter to continue)" + ) + + side_to_main_transfer(mc_app, sc_app, bob, params.user_account, XRP(9), + params) + side_to_main_transfer(mc_app, sc_app, bob, params.user_account, + XRP(11), params) + + if params.with_pauses: + input( + "Pausing to check for side -> main txns (press enter to continue)" + ) + finally: + _convert_log_files_to_json(mc_app.get_configs() + sc_app.get_configs(), + 'final.json') + + +def _rm_debug_log(config: ConfigFile): + try: + debug_log = config.debug_logfile.get_line() + if debug_log: + print(f'removing debug file: {debug_log}', flush=True) + os.remove(debug_log) + except: + pass + + +def _standalone_with_callback(params: Params, + callback: Callable[[App, App], None], + setup_user_accounts: bool = True): + + if (params.debug_mainchain): + input("Start mainchain server and press enter to continue: ") + else: + _rm_debug_log(params.mainchain_config) + with single_client_app(config=params.mainchain_config, + exe=params.mainchain_exe, + standalone=True, + run_server=not params.debug_mainchain) as mc_app: + + mc_connect_subscription(mc_app, params.mc_door_account) + setup_mainchain(mc_app, params, setup_user_accounts) + + if (params.debug_sidechain): + input("Start sidechain server and press enter to continue: ") + else: + _rm_debug_log(params.sidechain_config) + with single_client_app( + config=params.sidechain_config, + exe=params.sidechain_exe, + standalone=True, + run_server=not params.debug_sidechain) as sc_app: + + sc_connect_subscription(sc_app, params.sc_door_account) + setup_sidechain(sc_app, params, setup_user_accounts) + callback(mc_app, sc_app) + + +def _convert_log_files_to_json(to_convert: List[ConfigFile], suffix: str): + ''' + Convert the log file to json + ''' + for c in to_convert: + try: + debug_log = c.debug_logfile.get_line() + if not os.path.exists(debug_log): + continue + converted_log = f'{debug_log}.{suffix}' + if os.path.exists(converted_log): + os.remove(converted_log) + print(f'Converting log {debug_log} to {converted_log}', flush=True) + convert_log(debug_log, converted_log, pure_json=True) + except: + eprint(f'Exception converting log') + + +def _multinode_with_callback(params: Params, + callback: Callable[[App, App], None], + setup_user_accounts: bool = True): + + mainchain_cfg = ConfigFile( + file_name= + f'{params.configs_dir}/sidechain_testnet/main.no_shards.mainchain_0/rippled.cfg' + ) + _rm_debug_log(mainchain_cfg) + if params.debug_mainchain: + input("Start mainchain server and press enter to continue: ") + with single_client_app(config=mainchain_cfg, + exe=params.mainchain_exe, + standalone=True, + run_server=not params.debug_mainchain) as mc_app: + + if params.with_pauses: + input("Pausing after mainchain start (press enter to continue)") + + mc_connect_subscription(mc_app, params.mc_door_account) + setup_mainchain(mc_app, params, setup_user_accounts) + if params.with_pauses: + input("Pausing after mainchain setup (press enter to continue)") + + testnet_configs = configs_for_testnet( + f'{params.configs_dir}/sidechain_testnet/sidechain_') + for c in testnet_configs: + _rm_debug_log(c) + + run_server_list = [True] * len(testnet_configs) + if params.debug_sidechain: + run_server_list[0] = False + input( + f'Start testnet server {testnet_configs[0].get_file_name()} and press enter to continue: ' + ) + + with testnet_app(exe=params.sidechain_exe, + configs=testnet_configs, + run_server=run_server_list) as n_app: + + if params.with_pauses: + input("Pausing after testnet start (press enter to continue)") + + sc_connect_subscription(n_app, params.sc_door_account) + setup_sidechain(n_app, params, setup_user_accounts) + if params.with_pauses: + input( + "Pausing after sidechain setup (press enter to continue)") + callback(mc_app, n_app) + + +def standalone_test(params: Params): + def callback(mc_app: App, sc_app: App): + simple_test(mc_app, sc_app, params) + + _standalone_with_callback(params, callback) + + +def multinode_test(params: Params): + def callback(mc_app: App, sc_app: App): + simple_test(mc_app, sc_app, params) + + _multinode_with_callback(params, callback) + + +# The mainchain runs in standalone mode. Most operations - like cross chain +# paymens - will automatically close ledgers. However, some operations, like +# refunds need an extra close. This loop automatically closes ledgers. +def close_mainchain_ledgers(stop_token: Value, params: Params, sleep_time=4): + with single_client_app(config=params.mainchain_config, + exe=params.mainchain_exe, + standalone=True, + run_server=False) as mc_app: + while stop_token.value != 0: + mc_app.maybe_ledger_accept() + time.sleep(sleep_time) + + +def standalone_interactive_repl(params: Params): + def callback(mc_app: App, sc_app: App): + # process will run while stop token is non-zero + stop_token = Value('i', 1) + p = None + if mc_app.standalone: + p = Process(target=close_mainchain_ledgers, + args=(stop_token, params)) + p.start() + try: + interactive.repl(mc_app, sc_app) + finally: + if p: + stop_token.value = 0 + p.join() + + _standalone_with_callback(params, callback, setup_user_accounts=False) + + +def multinode_interactive_repl(params: Params): + def callback(mc_app: App, sc_app: App): + # process will run while stop token is non-zero + stop_token = Value('i', 1) + p = None + if mc_app.standalone: + p = Process(target=close_mainchain_ledgers, + args=(stop_token, params)) + p.start() + try: + interactive.repl(mc_app, sc_app) + finally: + if p: + stop_token.value = 0 + p.join() + + _multinode_with_callback(params, callback, setup_user_accounts=False) + + +def main(): + params = Params() + interactive.set_hooks_dir(params.hooks_dir) + + if err_str := params.check_error(): + eprint(err_str) + sys.exit(1) + + if params.quiet: + print("Disabling eprint") + disable_eprint() + + if params.interactive: + if params.standalone: + standalone_interactive_repl(params) + else: + multinode_interactive_repl(params) + elif params.standalone: + standalone_test(params) + else: + multinode_test(params) + + +if __name__ == '__main__': + main() diff --git a/bin/sidechain/python/test_utils.py b/bin/sidechain/python/test_utils.py new file mode 100644 index 0000000000..fb631f2fec --- /dev/null +++ b/bin/sidechain/python/test_utils.py @@ -0,0 +1,174 @@ +import asyncio +import collections +from contextlib import contextmanager +import json +import logging +import pprint +import time +from typing import Callable, Dict, List, Optional + +from app import App, balances_dataframe +from common import Account, Asset, XRP, eprint +from command import Subscribe + +MC_SUBSCRIBE_QUEUE = [] +SC_SUBSCRIBE_QUEUE = [] + + +def _mc_subscribe_callback(v: dict): + MC_SUBSCRIBE_QUEUE.append(v) + logging.info(f'mc subscribe_callback:\n{json.dumps(v, indent=1)}') + + +def _sc_subscribe_callback(v: dict): + SC_SUBSCRIBE_QUEUE.append(v) + logging.info(f'sc subscribe_callback:\n{json.dumps(v, indent=1)}') + + +def mc_connect_subscription(app: App, door_account: Account): + app(Subscribe(accounts=[door_account]), _mc_subscribe_callback) + + +def sc_connect_subscription(app: App, door_account: Account): + app(Subscribe(accounts=[door_account]), _sc_subscribe_callback) + + +# This pops elements off the subscribe_queue until the transaction is found +# It mofifies the queue in place. +async def async_wait_for_payment_detect(app: App, subscribe_queue: List[dict], + src: Account, dst: Account, + amt_asset: Asset): + logging.info( + f'Wait for payment {src.account_id = } {dst.account_id = } {amt_asset = }' + ) + n_txns = 10 # keep this many txn in a circular buffer. + # If the payment is not detected, write them to the log. + last_n_paytxns = collections.deque(maxlen=n_txns) + for i in range(30): + while subscribe_queue: + d = subscribe_queue.pop(0) + if 'transaction' not in d: + continue + txn = d['transaction'] + if txn['TransactionType'] != 'Payment': + continue + + txn_asset = Asset(from_rpc_result=txn['Amount']) + if txn['Account'] == src.account_id and txn[ + 'Destination'] == dst.account_id and txn_asset == amt_asset: + if d['engine_result_code'] == 0: + logging.info( + f'Found payment {src.account_id = } {dst.account_id = } {amt_asset = }' + ) + return + else: + logging.error( + f'Expected payment failed {src.account_id = } {dst.account_id = } {amt_asset = }' + ) + raise ValueError( + f'Expected payment failed {src.account_id = } {dst.account_id = } {amt_asset = }' + ) + else: + last_n_paytxns.append(txn) + if i > 0 and not (i % 5): + logging.warning( + f'Waiting for txn detect {src.account_id = } {dst.account_id = } {amt_asset = }' + ) + # side chain can send transactions to the main chain, but won't close the ledger + # We don't know when the transaction will be sent, so may need to close the ledger here + await app.async_maybe_ledger_accept() + await asyncio.sleep(2) + logging.warning( + f'Last {len(last_n_paytxns)} pay txns while waiting for payment detect' + ) + for t in last_n_paytxns: + logging.warning( + f'Detected pay transaction while waiting for payment: {t}') + logging.error( + f'Expected txn detect {src.account_id = } {dst.account_id = } {amt_asset = }' + ) + raise ValueError( + f'Expected txn detect {src.account_id = } {dst.account_id = } {amt_asset = }' + ) + + +def mc_wait_for_payment_detect(app: App, src: Account, dst: Account, + amt_asset: Asset): + logging.info(f'mainchain waiting for payment detect') + return asyncio.get_event_loop().run_until_complete( + async_wait_for_payment_detect(app, MC_SUBSCRIBE_QUEUE, src, dst, + amt_asset)) + + +def sc_wait_for_payment_detect(app: App, src: Account, dst: Account, + amt_asset: Asset): + logging.info(f'sidechain waiting for payment detect') + return asyncio.get_event_loop().run_until_complete( + async_wait_for_payment_detect(app, SC_SUBSCRIBE_QUEUE, src, dst, + amt_asset)) + + +def wait_for_balance_change(app: App, + acc: Account, + pre_balance: Asset, + final_diff: Optional[Asset] = None): + logging.info( + f'waiting for balance change {acc.account_id = } {pre_balance = } {final_diff = }' + ) + for i in range(30): + new_bal = app.get_balance(acc, pre_balance(0)) + diff = new_bal - pre_balance + if new_bal != pre_balance: + logging.info( + f'Balance changed {acc.account_id = } {pre_balance = } {new_bal = } {diff = } {final_diff = }' + ) + if final_diff is None or diff == final_diff: + return + app.maybe_ledger_accept() + time.sleep(2) + if i > 0 and not (i % 5): + logging.warning( + f'Waiting for balance to change {acc.account_id = } {pre_balance = }' + ) + logging.error( + f'Expected balance to change {acc.account_id = } {pre_balance = } {new_bal = } {diff = } {final_diff = }' + ) + raise ValueError( + f'Expected balance to change {acc.account_id = } {pre_balance = } {new_bal = } {diff = } {final_diff = }' + ) + + +def log_chain_state(mc_app, sc_app, log, msg='Chain State'): + chains = [mc_app, sc_app] + chain_names = ['mainchain', 'sidechain'] + balances = balances_dataframe(chains, chain_names) + df_as_str = balances.to_string(float_format=lambda x: f'{x:,.6f}') + log(f'{msg} Balances: \n{df_as_str}') + federator_info = sc_app.federator_info() + log(f'{msg} Federator Info: \n{pprint.pformat(federator_info)}') + + +# Tests can set this to True to help debug test failures by showing account +# balances in the log before the test runs +test_context_verbose_logging = False + + +@contextmanager +def test_context(mc_app, sc_app, verbose_logging: Optional[bool] = None): + '''Write extra context info to the log on test failure''' + global test_context_verbose_logging + if verbose_logging is None: + verbose_logging = test_context_verbose_logging + try: + if verbose_logging: + log_chain_state(mc_app, sc_app, logging.info) + start_time = time.monotonic() + yield + except: + log_chain_state(mc_app, sc_app, logging.error) + raise + finally: + elapased_time = time.monotonic() - start_time + logging.info(f'Test elapsed time: {elapased_time}') + if verbose_logging: + log_chain_state(mc_app, sc_app, logging.info) diff --git a/bin/sidechain/python/testnet.py b/bin/sidechain/python/testnet.py new file mode 100644 index 0000000000..efcfd9ef66 --- /dev/null +++ b/bin/sidechain/python/testnet.py @@ -0,0 +1,206 @@ +''' +Bring up a rippled testnetwork from a set of config files with fixed ips. +''' + +from contextlib import contextmanager +import glob +import os +import subprocess +import time +from typing import Callable, List, Optional, Set, Union + +from command import ServerInfo +from config_file import ConfigFile +from ripple_client import RippleClient + + +class Network: + # If run_server is None, run all the servers. + # This is useful to help debugging + def __init__(self, + exe: str, + configs: List[ConfigFile], + *, + command_logs: Optional[List[str]] = None, + run_server: Optional[List[bool]] = None, + extra_args: Optional[List[List[str]]] = None): + if not configs: + raise ValueError(f'Must specify at least one config') + + if run_server and len(run_server) != len(configs): + raise ValueError( + f'run_server length must match number of configs (or be None): {len(configs) = } {len(run_server) = }' + ) + + self.configs = configs + self.clients = [] + self.running_server_indexes = set() + self.processes = {} + + if not run_server: + run_server = [] + run_server += [True] * (len(configs) - len(run_server)) + + self.run_server = run_server + + if not command_logs: + command_logs = [] + command_logs += [None] * (len(configs) - len(command_logs)) + + self.command_logs = command_logs + + # remove the old database directories. + # we want tests to start from the same empty state every time + for config in self.configs: + db_path = config.database_path.get_line() + if db_path and os.path.isdir(db_path): + files = glob.glob(f'{db_path}/**', recursive=True) + for f in files: + if os.path.isdir(f): + continue + os.unlink(f) + + for config, log in zip(self.configs, self.command_logs): + client = RippleClient(config=config, command_log=log, exe=exe) + self.clients.append(client) + + self.servers_start(extra_args=extra_args) + + def shutdown(self): + for a in self.clients: + a.shutdown() + + self.servers_stop() + + def num_clients(self) -> int: + return len(self.clients) + + def get_client(self, i: int) -> RippleClient: + return self.clients[i] + + def get_configs(self) -> List[ConfigFile]: + return [c.config for c in self.clients] + + def get_pids(self) -> List[int]: + return [c.get_pid() for c in self.clients if c.get_pid() is not None] + + # Get a dict of the server_state, validated_ledger_seq, and complete_ledgers + def get_brief_server_info(self) -> dict: + ret = {'server_state': [], 'ledger_seq': [], 'complete_ledgers': []} + for c in self.clients: + r = c.get_brief_server_info() + for (k, v) in r.items(): + ret[k].append(v) + return ret + + # returns true if the server is running, false if not. Note, this relies on + # servers being shut down through the `servers_stop` interface. If a server + # crashes, or is started or stopped through other means, an incorrect status + # may be reported. + def get_running_status(self) -> List[bool]: + return [ + i in self.running_server_indexes for i in range(len(self.clients)) + ] + + def is_running(self, index: int) -> bool: + return index in self.running_server_indexes + + def wait_for_validated_ledger(self, server_index: Optional[int] = None): + ''' + Don't return until the network has at least one validated ledger + ''' + + if server_index is None: + for i in range(len(self.configs)): + self.wait_for_validated_ledger(i) + return + + client = self.clients[server_index] + for i in range(600): + r = client.send_command(ServerInfo()) + state = None + if 'info' in r: + state = r['info']['server_state'] + if state == 'proposing': + print(f'Synced: {server_index} : {state}', flush=True) + break + if not i % 10: + print(f'Waiting for sync: {server_index} : {state}', + flush=True) + time.sleep(1) + + for i in range(600): + r = client.send_command(ServerInfo()) + state = None + if 'info' in r: + complete_ledgers = r['info']['complete_ledgers'] + if complete_ledgers and complete_ledgers != 'empty': + print(f'Have complete ledgers: {server_index} : {state}', + flush=True) + return + if not i % 10: + print( + f'Waiting for complete_ledgers: {server_index} : {complete_ledgers}', + flush=True) + time.sleep(1) + + raise ValueError('Could not sync server {client.config_file_name}') + + def servers_start(self, + server_indexes: Optional[Union[Set[int], + List[int]]] = None, + *, + extra_args: Optional[List[List[str]]] = None): + if server_indexes is None: + server_indexes = [i for i in range(len(self.clients))] + + if extra_args is None: + extra_args = [] + extra_args += [list()] * (len(self.configs) - len(extra_args)) + + for i in server_indexes: + if i in self.running_server_indexes or not self.run_server[i]: + continue + + client = self.clients[i] + to_run = [client.exe, '--conf', client.config_file_name] + print(f'Starting server {client.config_file_name}') + fout = open(os.devnull, 'w') + p = subprocess.Popen(to_run + extra_args[i], + stdout=fout, + stderr=subprocess.STDOUT) + client.set_pid(p.pid) + print( + f'started rippled: config: {client.config_file_name} PID: {p.pid}', + flush=True) + self.running_server_indexes.add(i) + self.processes[i] = p + + time.sleep(2) # give servers time to start + + def servers_stop(self, + server_indexes: Optional[Union[Set[int], + List[int]]] = None): + if server_indexes is None: + server_indexes = self.running_server_indexes.copy() + + if 0 in server_indexes: + print( + f'WARNING: Server 0 is being stopped. RPC commands cannot be sent until this is restarted.' + ) + + for i in server_indexes: + if i not in self.running_server_indexes: + continue + client = self.clients[i] + to_run = [client.exe, '--conf', client.config_file_name] + fout = open(os.devnull, 'w') + subprocess.Popen(to_run + ['stop'], + stdout=fout, + stderr=subprocess.STDOUT) + self.running_server_indexes.discard(i) + + for i in server_indexes: + self.processes[i].wait() + del self.processes[i] + self.get_client(i).set_pid(-1) diff --git a/bin/sidechain/python/tests/conftest.py b/bin/sidechain/python/tests/conftest.py new file mode 100644 index 0000000000..eaddafc977 --- /dev/null +++ b/bin/sidechain/python/tests/conftest.py @@ -0,0 +1,64 @@ +# Add parent directory to module path +import os, sys +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from common import Account, Asset, XRP +import create_config_files +import sidechain + +import pytest +''' +Sidechains uses argparse.ArgumentParser to add command line options. +The function call to add an argument is `add_argument`. pytest uses `addoption`. +This wrapper class changes calls from `add_argument` to calls to `addoption`. +To avoid conflicts between pytest and sidechains, all sidechain arguments have +the suffix `_sc` appended to them. I.e. `--verbose` is for pytest, `--verbose_sc` +is for sidechains. +''' + + +class ArgumentParserWrapper: + def __init__(self, wrapped): + self.wrapped = wrapped + + def add_argument(self, *args, **kwargs): + for a in args: + if not a.startswith('--'): + continue + a = a + '_sc' + self.wrapped.addoption(a, **kwargs) + + +def pytest_addoption(parser): + wrapped = ArgumentParserWrapper(parser) + sidechain.parse_args_helper(wrapped) + + +def _xchain_assets(ratio: int = 1): + assets = {} + assets['xrp_xrp_sidechain_asset'] = create_config_files.XChainAsset( + XRP(0), XRP(0), 1, 1 * ratio, 200, 200 * ratio) + root_account = Account(account_id="rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh") + main_iou_asset = Asset(value=0, currency='USD', issuer=root_account) + side_iou_asset = Asset(value=0, currency='USD', issuer=root_account) + assets['iou_iou_sidechain_asset'] = create_config_files.XChainAsset( + main_iou_asset, side_iou_asset, 1, 1 * ratio, 0.02, 0.02 * ratio) + return assets + + +# Diction of config dirs. Key is ratio +_config_dirs = None + + +@pytest.fixture +def configs_dirs_dict(tmp_path): + global _config_dirs + if not _config_dirs: + params = create_config_files.Params() + _config_dirs = {} + for ratio in (1, 2): + params.configs_dir = str(tmp_path / f'test_config_files_{ratio}') + create_config_files.main(params, _xchain_assets(ratio)) + _config_dirs[ratio] = params.configs_dir + + return _config_dirs diff --git a/bin/sidechain/python/tests/simple_xchain_transfer_test.py b/bin/sidechain/python/tests/simple_xchain_transfer_test.py new file mode 100644 index 0000000000..d92d7fb703 --- /dev/null +++ b/bin/sidechain/python/tests/simple_xchain_transfer_test.py @@ -0,0 +1,191 @@ +import logging +import pprint +import pytest +from multiprocessing import Process, Value +from typing import Dict +import sys + +from app import App +from common import Asset, eprint, disable_eprint, XRP +import interactive +from sidechain import Params +import sidechain +import test_utils +import time +from transaction import Payment, Trust + + +def simple_xrp_test(mc_app: App, sc_app: App, params: Params): + alice = mc_app.account_from_alias('alice') + adam = sc_app.account_from_alias('adam') + + # main to side + # First txn funds the side chain account + with test_utils.test_context(mc_app, sc_app): + to_send_asset = XRP(1000) + pre_bal = sc_app.get_balance(adam, to_send_asset) + sidechain.main_to_side_transfer(mc_app, sc_app, alice, adam, + to_send_asset, params) + test_utils.wait_for_balance_change(sc_app, adam, pre_bal, + to_send_asset) + + for i in range(2): + # even amounts for main to side + for value in range(10, 20, 2): + with test_utils.test_context(mc_app, sc_app): + to_send_asset = XRP(value) + pre_bal = sc_app.get_balance(adam, to_send_asset) + sidechain.main_to_side_transfer(mc_app, sc_app, alice, adam, + to_send_asset, params) + test_utils.wait_for_balance_change(sc_app, adam, pre_bal, + to_send_asset) + + # side to main + # odd amounts for side to main + for value in range(9, 19, 2): + with test_utils.test_context(mc_app, sc_app): + to_send_asset = XRP(value) + pre_bal = mc_app.get_balance(alice, to_send_asset) + sidechain.side_to_main_transfer(mc_app, sc_app, adam, alice, + to_send_asset, params) + test_utils.wait_for_balance_change(mc_app, alice, pre_bal, + to_send_asset) + + +def simple_iou_test(mc_app: App, sc_app: App, params: Params): + alice = mc_app.account_from_alias('alice') + adam = sc_app.account_from_alias('adam') + + mc_asset = Asset(value=0, + currency='USD', + issuer=mc_app.account_from_alias('root')) + sc_asset = Asset(value=0, + currency='USD', + issuer=sc_app.account_from_alias('door')) + mc_app.add_asset_alias(mc_asset, 'mcd') # main chain dollar + sc_app.add_asset_alias(sc_asset, 'scd') # side chain dollar + mc_app(Trust(account=alice, limit_amt=mc_asset(1_000_000))) + + ## make sure adam account on the side chain exists and set the trust line + with test_utils.test_context(mc_app, sc_app): + sidechain.main_to_side_transfer(mc_app, sc_app, alice, adam, XRP(300), + params) + + # create a trust line to alice and pay her USD/root + mc_app(Trust(account=alice, limit_amt=mc_asset(1_000_000))) + mc_app.maybe_ledger_accept() + mc_app( + Payment(account=mc_app.account_from_alias('root'), + dst=alice, + amt=mc_asset(10_000))) + mc_app.maybe_ledger_accept() + + # create a trust line for adam + sc_app(Trust(account=adam, limit_amt=sc_asset(1_000_000))) + + for i in range(2): + # even amounts for main to side + for value in range(10, 20, 2): + with test_utils.test_context(mc_app, sc_app): + to_send_asset = mc_asset(value) + rcv_asset = sc_asset(value) + pre_bal = sc_app.get_balance(adam, rcv_asset) + sidechain.main_to_side_transfer(mc_app, sc_app, alice, adam, + to_send_asset, params) + test_utils.wait_for_balance_change(sc_app, adam, pre_bal, + rcv_asset) + + # side to main + # odd amounts for side to main + for value in range(9, 19, 2): + with test_utils.test_context(mc_app, sc_app): + to_send_asset = sc_asset(value) + rcv_asset = mc_asset(value) + pre_bal = mc_app.get_balance(alice, to_send_asset) + sidechain.side_to_main_transfer(mc_app, sc_app, adam, alice, + to_send_asset, params) + test_utils.wait_for_balance_change(mc_app, alice, pre_bal, + rcv_asset) + + +def run(mc_app: App, sc_app: App, params: Params): + # process will run while stop token is non-zero + stop_token = Value('i', 1) + p = None + if mc_app.standalone: + p = Process(target=sidechain.close_mainchain_ledgers, + args=(stop_token, params)) + p.start() + try: + # TODO: Tests fail without this sleep. Fix this bug. + time.sleep(10) + setup_accounts(mc_app, sc_app, params) + simple_xrp_test(mc_app, sc_app, params) + simple_iou_test(mc_app, sc_app, params) + finally: + if p: + stop_token.value = 0 + p.join() + sidechain._convert_log_files_to_json( + mc_app.get_configs() + sc_app.get_configs(), 'final.json') + + +def standalone_test(params: Params): + def callback(mc_app: App, sc_app: App): + run(mc_app, sc_app, params) + + sidechain._standalone_with_callback(params, + callback, + setup_user_accounts=False) + + +def setup_accounts(mc_app: App, sc_app: App, params: Params): + # Setup a funded user account on the main chain, and add an unfunded account. + # Setup address book and add a funded account on the mainchain. + # Typical female names are addresses on the mainchain. + # The first account is funded. + alice = mc_app.create_account('alice') + beth = mc_app.create_account('beth') + carol = mc_app.create_account('carol') + deb = mc_app.create_account('deb') + ella = mc_app.create_account('ella') + mc_app(Payment(account=params.genesis_account, dst=alice, amt=XRP(20_000))) + mc_app.maybe_ledger_accept() + + # Typical male names are addresses on the sidechain. + # All accounts are initially unfunded + adam = sc_app.create_account('adam') + bob = sc_app.create_account('bob') + charlie = sc_app.create_account('charlie') + dan = sc_app.create_account('dan') + ed = sc_app.create_account('ed') + + +def multinode_test(params: Params): + def callback(mc_app: App, sc_app: App): + run(mc_app, sc_app, params) + + sidechain._multinode_with_callback(params, + callback, + setup_user_accounts=False) + + +def test_simple_xchain(configs_dirs_dict: Dict[int, str]): + params = sidechain.Params(configs_dir=configs_dirs_dict[1]) + + if err_str := params.check_error(): + eprint(err_str) + sys.exit(1) + + if params.verbose: + print("eprint enabled") + else: + disable_eprint() + + # Set to true to help debug tests + test_utils.test_context_verbose_logging = True + + if params.standalone: + standalone_test(params) + else: + multinode_test(params) diff --git a/bin/sidechain/python/transaction.py b/bin/sidechain/python/transaction.py new file mode 100644 index 0000000000..4aac4b466d --- /dev/null +++ b/bin/sidechain/python/transaction.py @@ -0,0 +1,366 @@ +import datetime +import json +from typing import Dict, List, Optional, Union + +from command import Command +from common import Account, Asset, Path, PathList, to_rippled_epoch + + +class Transaction(Command): + '''Interface for all transactions''' + def __init__( + self, + *, + account: Account, + flags: Optional[int] = None, + fee: Optional[Union[Asset, int]] = None, + sequence: Optional[int] = None, + account_txn_id: Optional[str] = None, + last_ledger_sequence: Optional[int] = None, + src_tag: Optional[int] = None, + memos: Optional[List[Dict[str, dict]]] = None, + ): + super().__init__() + self.account = account + # set even if None + self.flags = flags + self.fee = fee + self.sequence = sequence + self.account_txn_id = account_txn_id + self.last_ledger_sequence = last_ledger_sequence + self.src_tag = src_tag + self.memos = memos + + def cmd_name(self) -> str: + return 'submit' + + def set_seq_and_fee(self, seq: int, fee: Union[Asset, int]): + self.sequence = seq + self.fee = fee + + def to_cmd_obj(self) -> dict: + txn = { + 'Account': self.account.account_id, + } + if self.flags is not None: + txn['Flags'] = flags + if self.fee is not None: + if isinstance(self.fee, int): + txn['Fee'] = f'{self.fee}' # must be a string + else: + txn['Fee'] = self.fee.to_cmd_obj() + if self.sequence is not None: + txn['Sequence'] = self.sequence + if self.account_txn_id is not None: + txn['AccountTxnID'] = self.account_txn_id + if self.last_ledger_sequence is not None: + txn['LastLedgerSequence'] = self.last_ledger_sequence + if self.src_tag is not None: + txn['SourceTag'] = self.src_tag + if self.memos is not None: + txn['Memos'] = self.memos + return txn + + +class Payment(Transaction): + '''A payment transaction''' + def __init__(self, + *, + dst: Account, + amt: Asset, + send_max: Optional[Asset] = None, + paths: Optional[PathList] = None, + dst_tag: Optional[int] = None, + deliver_min: Optional[Asset] = None, + **rest): + super().__init__(**rest) + self.dst = dst + self.amt = amt + self.send_max = send_max + if paths is not None and isinstance(paths, Path): + # allow paths = Path([...]) special case + self.paths = PathList([paths]) + else: + self.paths = paths + self.dst_tag = dst_tag + self.deliver_min = deliver_min + + def set_partial_payment(self, value: bool = True): + '''Set or clear the partial payment flag''' + self._set_flag(0x0002_0000, value) + + def to_cmd_obj(self) -> dict: + '''convert to transaction form (suitable for using json.dumps or similar)''' + txn = super().to_cmd_obj() + txn = { + **txn, + 'TransactionType': 'Payment', + 'Destination': self.dst.account_id, + 'Amount': self.amt.to_cmd_obj(), + } + if self.paths is not None: + txn['Paths'] = self.paths.to_cmd_obj() + if self.send_max is not None: + txn['SendMax'] = self.send_max.to_cmd_obj() + if self.dst_tag is not None: + txn['DestinationTag'] = self.dst_tag + if self.deliver_min is not None: + txn['DeliverMin'] = self.deliver_min + return txn + + +class Trust(Transaction): + '''A trust set transaction''' + def __init__(self, + *, + limit_amt: Optional[Asset] = None, + qin: Optional[int] = None, + qout: Optional[int] = None, + **rest): + super().__init__(**rest) + self.limit_amt = limit_amt + self.qin = qin + self.qout = qout + + def set_auth(self): + '''Set the auth flag (cannot be cleared)''' + self._set_flag(0x00010000) + return self + + def set_no_ripple(self, value: bool = True): + '''Set or clear the noRipple flag''' + self._set_flag(0x0002_0000, value) + self._set_flag(0x0004_0000, not value) + return self + + def set_freeze(self, value: bool = True): + '''Set or clear the freeze flag''' + self._set_flag(0x0020_0000, value) + self._set_flag(0x0040_0000, not value) + return self + + def to_cmd_obj(self) -> dict: + '''convert to transaction form (suitable for using json.dumps or similar)''' + result = super().to_cmd_obj() + result = { + **result, + 'TransactionType': 'TrustSet', + 'LimitAmount': self.limit_amt.to_cmd_obj(), + } + if self.qin is not None: + result['QualityIn'] = self.qin + if self.qout is not None: + result['QualityOut'] = self.qout + return result + + +class SetRegularKey(Transaction): + '''A SetRegularKey transaction''' + def __init__(self, *, key: str, **rest): + super().__init__(**rest) + self.key = key + + def to_cmd_obj(self) -> dict: + '''convert to transaction form (suitable for using json.dumps or similar)''' + result = super().to_cmd_obj() + result = { + **result, + 'TransactionType': 'SetRegularKey', + 'RegularKey': self.key, + } + return result + + +class SignerListSet(Transaction): + '''A SignerListSet transaction''' + def __init__(self, + *, + keys: List[str], + weights: Optional[List[int]] = None, + quorum: int, + **rest): + super().__init__(**rest) + self.keys = keys + self.quorum = quorum + if weights: + if len(weights) != len(keys): + raise ValueError( + f'SignerSetList number of weights must equal number of keys (or be empty). Weights: {weights} Keys: {keys}' + ) + self.weights = weights + else: + self.weights = [1] * len(keys) + + def to_cmd_obj(self) -> dict: + '''convert to transaction form (suitable for using json.dumps or similar)''' + result = super().to_cmd_obj() + result = { + **result, + 'TransactionType': 'SignerListSet', + 'SignerQuorum': self.quorum, + } + entries = [] + for k, w in zip(self.keys, self.weights): + entries.append({'SignerEntry': {'Account': k, 'SignerWeight': w}}) + result['SignerEntries'] = entries + return result + + +class AccountSet(Transaction): + '''An account set transaction''' + def __init__(self, account: Account, **rest): + super().__init__(account=account, **rest) + self.clear_flag = None + self.set_flag = None + self.transfer_rate = None + self.tick_size = None + + def _set_account_flag(self, flag_id: int, value): + if value: + self.set_flag = flag_id + else: + self.clear_flag = flag_id + return self + + def set_account_txn_id(self, value: bool = True): + '''Set or clear the asfAccountTxnID flag''' + return self._set_account_flag(5, value) + + def set_default_ripple(self, value: bool = True): + '''Set or clear the asfDefaultRipple flag''' + return self._set_account_flag(8, value) + + def set_deposit_auth(self, value: bool = True): + '''Set or clear the asfDepositAuth flag''' + return self._set_account_flag(9, value) + + def set_disable_master(self, value: bool = True): + '''Set or clear the asfDisableMaster flag''' + return self._set_account_flag(4, value) + + def set_disallow_xrp(self, value: bool = True): + '''Set or clear the asfDisallowXRP flag''' + return self._set_account_flag(3, value) + + def set_global_freeze(self, value: bool = True): + '''Set or clear the asfGlobalFreeze flag''' + return self._set_account_flag(7, value) + + def set_no_freeze(self, value: bool = True): + '''Set or clear the asfNoFreeze flag''' + return self._set_account_flag(6, value) + + def set_require_auth(self, value: bool = True): + '''Set or clear the asfRequireAuth flag''' + return self._set_account_flag(2, value) + + def set_require_dest(self, value: bool = True): + '''Set or clear the asfRequireDest flag''' + return self._set_account_flag(1, value) + + def set_transfer_rate(self, value: int): + '''Set the fee to change when users transfer this account's issued currencies''' + self.transfer_rate = value + return self + + def set_tick_size(self, value: int): + '''Tick size to use for offers involving a currency issued by this address''' + self.tick_size = value + return self + + def to_cmd_obj(self) -> dict: + '''convert to transaction form (suitable for using json.dumps or similar)''' + result = super().to_cmd_obj() + result = { + **result, + 'TransactionType': 'AccountSet', + } + if self.clear_flag is not None: + result['ClearFlag'] = self.clear_flag + if self.set_flag is not None: + result['SetFlag'] = self.set_flag + if self.transfer_rate is not None: + result['TransferRate'] = self.transfer_rate + if self.tick_size is not None: + result['TickSize'] = self.tick_size + return result + + +class Offer(Transaction): + '''An offer transaction''' + def __init__(self, + *, + taker_pays: Asset, + taker_gets: Asset, + expiration: Optional[int] = None, + offer_sequence: Optional[int] = None, + **rest): + super().__init__(**rest) + self.taker_pays = taker_pays + self.taker_gets = taker_gets + self.expiration = expiration + self.offer_sequence = offer_sequence + + def set_passive(self, value: bool = True): + return self._set_flag(0x0001_0000, value) + + def set_immediate_or_cancel(self, value: bool = True): + return self._set_flag(0x0002_0000, value) + + def set_fill_or_kill(self, value: bool = True): + return self._set_flag(0x0004_0000, value) + + def set_sell(self, value: bool = True): + return self._set_flag(0x0008_0000, value) + + def to_cmd_obj(self) -> dict: + txn = super().to_cmd_obj() + txn = { + **txn, + 'TransactionType': 'OfferCreate', + 'TakerPays': self.taker_pays.to_cmd_obj(), + 'TakerGets': self.taker_gets.to_cmd_obj(), + } + if self.expiration is not None: + txn['Expiration'] = self.expiration + if self.offer_sequence is not None: + txn['OfferSequence'] = self.offer_sequence + return txn + + +class Ticket(Transaction): + '''A ticket create transaction''' + def __init__(self, *, count: int = 1, **rest): + super().__init__(**rest) + self.count = count + + def to_cmd_obj(self) -> dict: + txn = super().to_cmd_obj() + txn = { + **txn, + 'TransactionType': 'TicketCreate', + 'TicketCount': self.count, + } + return txn + + +class SetHook(Transaction): + '''A SetHook transaction for the experimental hook amendment''' + def __init__(self, + *, + create_code: str, + hook_on: str = '0000000000000000', + **rest): + super().__init__(**rest) + self.create_code = create_code + self.hook_on = hook_on + + def to_cmd_obj(self) -> dict: + txn = super().to_cmd_obj() + txn = { + **txn, + 'TransactionType': 'SetHook', + 'CreateCode': self.create_code, + 'HookOn': self.hook_on, + } + return txn