ARCHIVE/rippled
1
0
Fork 0
mirror of https://github.com/XRPLF/rippled.git synced 2026-06-05 01:37:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Sidechain python test environment and repl

Browse Source
This commit is contained in:
seelabs
2021-09-10 11:25:17 -04:00
parent e56a85cb3b
commit 24cf8ab8c7
15 changed files with 5574 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

612
bin/sidechain/python/app.py Normal file
View File

@@ -0,0 +1,612 @@
from contextlib import contextmanager
import json
import os
import pandas as pd
from pathlib import Path
import subprocess
import time
from typing import Callable, Dict, List, Optional, Set, Union
from ripple_client import RippleClient
from common import Account, Asset, XRP
from command import AccountInfo, AccountLines, BookOffers, Command, FederatorInfo, LedgerAccept, Sign, Submit, SubscriptionCommand, WalletPropose
from config_file import ConfigFile
import testnet
from transaction import Payment, Transaction
class KeyManager:
def __init__(self):
self._aliases = {} # alias -> account
self._accounts = {} # account id -> account
def add(self, account: Account) -> bool:
if account.nickname:
self._aliases[account.nickname] = account
self._accounts[account.account_id] = account
def is_alias(self, name: str):
return name in self._aliases
def account_from_alias(self, name: str) -> Account:
assert name in self._aliases
return self._aliases[name]
def known_accounts(self) -> List[Account]:
return list(self._accounts.values())
def account_id_dict(self) -> Dict[str, Account]:
return self._accounts
def alias_or_account_id(self, id: Union[Account, str]) -> str:
'''
return the alias if it exists, otherwise return the id
'''
if isinstance(id, Account):
return id.alias_or_account_id()
if id in self._accounts:
return self._accounts[id].nickname
return id
def alias_to_account_id(self, alias: str) -> Optional[str]:
if id in self._aliases:
return self._aliases[id].account_id
return None
def to_string(self, nickname: Optional[str] = None):
names = []
account_ids = []
if nickname:
names = [nickname]
if nickname in self._aliases:
account_ids = [self._aliases[nickname].account_id]
else:
account_id = ['NA']
else:
for (k, v) in self._aliases.items():
names.append(k)
account_ids.append(v.account_id)
# use a dataframe to get a nice table output
df = pd.DataFrame(data={'name': names, 'id': account_ids})
return f'{df.to_string(index=False)}'
class AssetAliases:
def __init__(self):
self._aliases = {} # alias -> asset
def add(self, asset: Asset, name: str):
self._aliases[name] = asset
def is_alias(self, name: str):
return name in self._aliases
def asset_from_alias(self, name: str) -> Asset:
assert name in self._aliases
return self._aliases[name]
def known_aliases(self) -> List[str]:
return list(self._aliases.keys())
def known_assets(self) -> List[Asset]:
return list(self._aliases.values())
def to_string(self, nickname: Optional[str] = None):
names = []
currencies = []
issuers = []
if nickname:
names = [nickname]
if nickname in self._aliases:
v = self._aliases[nickname]
currencies = [v.currency]
iss = v.issuer if v.issuer else ''
issuers = [v.issuer if v.issuer else '']
else:
currencies = ['NA']
issuers = ['NA']
else:
for (k, v) in self._aliases.items():
names.append(k)
currencies.append(v.currency)
issuers.append(v.issuer if v.issuer else '')
# use a dataframe to get a nice table output
df = pd.DataFrame(data={
'name': names,
'currency': currencies,
'issuer': issuers
})
return f'{df.to_string(index=False)}'
class App:
'''App to to interact with rippled servers'''
def __init__(self,
*,
standalone: bool,
network: Optional[testnet.Network] = None,
client: Optional[RippleClient] = None):
if network and client:
raise ValueError('Cannot specify both a testnet and client in App')
if not network and not client:
raise ValueError('Must specify a testnet or a client in App')
self.standalone = standalone
self.network = network
if client:
self.client = client
else:
self.client = self.network.get_client(0)
self.key_manager = KeyManager()
self.asset_aliases = AssetAliases()
root_account = Account(nickname='root',
account_id='rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh',
secret_key='masterpassphrase')
self.key_manager.add(root_account)
def shutdown(self):
if self.network:
self.network.shutdown()
else:
self.client.shutdown()
def send_signed(self, txn: Transaction) -> dict:
'''Sign then send the given transaction'''
if not txn.account.secret_key:
raise ValueError('Cannot sign transaction without secret key')
r = self(Sign(txn.account.secret_key, txn.to_cmd_obj()))
raw_signed = r['tx_blob']
return self(Submit(raw_signed))
def send_command(self, cmd: Command) -> dict:
'''Send the command to the rippled server'''
return self.client.send_command(cmd)
# Need async version to close ledgers from async functions
async def async_send_command(self, cmd: Command) -> dict:
'''Send the command to the rippled server'''
return await self.client.async_send_command(cmd)
def send_subscribe_command(
self,
cmd: SubscriptionCommand,
callback: Optional[Callable[[dict], None]] = None) -> dict:
'''Send the subscription command to the rippled server. If already subscribed, it will unsubscribe'''
return self.client.send_subscribe_command(cmd, callback)
def get_pids(self) -> List[int]:
if self.network:
return self.network.get_pids()
if pid := self.client.get_pid():
return [pid]
def get_running_status(self) -> List[bool]:
if self.network:
return self.network.get_running_status()
if self.client.get_pid():
return [True]
else:
return [False]
# Get a dict of the server_state, validated_ledger_seq, and complete_ledgers
def get_brief_server_info(self) -> dict:
if self.network:
return self.network.get_brief_server_info()
else:
ret = {}
for (k, v) in self.client.get_brief_server_info().items():
ret[k] = [v]
return ret
def servers_start(self,
server_indexes: Optional[Union[Set[int],
List[int]]] = None,
*,
extra_args: Optional[List[List[str]]] = None):
if self.network:
return self.network.servers_start(server_indexes,
extra_args=extra_args)
else:
raise ValueError('Cannot start stand alone server')
def servers_stop(self,
server_indexes: Optional[Union[Set[int],
List[int]]] = None):
if self.network:
return self.network.servers_stop(server_indexes)
else:
raise ValueError('Cannot stop stand alone server')
def federator_info(self,
server_indexes: Optional[Union[Set[int],
List[int]]] = None):
# key is server index. value is federator_info result
result_dict = {}
if self.network:
if not server_indexes:
server_indexes = [
i for i in range(self.network.num_clients())
if self.network.is_running(i)
]
for i in server_indexes:
if self.network.is_running(i):
result_dict[i] = self.network.get_client(i).send_command(
FederatorInfo())
else:
if 0 in server_indexes:
result_dict[0] = self.client.send_command(FederatorInfo())
return result_dict
def __call__(self,
to_send: Union[Transaction, Command, SubscriptionCommand],
callback: Optional[Callable[[dict], None]] = None,
*,
insert_seq_and_fee=False) -> dict:
'''Call `send_signed` for transactions or `send_command` for commands'''
if isinstance(to_send, SubscriptionCommand):
return self.send_subscribe_command(to_send, callback)
assert callback is None
if isinstance(to_send, Transaction):
if insert_seq_and_fee:
self.insert_seq_and_fee(to_send)
return self.send_signed(to_send)
if isinstance(to_send, Command):
return self.send_command(to_send)
raise ValueError(
'Expected `to_send` to be either a Transaction, Command, or SubscriptionCommand'
)
def get_configs(self) -> List[str]:
if self.network:
return self.network.get_configs()
return [self.client.config]
def create_account(self, name: str) -> Account:
''' Create an account. Use the name as the alias. '''
if name == 'root':
return
assert not self.key_manager.is_alias(name)
account = Account(nickname=name, result_dict=self(WalletPropose()))
self.key_manager.add(account)
return account
def create_accounts(self,
names: List[str],
funding_account: Union[Account, str] = 'root',
amt: Union[int, Asset] = 1000000000) -> List[Account]:
'''Fund the accounts with nicknames 'names' by using the funding account and amt'''
accounts = [self.create_account(n) for n in names]
if not isinstance(funding_account, Account):
org_funding_account = funding_account
funding_account = self.key_manager.account_from_alias(
funding_account)
if not isinstance(funding_account, Account):
raise ValueError(
f'Could not find funding account {org_funding_account}')
if not isinstance(amt, Asset):
assert isinstance(amt, int)
amt = Asset(value=amt)
for a in accounts:
p = Payment(account=funding_account, dst=a, amt=amt)
self.send_signed(p)
return accounts
def maybe_ledger_accept(self):
if not self.standalone:
return
self(LedgerAccept())
# Need async version to close ledgers from async functions
async def async_maybe_ledger_accept(self):
if not self.standalone:
return
await self.async_send_command(LedgerAccept())
def get_balances(
self,
account: Union[Account, List[Account], None] = None,
asset: Union[Asset, List[Asset]] = Asset()
) -> pd.DataFrame:
'''Return a pandas dataframe of account balances. If account is None, treat as a wildcard (use address book)'''
if account is None:
account = self.key_manager.known_accounts()
if isinstance(account, list):
result = [self.get_balances(acc, asset) for acc in account]
return pd.concat(result, ignore_index=True)
if isinstance(asset, list):
result = [self.get_balances(account, ass) for ass in asset]
return pd.concat(result, ignore_index=True)
if asset.is_xrp():
try:
df = self.get_account_info(account)
except:
# Most likely the account does not exist on the ledger. Give a balance of zero.
df = pd.DataFrame({
'account': [account],
'balance': [0],
'flags': [0],
'owner_count': [0],
'previous_txn_id': ['NA'],
'previous_txn_lgr_seq': [-1],
'sequence': [-1]
})
df = df.assign(currency='XRP', peer='', limit='')
return df.loc[:,
['account', 'balance', 'currency', 'peer', 'limit']]
else:
try:
df = self.get_trust_lines(account)
if df.empty: return df
df = df[(df['peer'] == asset.issuer.account_id)
& (df['currency'] == asset.currency)]
except:
# Most likely the account does not exist on the ledger. Return an empty data frame
df = pd.DataFrame({
'account': [],
'balance': [],
'currency': [],
'peer': [],
'limit': [],
})
return df.loc[:,
['account', 'balance', 'currency', 'peer', 'limit']]
def get_balance(self, account: Account, asset: Asset) -> Asset:
'''Get a balance from a single account in a single asset'''
try:
df = self.get_balances(account, asset)
return asset(df.iloc[0]['balance'])
except:
return asset(0)
def get_account_info(self,
account: Optional[Account] = None) -> pd.DataFrame:
'''Return a pandas dataframe of account info. If account is None, treat as a wildcard (use address book)'''
if account is None:
known_accounts = self.key_manager.known_accounts()
result = [self.get_account_info(acc) for acc in known_accounts]
return pd.concat(result, ignore_index=True)
try:
result = self.client.send_command(AccountInfo(account))
except:
# Most likely the account does not exist on the ledger. Give a balance of zero.
return pd.DataFrame({
'account': [account],
'balance': [0],
'flags': [0],
'owner_count': [0],
'previous_txn_id': ['NA'],
'previous_txn_lgr_seq': [-1],
'sequence': [-1]
})
if 'account_data' not in result:
raise ValueError('Bad result from account_info command')
info = result['account_data']
for dk in ['LedgerEntryType', 'index']:
del info[dk]
df = pd.DataFrame([info])
df.rename(columns={
'Account': 'account',
'Balance': 'balance',
'Flags': 'flags',
'OwnerCount': 'owner_count',
'PreviousTxnID': 'previous_txn_id',
'PreviousTxnLgrSeq': 'previous_txn_lgr_seq',
'Sequence': 'sequence'
},
inplace=True)
df['balance'] = df['balance'].astype(int)
return df
def get_trust_lines(self,
account: Account,
peer: Optional[Account] = None) -> pd.DataFrame:
'''Return a pandas dataframe account trust lines. If peer account is None, treat as a wildcard'''
result = self.send_command(AccountLines(account, peer=peer))
if 'lines' not in result or 'account' not in result:
raise ValueError('Bad result from account_lines command')
account = result['account']
lines = result['lines']
for d in lines:
d['peer'] = d['account']
d['account'] = account
return pd.DataFrame(lines)
def get_offers(self, taker_pays: Asset, taker_gets: Asset) -> pd.DataFrame:
'''Return a pandas dataframe of offers'''
result = self.send_command(BookOffers(taker_pays, taker_gets))
if 'offers' not in result:
raise ValueError('Bad result from book_offers command')
offers = result['offers']
delete_keys = [
'BookDirectory', 'BookNode', 'LedgerEntryType', 'OwnerNode',
'PreviousTxnID', 'PreviousTxnLgrSeq', 'Sequence', 'index'
]
for d in offers:
for dk in delete_keys:
del d[dk]
for t in ['TakerPays', 'TakerGets', 'owner_funds']:
if 'value' in d[t]:
d[t] = d[t]['value']
df = pd.DataFrame(offers)
df.rename(columns={
'Account': 'account',
'Flags': 'flags',
'TakerGets': 'taker_gets',
'TakerPays': 'taker_pays'
},
inplace=True)
return df
def account_balance(self, account: Account, asset: Asset) -> Asset:
'''get the account's balance of the asset'''
pass
def substitute_nicknames(
self,
df: pd.DataFrame,
cols: List[str] = ['account', 'peer']) -> pd.DataFrame:
result = df.copy(deep=True)
for c in cols:
if c not in result:
continue
result[c] = result[c].map(
lambda x: self.key_manager.alias_or_account_id(x))
return result
def add_to_keymanager(self, account: Account):
self.key_manager.add(account)
def is_alias(self, name: str) -> bool:
return self.key_manager.is_alias(name)
def account_from_alias(self, name: str) -> Account:
return self.key_manager.account_from_alias(name)
def known_accounts(self) -> List[Account]:
return self.key_manager.known_accounts()
def known_asset_aliases(self) -> List[str]:
return self.asset_aliases.known_aliases()
def known_iou_assets(self) -> List[Asset]:
return self.asset_aliases.known_assets()
def is_asset_alias(self, name: str) -> bool:
return self.asset_aliases.is_alias(name)
def add_asset_alias(self, asset: Asset, name: str):
self.asset_aliases.add(asset, name)
def asset_from_alias(self, name: str) -> Asset:
return self.asset_aliases.asset_from_alias(name)
def insert_seq_and_fee(self, txn: Transaction):
acc_info = self(AccountInfo(txn.account))
# TODO: set better fee (Hard code a fee of 15 for now)
txn.set_seq_and_fee(acc_info['account_data']['Sequence'], 15)
def get_client(self) -> RippleClient:
return self.client
def balances_dataframe(chains: List[App],
chain_names: List[str],
account_ids: Optional[List[Account]] = None,
assets: List[Asset] = None,
in_drops: bool = False):
def _removesuffix(self: str, suffix: str) -> str:
if suffix and self.endswith(suffix):
return self[:-len(suffix)]
else:
return self[:]
def _balance_df(chain: App, acc: Optional[Account],
asset: Union[Asset, List[Asset]], in_drops: bool):
b = chain.get_balances(acc, asset)
if not in_drops:
b.loc[b['currency'] == 'XRP', 'balance'] /= 1_000_000
b = chain.substitute_nicknames(b)
b = b.set_index('account')
return b
if account_ids is None:
account_ids = [None] * len(chains)
if assets is None:
# XRP and all assets in the assets alias list
assets = [[XRP(0)] + c.known_iou_assets() for c in chains]
dfs = []
keys = []
for chain, chain_name, acc, asset in zip(chains, chain_names, account_ids,
assets):
dfs.append(_balance_df(chain, acc, asset, in_drops))
keys.append(_removesuffix(chain_name, 'chain'))
df = pd.concat(dfs, keys=keys)
return df
# Start an app with a single client
@contextmanager
def single_client_app(*,
config: ConfigFile,
command_log: Optional[str] = None,
server_out=os.devnull,
run_server: bool = True,
exe: Optional[str] = None,
extra_args: Optional[List[str]] = None,
standalone=False):
'''Start a ripple server and return an app'''
try:
if extra_args is None:
extra_args = []
to_run = None
app = None
client = RippleClient(config=config, command_log=command_log, exe=exe)
if run_server:
to_run = [client.exe, '--conf', client.config_file_name]
if standalone:
to_run.append('-a')
fout = open(server_out, 'w')
p = subprocess.Popen(to_run + extra_args,
stdout=fout,
stderr=subprocess.STDOUT)
client.set_pid(p.pid)
print(
f'started rippled: config: {client.config_file_name} PID: {p.pid}',
flush=True)
time.sleep(1.5) # give process time to startup
app = App(client=client, standalone=standalone)
yield app
finally:
if app:
app.shutdown()
if run_server and to_run:
subprocess.Popen(to_run + ['stop'],
stdout=fout,
stderr=subprocess.STDOUT)
p.wait()
def configs_for_testnet(config_file_prefix: str) -> List[ConfigFile]:
configs = []
p = Path(config_file_prefix)
dir = p.parent
file = p.name
file_names = []
for f in os.listdir(dir):
cfg = os.path.join(dir, f, 'rippled.cfg')
if f.startswith(file) and os.path.exists(cfg):
file_names.append(cfg)
file_names.sort()
return [ConfigFile(file_name=f) for f in file_names]
# Start an app for a network with the config files matched by `config_file_prefix*/rippled.cfg`
@contextmanager
def testnet_app(*,
exe: str,
configs: List[ConfigFile],
command_logs: Optional[List[str]] = None,
run_server: Optional[List[bool]] = None,
extra_args: Optional[List[List[str]]] = None):
'''Start a ripple testnet and return an app'''
try:
app = None
network = testnet.Network(exe,
configs,
command_logs=command_logs,
run_server=run_server,
extra_args=extra_args)
network.wait_for_validated_ledger()
app = App(network=network, standalone=False)
yield app
finally:
if app:
app.shutdown()

557
bin/sidechain/python/command.py Normal file
View File

@@ -0,0 +1,557 @@
import json
from typing import List, Optional, Union
from common import Account, Asset
class Command:
'''Interface for all commands sent to the server'''
# command id useful for websocket messages
next_cmd_id = 1
def __init__(self):
self.cmd_id = Command.next_cmd_id
Command.next_cmd_id += 1
def cmd_name(self) -> str:
'''Return the command name for use in a command line'''
assert False
return ''
def get_command_line_list(self) -> List[str]:
'''Return a list of strings suitable for a command line command for a rippled server'''
return [self.cmd_name, json.dumps(self.to_cmd_obj())]
def get_websocket_dict(self) -> dict:
'''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket'''
result = self.to_cmd_obj()
return self.add_websocket_fields(result)
def to_cmd_obj(self) -> dict:
'''Return an object suitalbe for use in a command (input to json.dumps or similar)'''
assert False
return {}
def add_websocket_fields(self, cmd_dict: dict) -> dict:
cmd_dict['id'] = self.cmd_id
cmd_dict['command'] = self.cmd_name()
return cmd_dict
def _set_flag(self, flag_bit: int, value: bool = True):
'''Set or clear the flag bit'''
if value:
self.flags |= flag_bit
else:
self.flags &= ~flag_bit
return self
class SubscriptionCommand(Command):
def __init__(self):
super().__init__()
class PathFind(Command):
'''Rippled ripple_path_find command'''
def __init__(self,
*,
src: Account,
dst: Account,
amt: Asset,
send_max: Optional[Asset] = None,
src_currencies: Optional[List[Asset]] = None,
ledger_hash: Optional[str] = None,
ledger_index: Optional[Union[int, str]] = None):
super().__init__()
self.src = src
self.dst = dst
self.amt = amt
self.send_max = send_max
self.src_currencies = src_currencies
self.ledger_hash = ledger_hash
self.ledger_index = ledger_index
def cmd_name(self) -> str:
return 'ripple_path_find'
def add_websocket_fields(self, cmd_dict: dict) -> dict:
cmd_dict = super().add_websocket_fields(cmd_dict)
cmd_dict['subcommand'] = 'create'
return cmd_dict
def to_cmd_obj(self) -> dict:
'''convert to transaction form (suitable for using json.dumps or similar)'''
cmd = {
'source_account': self.src.account_id,
'destination_account': self.dst.account_id,
'destination_amount': self.amt.to_cmd_obj()
}
if self.send_max is not None:
cmd['send_max'] = self.send_max.to_cmd_obj()
if self.ledger_hash is not None:
cmd['ledger_hash'] = self.ledger_hash
if self.ledger_index is not None:
cmd['ledger_index'] = self.ledger_index
if self.src_currencies:
c = []
for sc in self.src_currencies:
d = {'currency': sc.currency, 'issuer': sc.issuer.account_id}
c.append(d)
cmd['source_currencies'] = c
return cmd
class Sign(Command):
'''Rippled sign command'''
def __init__(self, secret: str, tx: dict):
super().__init__()
self.tx = tx
self.secret = secret
def cmd_name(self) -> str:
return 'sign'
def get_command_line_list(self) -> List[str]:
'''Return a list of strings suitable for a command line command for a rippled server'''
return [self.cmd_name(), self.secret, f'{json.dumps(self.tx)}']
def get_websocket_dict(self) -> dict:
'''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket'''
result = {'secret': self.secret, 'tx_json': self.tx}
return self.add_websocket_fields(result)
class Submit(Command):
'''Rippled submit command'''
def __init__(self, tx_blob: str):
super().__init__()
self.tx_blob = tx_blob
def cmd_name(self) -> str:
return 'submit'
def get_command_line_list(self) -> List[str]:
'''Return a list of strings suitable for a command line command for a rippled server'''
return [self.cmd_name(), self.tx_blob]
def get_websocket_dict(self) -> dict:
'''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket'''
result = {'tx_blob': self.tx_blob}
return self.add_websocket_fields(result)
class LedgerAccept(Command):
'''Rippled ledger_accept command'''
def __init__(self):
super().__init__()
def cmd_name(self) -> str:
return 'ledger_accept'
def get_command_line_list(self) -> List[str]:
'''Return a list of strings suitable for a command line command for a rippled server'''
return [self.cmd_name()]
def get_websocket_dict(self) -> dict:
'''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket'''
result = {}
return self.add_websocket_fields(result)
class Stop(Command):
'''Rippled stop command'''
def __init__(self):
super().__init__()
def cmd_name(self) -> str:
return 'stop'
def get_command_line_list(self) -> List[str]:
'''Return a list of strings suitable for a command line command for a rippled server'''
return [self.cmd_name()]
def get_websocket_dict(self) -> dict:
'''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket'''
result = {}
return self.add_websocket_fields(result)
class LogLevel(Command):
'''Rippled log_level command'''
def __init__(self, severity: str, *, partition: Optional[str] = None):
super().__init__()
self.severity = severity
self.partition = partition
def cmd_name(self) -> str:
return 'log_level'
def get_command_line_list(self) -> List[str]:
'''Return a list of strings suitable for a command line command for a rippled server'''
if self.partition is not None:
return [self.cmd_name(), self.partition, self.severity]
return [self.cmd_name(), self.severity]
def get_websocket_dict(self) -> dict:
'''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket'''
result = {'severity': self.severity}
if self.partition is not None:
result['partition'] = self.partition
return self.add_websocket_fields(result)
class WalletPropose(Command):
'''Rippled wallet_propose command'''
def __init__(self,
*,
passphrase: Optional[str] = None,
seed: Optional[str] = None,
seed_hex: Optional[str] = None,
key_type: Optional[str] = None):
super().__init__()
self.passphrase = passphrase
self.seed = seed
self.seed_hex = seed_hex
self.key_type = key_type
def cmd_name(self) -> str:
return 'wallet_propose'
def get_command_line_list(self) -> List[str]:
'''Return a list of strings suitable for a command line command for a rippled server'''
assert not self.seed and not self.seed_hex and (
not self.key_type or self.key_type == 'secp256k1')
if self.passphrase:
return [self.cmd_name(), self.passphrase]
return [self.cmd_name()]
def get_websocket_dict(self) -> dict:
'''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket'''
result = {}
if self.seed is not None:
result['seed'] = self.seed
if self.seed_hex is not None:
result['seed_hex'] = self.seed_hex
if self.passphrase is not None:
result['passphrase'] = self.passphrase
if self.key_type is not None:
result['key_type'] = self.key_type
return self.add_websocket_fields(result)
class ValidationCreate(Command):
'''Rippled validation_create command'''
def __init__(self, *, secret: Optional[str] = None):
super().__init__()
self.secret = secret
def cmd_name(self) -> str:
return 'validation_create'
def get_command_line_list(self) -> List[str]:
'''Return a list of strings suitable for a command line command for a rippled server'''
if self.secret:
return [self.cmd_name(), self.secret]
return [self.cmd_name()]
def get_websocket_dict(self) -> dict:
'''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket'''
result = {}
if self.secret is not None:
result['secret'] = self.secret
return self.add_websocket_fields(result)
class AccountInfo(Command):
'''Rippled account_info command'''
def __init__(self,
account: Account,
*,
strict: Optional[bool] = None,
ledger_hash: Optional[str] = None,
ledger_index: Optional[Union[str, int]] = None,
queue: Optional[bool] = None,
signers_list: Optional[bool] = None):
super().__init__()
self.account = account
self.strict = strict
self.ledger_hash = ledger_hash
self.ledger_index = ledger_index
self.queue = queue
self.signers_list = signers_list
assert not ((ledger_hash is not None) and (ledger_index is not None))
def cmd_name(self) -> str:
return 'account_info'
def get_command_line_list(self) -> List[str]:
'''Return a list of strings suitable for a command line command for a rippled server'''
result = [self.cmd_name(), self.account.account_id]
if self.ledger_index is not None:
result.append(self.ledger_index)
if self.ledger_hash is not None:
result.append(self.ledger_hash)
if self.strict is not None:
result.append(self.strict)
return result
def get_websocket_dict(self) -> dict:
'''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket'''
result = {'account': self.account.account_id}
if self.ledger_index is not None:
result['ledger_index'] = self.ledger_index
if self.ledger_hash is not None:
result['ledger_hash'] = self.ledger_hash
if self.strict is not None:
result['strict'] = self.strict
if self.queue is not None:
result['queue'] = self.queue
return self.add_websocket_fields(result)
class AccountLines(Command):
'''Rippled account_lines command'''
def __init__(self,
account: Account,
*,
peer: Optional[Account] = None,
ledger_hash: Optional[str] = None,
ledger_index: Optional[Union[str, int]] = None,
limit: Optional[int] = None,
marker=None):
super().__init__()
self.account = account
self.peer = peer
self.ledger_hash = ledger_hash
self.ledger_index = ledger_index
self.limit = limit
self.marker = marker
assert not ((ledger_hash is not None) and (ledger_index is not None))
def cmd_name(self) -> str:
return 'account_lines'
def get_command_line_list(self) -> List[str]:
'''Return a list of strings suitable for a command line command for a rippled server'''
assert sum(x is None for x in [
self.ledger_index, self.ledger_hash, self.limit, self.marker
]) == 4
result = [self.cmd_name(), self.account.account_id]
if self.peer is not None:
result.append(self.peer)
return result
def get_websocket_dict(self) -> dict:
'''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket'''
result = {'account': self.account.account_id}
if self.peer is not None:
result['peer'] = self.peer
if self.ledger_index is not None:
result['ledger_index'] = self.ledger_index
if self.ledger_hash is not None:
result['ledger_hash'] = self.ledger_hash
if self.limit is not None:
result['limit'] = self.limit
if self.marker is not None:
result['marker'] = self.marker
return self.add_websocket_fields(result)
class AccountTx(Command):
def __init__(self,
account: Account,
*,
limit: Optional[int] = None,
marker=None):
super().__init__()
self.account = account
self.limit = limit
self.marker = marker
def cmd_name(self) -> str:
return 'account_tx'
def get_command_line_list(self) -> List[str]:
'''Return a list of strings suitable for a command line command for a rippled server'''
result = [self.cmd_name(), self.account.account_id]
return result
def get_websocket_dict(self) -> dict:
'''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket'''
result = {'account': self.account.account_id}
if self.limit is not None:
result['limit'] = self.limit
if self.marker is not None:
result['marker'] = self.marker
return self.add_websocket_fields(result)
class BookOffers(Command):
'''Rippled book_offers command'''
def __init__(self,
taker_pays: Asset,
taker_gets: Asset,
*,
taker: Optional[Account] = None,
ledger_hash: Optional[str] = None,
ledger_index: Optional[Union[str, int]] = None,
limit: Optional[int] = None,
marker=None):
super().__init__()
self.taker_pays = taker_pays
self.taker_gets = taker_gets
self.taker = taker
self.ledger_hash = ledger_hash
self.ledger_index = ledger_index
self.limit = limit
self.marker = marker
assert not ((ledger_hash is not None) and (ledger_index is not None))
def cmd_name(self) -> str:
return 'book_offers'
def get_command_line_list(self) -> List[str]:
'''Return a list of strings suitable for a command line command for a rippled server'''
assert sum(x is None for x in [
self.ledger_index, self.ledger_hash, self.limit, self.marker
]) == 4
return [
self.cmd_name(),
self.taker_pays.cmd_str(),
self.taker_gets.cmd_str()
]
def get_websocket_dict(self) -> dict:
'''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket'''
result = {
'taker_pays': self.taker_pays.to_cmd_obj(),
'taker_gets': self.taker_gets.to_cmd_obj()
}
if self.taker is not None:
result['taker'] = self.taker.account_id
if self.ledger_index is not None:
result['ledger_index'] = self.ledger_index
if self.ledger_hash is not None:
result['ledger_hash'] = self.ledger_hash
if self.limit is not None:
result['limit'] = self.limit
if self.marker is not None:
result['marker'] = self.marker
return self.add_websocket_fields(result)
class BookSubscription:
'''Spec for a book in a subscribe command'''
def __init__(self,
taker_pays: Asset,
taker_gets: Asset,
*,
taker: Optional[Account] = None,
snapshot: Optional[bool] = None,
both: Optional[bool] = None):
self.taker_pays = taker_pays
self.taker_gets = taker_gets
self.taker = taker
self.snapshot = snapshot
self.both = both
def to_cmd_obj(self) -> dict:
'''Return an object suitalbe for use in a command'''
result = {
'taker_pays': self.taker_pays.to_cmd_obj(),
'taker_gets': self.taker_gets.to_cmd_obj()
}
if self.taker is not None:
result['taker'] = self.taker.account_id
if self.snapshot is not None:
result['snapshot'] = self.snapshot
if self.both is not None:
result['both'] = self.both
return result
class ServerInfo(Command):
'''Rippled server_info command'''
def __init__(self):
super().__init__()
def cmd_name(self) -> str:
return 'server_info'
def get_command_line_list(self) -> List[str]:
'''Return a list of strings suitable for a command line command for a rippled server'''
return [self.cmd_name()]
def get_websocket_dict(self) -> dict:
'''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket'''
result = {}
return self.add_websocket_fields(result)
class FederatorInfo(Command):
'''Rippled federator_info command'''
def __init__(self):
super().__init__()
def cmd_name(self) -> str:
return 'federator_info'
def get_command_line_list(self) -> List[str]:
'''Return a list of strings suitable for a command line command for a rippled server'''
return [self.cmd_name()]
def get_websocket_dict(self) -> dict:
'''Return a dictionary suitable for converting to json and sending to a rippled server using a websocket'''
result = {}
return self.add_websocket_fields(result)
class Subscribe(SubscriptionCommand):
'''The subscribe method requests periodic notifications from the server
when certain events happen. See: https://developers.ripple.com/subscribe.html'''
def __init__(
self,
*,
streams: Optional[List[str]] = None,
accounts: Optional[List[Account]] = None,
accounts_proposed: Optional[List[Account]] = None,
books: Optional[
List[BookSubscription]] = None, # taker_pays, taker_gets
url: Optional[str] = None,
url_username: Optional[str] = None,
url_password: Optional[str] = None):
super().__init__()
self.streams = streams
self.accounts = accounts
self.accounts_proposed = accounts_proposed
self.books = books
self.url = url
self.url_username = url_username
self.url_password = url_password
self.websocket = None
def cmd_name(self) -> str:
if self.websocket:
return 'unsubscribe'
return 'subscribe'
def to_cmd_obj(self) -> dict:
d = {}
if self.streams is not None:
d['streams'] = self.streams
if self.accounts is not None:
d['accounts'] = [a.account_id for a in self.accounts]
if self.accounts_proposed is not None:
d['accounts_proposed'] = [
a.account_id for a in self.accounts_proposed
]
if self.books is not None:
d['books'] = [b.to_cmd_obj() for b in self.books]
if self.url is not None:
d['url'] = self.url
if self.url_username is not None:
d['url_username'] = self.url_username
if self.url_password is not None:
d['url_password'] = self.url_password
return d

250
bin/sidechain/python/common.py Normal file
View File

@@ -0,0 +1,250 @@
import binascii
import datetime
from typing import List, Optional, Union
import pandas as pd
import pytz
import sys
EPRINT_ENABLED = True
def disable_eprint():
global EPRINT_ENABLED
EPRINT_ENABLED = False
def enable_eprint():
global EPRINT_ENABLED
EPRINT_ENABLED = True
def eprint(*args, **kwargs):
if not EPRINT_ENABLED:
return
print(*args, file=sys.stderr, flush=True, **kwargs)
def to_rippled_epoch(d: datetime.datetime) -> int:
'''Convert from a datetime to the number of seconds since Jan 1, 2000 (rippled epoch)'''
start = datetime.datetime(2000, 1, 1, tzinfo=pytz.utc)
return int((d - start).total_seconds())
class Account: # pylint: disable=too-few-public-methods
'''
Account in the ripple ledger
'''
def __init__(self,
*,
account_id: Optional[str] = None,
nickname: Optional[str] = None,
public_key: Optional[str] = None,
public_key_hex: Optional[str] = None,
secret_key: Optional[str] = None,
result_dict: Optional[dict] = None):
self.account_id = account_id
self.nickname = nickname
self.public_key = public_key
self.public_key_hex = public_key_hex
self.secret_key = secret_key
if result_dict is not None:
self.account_id = result_dict['account_id']
self.public_key = result_dict['public_key']
self.public_key_hex = result_dict['public_key_hex']
self.secret_key = result_dict['master_key']
# Accounts are equal if they represent the same account on the ledger
# I.e. only check the account_id field for equality.
def __eq__(self, lhs):
if not isinstance(lhs, self.__class__):
return False
return self.account_id == lhs.account_id
def __ne__(self, lhs):
return not self.__eq__(lhs)
def __str__(self) -> str:
if self.nickname is not None:
return self.nickname
return self.account_id
def alias_or_account_id(self) -> str:
'''
return the alias if it exists, otherwise return the id
'''
if self.nickname is not None:
return self.nickname
return self.account_id
def account_id_str_as_hex(self) -> str:
return binascii.hexlify(self.account_id.encode()).decode('utf-8')
def to_cmd_obj(self) -> dict:
return {
'account_id': self.account_id,
'nickname': self.nickname,
'public_key': self.public_key,
'public_key_hex': self.public_key_hex,
'secret_key': self.secret_key
}
class Asset:
'''An XRP or IOU value'''
def __init__(
self,
*,
value: Union[int, float, None] = None,
currency: Optional[
str] = None, # Will default to 'XRP' if not specified
issuer: Optional[Account] = None,
from_asset=None, # asset is of type Optional[Asset]
# from_rpc_result is a python object resulting from an rpc command
from_rpc_result: Optional[Union[dict, str]] = None):
assert from_asset is None or from_rpc_result is None
self.value = value
self.issuer = issuer
self.currency = currency
if from_asset is not None:
if self.value is None:
self.value = from_asset.value
if self.issuer is None:
self.issuer = from_asset.issuer
if self.currency is None:
self.currency = from_asset.currency
if from_rpc_result is not None:
if isinstance(from_rpc_result, str):
self.value = int(from_rpc_result)
self.currency = 'XRP'
else:
self.value = from_rpc_result['value']
self.currency = float(from_rpc_result['currency'])
self.issuer = Account(account_id=from_rpc_result['issuer'])
if self.currency is None:
self.currency = 'XRP'
if isinstance(self.value, str):
if self.is_xrp():
self.value = int(value)
else:
self.value = float(value)
def __call__(self, value: Union[int, float]):
'''Call operator useful for a terse syntax for assets in tests. I.e. USD(100)'''
return Asset(value=value, from_asset=self)
def __add__(self, lhs):
assert (self.issuer == lhs.issuer and self.currency == lhs.currency)
return Asset(value=self.value + lhs.value,
currency=self.currency,
issuer=self.issuer)
def __sub__(self, lhs):
assert (self.issuer == lhs.issuer and self.currency == lhs.currency)
return Asset(value=self.value - lhs.value,
currency=self.currency,
issuer=self.issuer)
def __eq__(self, lhs):
if not isinstance(lhs, self.__class__):
return False
return (self.value == lhs.value and self.currency == lhs.currency
and self.issuer == lhs.issuer)
def __ne__(self, lhs):
return not self.__eq__(lhs)
def __str__(self) -> str:
value_part = '' if self.value is None else f'{self.value}/'
issuer_part = '' if self.issuer is None else f'/{self.issuer}'
return f'{value_part}{self.currency}{issuer_part}'
def __repr__(self) -> str:
return self.__str__()
def is_xrp(self) -> bool:
''' return true if the asset represents XRP'''
return self.currency == 'XRP'
def cmd_str(self) -> str:
value_part = '' if self.value is None else f'{self.value}/'
issuer_part = '' if self.issuer is None else f'/{self.issuer.account_id}'
return f'{value_part}{self.currency}{issuer_part}'
def to_cmd_obj(self) -> dict:
'''Return an object suitalbe for use in a command'''
if self.currency == 'XRP':
if self.value is not None:
return f'{self.value}' # must be a string
return {'currency': self.currency}
result = {'currency': self.currency, 'issuer': self.issuer.account_id}
if self.value is not None:
result['value'] = f'{self.value}' # must be a string
return result
def XRP(v: Union[int, float]) -> Asset:
return Asset(value=v * 1_000_000)
class Path:
'''Payment Path'''
def __init__(self,
nodes: Optional[List[Union[Account, Asset]]] = None,
*,
result_list: Optional[List[dict]] = None):
assert not (nodes and result_list)
if result_list is not None:
self.result_list = result_list
return
if nodes is None:
self.result_list = []
return
self.result_list = [
self._create_account_path_node(n)
if isinstance(n, Account) else self._create_currency_path_node(n)
for n in nodes
]
def _create_account_path_node(self, account: Account) -> dict:
return {
'account': account.account_id,
'type': 1,
'type_hex': '0000000000000001'
}
def _create_currency_path_node(self, asset: Asset) -> dict:
result = {
'currency': asset.currency,
'type': 48,
'type_hex': '0000000000000030'
}
if not asset.is_xrp():
result['issuer'] = asset.issuer.account_id
return result
def to_cmd_obj(self) -> list:
'''Return an object suitalbe for use in a command'''
return self.result_list
class PathList:
'''Collection of paths for use in payments'''
def __init__(self,
path_list: Optional[List[Path]] = None,
*,
result_list: Optional[List[List[dict]]] = None):
# result_list can be the response from the rippled server
assert not (path_list and result_list)
if result_list is not None:
self.paths = [Path(result_list=l) for l in result_list]
return
self.paths = path_list
def to_cmd_obj(self) -> list:
'''Return an object suitalbe for use in a command'''
return [p.to_cmd_obj() for p in self.paths]

83
bin/sidechain/python/config_file.py Normal file
View File

@@ -0,0 +1,83 @@
from typing import List, Optional, Tuple
class Section:
def section_header(l: str) -> Optional[str]:
'''
If the line is a section header, return the section name
otherwise return None
'''
if l.startswith('[') and l.endswith(']'):
return l[1:-1]
return None
def __init__(self, name: str):
super().__setattr__('_name', name)
# lines contains all non key-value pairs
super().__setattr__('_lines', [])
super().__setattr__('_kv_pairs', {})
def get_name(self):
return self._name
def add_line(self, l):
s = l.split('=')
if len(s) == 2:
self._kv_pairs[s[0].strip()] = s[1].strip()
else:
self._lines.append(l)
def get_lines(self):
return self._lines
def get_line(self) -> Optional[str]:
if len(self._lines) > 0:
return self._lines[0]
return None
def __getattr__(self, name):
return self._kv_pairs[name]
def __setattr__(self, name, value):
if name in self.__dict__:
super().__setattr__(name, value)
else:
self._kv_pairs[name] = value
class ConfigFile:
def __init__(self, *, file_name: Optional[str] = None):
# parse the file
self._file_name = file_name
self._sections = {}
if not file_name:
return
cur_section = None
with open(file_name) as f:
for n, l in enumerate(f):
l = l.strip()
if l.startswith('#') or not l:
continue
if section_name := Section.section_header(l):
if cur_section:
self.add_section(cur_section)
cur_section = Section(section_name)
continue
if not cur_section:
raise ValueError(
f'Error parsing config file: {file_name} line_num: {n} line: {l}'
)
cur_section.add_line(l)
if cur_section:
self.add_section(cur_section)
def add_section(self, s: Section):
self._sections[s.get_name()] = s
def get_file_name(self):
return self._file_name
def __getattr__(self, name):
return self._sections[name]

630
bin/sidechain/python/create_config_files.py Executable file
View File

@@ -0,0 +1,630 @@
#!/usr/bin/env python3
# Generate rippled config files, each with their own ports, database paths, and validation_seeds.
# There will be configs for shards/no_shards, main/test nets, two config files for each combination
# (so one can run in a dogfood mode while another is tested). To avoid confusion,The directory path
# will be $data_dir/{main | test}.{shard | no_shard}.{dog | test}
# The config file will reside in that directory with the name rippled.cfg
# The validators file will reside in that directory with the name validators.txt
'''
Script to test and debug sidechains.
The rippled exe location can be set through the command line or
the environment variable RIPPLED_MAINCHAIN_EXE
The configs_dir (where the config files will reside) can be set through the command line
or the environment variable RIPPLED_SIDECHAIN_CFG_DIR
'''
import argparse
from dataclasses import dataclass
import json
import os
from pathlib import Path
import sys
from typing import Dict, List, Optional, Tuple, Union
from config_file import ConfigFile
from command import ValidationCreate, WalletPropose
from common import Account, Asset, eprint, XRP
from app import App, single_client_app
mainnet_validators = """
[validator_list_sites]
https://vl.ripple.com
[validator_list_keys]
ED2677ABFFD1B33AC6FBC3062B71F1E8397C1505E1C42C64D11AD1B28FF73F4734
"""
altnet_validators = """
[validator_list_sites]
https://vl.altnet.rippletest.net
[validator_list_keys]
ED264807102805220DA0F312E71FC2C69E1552C9C5790F6C25E3729DEB573D5860
"""
node_size = 'medium'
default_data_dir = '/home/swd/data/rippled'
@dataclass
class Keypair:
public_key: str
secret_key: str
account_id: Optional[str]
def generate_node_keypairs(n: int, rip: App) -> List[Keypair]:
'''
generate keypairs suitable for validator keys
'''
result = []
for i in range(n):
keys = rip(ValidationCreate())
result.append(
Keypair(public_key=keys["validation_public_key"],
secret_key=keys["validation_seed"],
account_id=None))
return result
def generate_federator_keypairs(n: int, rip: App) -> List[Keypair]:
'''
generate keypairs suitable for federator keys
'''
result = []
for i in range(n):
keys = rip(WalletPropose(key_type='ed25519'))
result.append(
Keypair(public_key=keys["public_key"],
secret_key=keys["master_seed"],
account_id=keys["account_id"]))
return result
class Ports:
'''
Port numbers for various services.
Port numbers differ by cfg_index so different configs can run
at the same time without interfering with each other.
'''
peer_port_base = 51235
http_admin_port_base = 5005
ws_public_port_base = 6005
def __init__(self, cfg_index: int):
self.peer_port = Ports.peer_port_base + cfg_index
self.http_admin_port = Ports.http_admin_port_base + cfg_index
self.ws_public_port = Ports.ws_public_port_base + (2 * cfg_index)
# note admin port uses public port base
self.ws_admin_port = Ports.ws_public_port_base + (2 * cfg_index) + 1
class Network:
def __init__(self, num_nodes: int, num_validators: int,
start_cfg_index: int, rip: App):
self.validator_keypairs = generate_node_keypairs(num_validators, rip)
self.ports = [Ports(start_cfg_index + i) for i in range(num_nodes)]
class SidechainNetwork(Network):
def __init__(self, num_nodes: int, num_federators: int,
num_validators: int, start_cfg_index: int, rip: App):
super().__init__(num_nodes, num_validators, start_cfg_index, rip)
self.federator_keypairs = generate_federator_keypairs(
num_federators, rip)
self.main_account = rip(WalletPropose(key_type='secp256k1'))
class XChainAsset:
def __init__(self, main_asset: Asset, side_asset: Asset,
main_value: Union[int, float], side_value: Union[int, float],
main_refund_penalty: Union[int, float],
side_refund_penalty: Union[int, float]):
self.main_asset = main_asset(main_value)
self.side_asset = side_asset(side_value)
self.main_refund_penalty = main_asset(main_refund_penalty)
self.side_refund_penalty = side_asset(side_refund_penalty)
def generate_asset_stanzas(
assets: Optional[Dict[str, XChainAsset]] = None) -> str:
if assets is None:
# default to xrp only at a 1:1 value
assets = {}
assets['xrp_xrp_sidechain_asset'] = XChainAsset(
XRP(0), XRP(0), 1, 1, 400, 400)
index_stanza = """
[sidechain_assets]"""
asset_stanzas = []
for name, xchainasset in assets.items():
index_stanza += '\n' + name
new_stanza = f"""
[{name}]
mainchain_asset={json.dumps(xchainasset.main_asset.to_cmd_obj())}
sidechain_asset={json.dumps(xchainasset.side_asset.to_cmd_obj())}
mainchain_refund_penalty={json.dumps(xchainasset.main_refund_penalty.to_cmd_obj())}
sidechain_refund_penalty={json.dumps(xchainasset.side_refund_penalty.to_cmd_obj())}"""
asset_stanzas.append(new_stanza)
return index_stanza + '\n' + '\n'.join(asset_stanzas)
# First element of the returned tuple is the sidechain stanzas
# second element is the bootstrap stanzas
def generate_sidechain_stanza(
mainchain_ports: Ports,
main_account: dict,
federators: List[Keypair],
signing_key: str,
mainchain_cfg_file: str,
xchain_assets: Optional[Dict[str,
XChainAsset]] = None) -> Tuple[str, str]:
mainchain_ip = "127.0.0.1"
federators_stanza = """
# federator signing public keys
[sidechain_federators]
"""
federators_secrets_stanza = """
# federator signing secret keys (for standalone-mode testing only; Normally won't be in a config file)
[sidechain_federators_secrets]
"""
bootstrap_federators_stanza = """
# first value is federator signing public key, second is the signing pk account
[sidechain_federators]
"""
assets_stanzas = generate_asset_stanzas(xchain_assets)
for fed in federators:
federators_stanza += f'{fed.public_key}\n'
federators_secrets_stanza += f'{fed.secret_key}\n'
bootstrap_federators_stanza += f'{fed.public_key} {fed.account_id}\n'
sidechain_stanzas = f"""
[sidechain]
signing_key={signing_key}
mainchain_account={main_account["account_id"]}
mainchain_ip={mainchain_ip}
mainchain_port_ws={mainchain_ports.ws_public_port}
# mainchain config file is: {mainchain_cfg_file}
{assets_stanzas}
{federators_stanza}
{federators_secrets_stanza}
"""
bootstrap_stanzas = f"""
[sidechain]
mainchain_secret={main_account["master_seed"]}
{bootstrap_federators_stanza}
"""
return (sidechain_stanzas, bootstrap_stanzas)
# cfg_type will typically be either 'dog' or 'test', but can be any string. It is only used
# to create the data directories.
def generate_cfg_dir(*,
ports: Ports,
with_shards: bool,
main_net: bool,
cfg_type: str,
sidechain_stanza: str,
sidechain_bootstrap_stanza: str,
validation_seed: Optional[str] = None,
validators: Optional[List[str]] = None,
fixed_ips: Optional[List[Ports]] = None,
data_dir: str,
full_history: bool = False,
with_hooks: bool = False) -> str:
ips_stanza = ''
this_ip = '127.0.0.1'
if fixed_ips:
ips_stanza = '# Fixed ips for a testnet.\n'
ips_stanza += '[ips_fixed]\n'
for i, p in enumerate(fixed_ips):
if p.peer_port == ports.peer_port:
continue
# rippled limits the number of connects per ip. So use the other loopback devices
ips_stanza += f'127.0.0.{i+1} {p.peer_port}\n'
else:
ips_stanza = '# Where to find some other servers speaking the Ripple protocol.\n'
ips_stanza += '[ips]\n'
if main_net:
ips_stanza += 'r.ripple.com 51235\n'
else:
ips_stanza += 'r.altnet.rippletest.net 51235\n'
disable_shards = '' if with_shards else '# '
disable_delete = '#' if full_history else ''
history_line = 'full' if full_history else '256'
earliest_seq_line = ''
if sidechain_stanza:
earliest_seq_line = 'earliest_seq=1'
hooks_line = 'Hooks' if with_hooks else ''
validation_seed_stanza = ''
if validation_seed:
validation_seed_stanza = f'''
[validation_seed]
{validation_seed}
'''
node_size = 'medium'
shard_str = 'shards' if with_shards else 'no_shards'
net_str = 'main' if main_net else 'test'
if not fixed_ips:
sub_dir = data_dir + f'/{net_str}.{shard_str}.{cfg_type}'
if sidechain_stanza:
sub_dir += '.sidechain'
else:
sub_dir = data_dir + f'/{cfg_type}'
db_path = sub_dir + '/db'
debug_logfile = sub_dir + '/debug.log'
shard_db_path = sub_dir + '/shards'
node_db_path = db_path + '/nudb'
cfg_str = f"""
[server]
port_rpc_admin_local
port_peer
port_ws_admin_local
port_ws_public
#ssl_key = /etc/ssl/private/server.key
#ssl_cert = /etc/ssl/certs/server.crt
[port_rpc_admin_local]
port = {ports.http_admin_port}
ip = {this_ip}
admin = {this_ip}
protocol = http
[port_peer]
port = {ports.peer_port}
ip = 0.0.0.0
protocol = peer
[port_ws_admin_local]
port = {ports.ws_admin_port}
ip = {this_ip}
admin = {this_ip}
protocol = ws
[port_ws_public]
port = {ports.ws_public_port}
ip = {this_ip}
protocol = ws
# protocol = wss
[node_size]
{node_size}
[ledger_history]
{history_line}
[node_db]
type=NuDB
path={node_db_path}
open_files=2000
filter_bits=12
cache_mb=256
file_size_mb=8
file_size_mult=2
{earliest_seq_line}
{disable_delete}online_delete=256
{disable_delete}advisory_delete=0
[database_path]
{db_path}
# This needs to be an absolute directory reference, not a relative one.
# Modify this value as required.
[debug_logfile]
{debug_logfile}
[sntp_servers]
time.windows.com
time.apple.com
time.nist.gov
pool.ntp.org
{ips_stanza}
[validators_file]
validators.txt
[rpc_startup]
{{ "command": "log_level", "severity": "fatal" }}
{{ "command": "log_level", "partition": "SidechainFederator", "severity": "trace" }}
[ssl_verify]
1
{validation_seed_stanza}
{disable_shards}[shard_db]
{disable_shards}type=NuDB
{disable_shards}path={shard_db_path}
{disable_shards}max_historical_shards=6
{sidechain_stanza}
[features]
{hooks_line}
PayChan
Flow
FlowCross
TickSize
fix1368
Escrow
fix1373
EnforceInvariants
SortedDirectories
fix1201
fix1512
fix1513
fix1523
fix1528
DepositAuth
Checks
fix1571
fix1543
fix1623
DepositPreauth
fix1515
fix1578
MultiSignReserve
fixTakerDryOfferRemoval
fixMasterKeyAsRegularKey
fixCheckThreading
fixPayChanRecipientOwnerDir
DeletableAccounts
fixQualityUpperBound
RequireFullyCanonicalSig
fix1781
HardenedValidations
fixAmendmentMajorityCalc
NegativeUNL
TicketBatch
FlowSortStrands
fixSTAmountCanonicalize
fixRmSmallIncreasedQOffers
CheckCashMakesTrustLine
"""
validators_str = ''
for p in [sub_dir, db_path, shard_db_path]:
Path(p).mkdir(parents=True, exist_ok=True)
# Add the validators.txt file
if validators:
validators_str = '[validators]\n'
for k in validators:
validators_str += f'{k}\n'
else:
validators_str = mainnet_validators if main_net else altnet_validators
with open(sub_dir + "/validators.txt", "w") as f:
f.write(validators_str)
# add the rippled.cfg file
with open(sub_dir + "/rippled.cfg", "w") as f:
f.write(cfg_str)
if sidechain_bootstrap_stanza:
# add the bootstrap file
with open(sub_dir + "/sidechain_bootstrap.cfg", "w") as f:
f.write(sidechain_bootstrap_stanza)
return sub_dir + "/rippled.cfg"
def generate_multinode_net(out_dir: str,
mainnet: Network,
sidenet: SidechainNetwork,
xchain_assets: Optional[Dict[str,
XChainAsset]] = None):
mainnet_cfgs = []
for i in range(len(mainnet.ports)):
validator_kp = mainnet.validator_keypairs[i]
ports = mainnet.ports[i]
mainchain_cfg_file = generate_cfg_dir(
ports=ports,
with_shards=False,
main_net=True,
cfg_type=f'mainchain_{i}',
sidechain_stanza='',
sidechain_bootstrap_stanza='',
validation_seed=validator_kp.secret_key,
data_dir=out_dir)
mainnet_cfgs.append(mainchain_cfg_file)
for i in range(len(sidenet.ports)):
validator_kp = sidenet.validator_keypairs[i]
ports = sidenet.ports[i]
mainnet_i = i % len(mainnet.ports)
sidechain_stanza, sidechain_bootstrap_stanza = generate_sidechain_stanza(
mainnet.ports[mainnet_i], sidenet.main_account,
sidenet.federator_keypairs,
sidenet.federator_keypairs[i].secret_key, mainnet_cfgs[mainnet_i],
xchain_assets)
generate_cfg_dir(
ports=ports,
with_shards=False,
main_net=True,
cfg_type=f'sidechain_{i}',
sidechain_stanza=sidechain_stanza,
sidechain_bootstrap_stanza=sidechain_bootstrap_stanza,
validation_seed=validator_kp.secret_key,
validators=[kp.public_key for kp in sidenet.validator_keypairs],
fixed_ips=sidenet.ports,
data_dir=out_dir,
full_history=True,
with_hooks=False)
def parse_args():
parser = argparse.ArgumentParser(
description=('Create config files for testing sidechains'))
parser.add_argument(
'--exe',
'-e',
help=('path to rippled executable'),
)
parser.add_argument(
'--usd',
'-u',
action='store_true',
help=('include a USD/root IOU asset for cross chain transfers'),
)
parser.add_argument(
'--cfgs_dir',
'-c',
help=
('path to configuration file dir (where the output config files will be located)'
),
)
return parser.parse_known_args()[0]
class Params:
def __init__(self):
args = parse_args()
self.exe = None
if 'RIPPLED_MAINCHAIN_EXE' in os.environ:
self.exe = os.environ['RIPPLED_MAINCHAIN_EXE']
if args.exe:
self.exe = args.exe
self.configs_dir = None
if 'RIPPLED_SIDECHAIN_CFG_DIR' in os.environ:
self.configs_dir = os.environ['RIPPLED_SIDECHAIN_CFG_DIR']
if args.cfgs_dir:
self.configs_dir = args.cfgs_dir
self.usd = False
if args.usd:
self.usd = args.usd
def check_error(self) -> str:
'''
Check for errors. Return `None` if no errors,
otherwise return a string describing the error
'''
if not self.exe:
return 'Missing exe location. Either set the env variable RIPPLED_MAINCHAIN_EXE or use the --exe_mainchain command line switch'
if not self.configs_dir:
return 'Missing configs directory location. Either set the env variable RIPPLED_SIDECHAIN_CFG_DIR or use the --cfgs_dir command line switch'
def main(params: Params,
xchain_assets: Optional[Dict[str, XChainAsset]] = None):
if err_str := params.check_error():
eprint(err_str)
sys.exit(1)
index = 0
nonvalidator_cfg_file_name = generate_cfg_dir(
ports=Ports(index),
with_shards=False,
main_net=True,
cfg_type='non_validator',
sidechain_stanza='',
sidechain_bootstrap_stanza='',
validation_seed=None,
data_dir=params.configs_dir)
index = index + 1
nonvalidator_config = ConfigFile(file_name=nonvalidator_cfg_file_name)
with single_client_app(exe=params.exe,
config=nonvalidator_config,
standalone=True) as rip:
mainnet = Network(num_nodes=1,
num_validators=1,
start_cfg_index=index,
rip=rip)
sidenet = SidechainNetwork(num_nodes=5,
num_federators=5,
num_validators=5,
start_cfg_index=index + 1,
rip=rip)
generate_multinode_net(
out_dir=f'{params.configs_dir}/sidechain_testnet',
mainnet=mainnet,
sidenet=sidenet,
xchain_assets=xchain_assets)
index = index + 2
(Path(params.configs_dir) / 'logs').mkdir(parents=True, exist_ok=True)
for with_shards in [True, False]:
for is_main_net in [True, False]:
for cfg_type in ['dog', 'test', 'one', 'two']:
if not is_main_net and cfg_type not in ['dog', 'test']:
continue
mainnet = Network(num_nodes=1,
num_validators=1,
start_cfg_index=index,
rip=rip)
mainchain_cfg_file = generate_cfg_dir(
data_dir=params.configs_dir,
ports=mainnet.ports[0],
with_shards=with_shards,
main_net=is_main_net,
cfg_type=cfg_type,
sidechain_stanza='',
sidechain_bootstrap_stanza='',
validation_seed=mainnet.validator_keypairs[0].
secret_key)
sidenet = SidechainNetwork(num_nodes=1,
num_federators=5,
num_validators=1,
start_cfg_index=index + 1,
rip=rip)
signing_key = sidenet.federator_keypairs[0].secret_key
sidechain_stanza, sizechain_bootstrap_stanza = generate_sidechain_stanza(
mainnet.ports[0], sidenet.main_account,
sidenet.federator_keypairs, signing_key,
mainchain_cfg_file, xchain_assets)
generate_cfg_dir(
data_dir=params.configs_dir,
ports=sidenet.ports[0],
with_shards=with_shards,
main_net=is_main_net,
cfg_type=cfg_type,
sidechain_stanza=sidechain_stanza,
sidechain_bootstrap_stanza=sizechain_bootstrap_stanza,
validation_seed=sidenet.validator_keypairs[0].
secret_key)
index = index + 2
if __name__ == '__main__':
params = Params()
xchain_assets = None
if params.usd:
xchain_assets = {}
xchain_assets['xrp_xrp_sidechain_asset'] = XChainAsset(
XRP(0), XRP(0), 1, 1, 200, 200)
root_account = Account(account_id="rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh")
main_iou_asset = Asset(value=0, currency='USD', issuer=root_account)
side_iou_asset = Asset(value=0, currency='USD', issuer=root_account)
xchain_assets['iou_iou_sidechain_asset'] = XChainAsset(
main_iou_asset, side_iou_asset, 1, 1, 0.02, 0.02)
main(params, xchain_assets)

1496
bin/sidechain/python/interactive.py Normal file
View File

File diff suppressed because it is too large Load Diff

143
bin/sidechain/python/log_analyzer.py Executable file
View File

@@ -0,0 +1,143 @@
#!/usr/bin/env python3
import argparse
import json
import re
from common import eprint
class LogLine:
UNSTRUCTURED_RE = re.compile(r'''(?x)
# The x flag enables insignificant whitespace mode (allowing comments)
^(?P<timestamp>.*UTC)
[\ ]
(?P<module>[^:]*):(?P<level>[^\ ]*)
[\ ]
(?P<msg>.*$)
''')
STRUCTURED_RE = re.compile(r'''(?x)
# The x flag enables insignificant whitespace mode (allowing comments)
^(?P<timestamp>.*UTC)
[\ ]
(?P<module>[^:]*):(?P<level>[^\ ]*)
[\ ]
(?P<msg>[^{]*)
[\ ]
(?P<json_data>.*$)
''')
def __init__(self, line: str):
self.raw_line = line
self.json_data = None
try:
if line.endswith('}'):
m = self.STRUCTURED_RE.match(line)
try:
self.json_data = json.loads(m.group('json_data'))
except:
m = self.UNSTRUCTURED_RE.match(line)
else:
m = self.UNSTRUCTURED_RE.match(line)
self.timestamp = m.group('timestamp')
self.level = m.group('level')
self.module = m.group('module')
self.msg = m.group('msg')
except Exception as e:
eprint(f'init exception: {e} line: {line}')
def to_mixed_json(self) -> str:
'''
return a pretty printed string as mixed json
'''
try:
r = f'{self.timestamp} {self.module}:{self.level} {self.msg}'
if self.json_data:
r += '\n' + json.dumps(self.json_data, indent=1)
return r
except:
eprint(f'Using raw line: {self.raw_line}')
return self.raw_line
def to_pure_json(self) -> str:
'''
return a pretty printed string as pure json
'''
try:
dict = {}
dict['t'] = self.timestamp
dict['m'] = self.module
dict['l'] = self.level
dict['msg'] = self.msg
if self.json_data:
dict['data'] = self.json_data
return json.dumps(dict, indent=1)
except:
return self.raw_line
def convert_log(in_file_name: str, out_file_name: str, *, pure_json=False):
try:
prev_lines = None
with open(in_file_name) as input:
with open(out_file_name, "w") as out:
for l in input:
l = l.strip()
if not l:
continue
if LogLine.UNSTRUCTURED_RE.match(l):
if prev_lines:
log_line = LogLine(prev_lines)
if log_line.module == 'SidechainFederator':
if pure_json:
print(log_line.to_pure_json(), file=out)
else:
print(log_line.to_mixed_json(), file=out)
prev_lines = l
else:
if not prev_lines:
eprint(
f'Error: Expected prev_lines. Cur line: {l}')
assert prev_lines
prev_lines += f' {l}'
if prev_lines:
log_line = LogLine(prev_lines)
if log_line.module == 'SidechainFederator':
if pure_json:
print(log_line.to_pure_json(),
file=out,
flush=True)
else:
print(log_line.to_mixed_json(),
file=out,
flush=True)
except Exception as e:
eprint(f'Excption: {e}')
raise e
def parse_args():
parser = argparse.ArgumentParser(
description=('python script to convert log files to json'))
parser.add_argument(
'--input',
'-i',
help=('input log file'),
)
parser.add_argument(
'--output',
'-o',
help=('output log file'),
)
return parser.parse_known_args()[0]
if __name__ == '__main__':
args = parse_args()
convert_log(args.input, args.output, pure_json=True)

33
bin/sidechain/python/riplrepl.py Executable file
View File

@@ -0,0 +1,33 @@
#!/usr/bin/env python3
'''
Script to run an interactive shell to test sidechains.
'''
from common import disable_eprint
import interactive
import sidechain
def main():
params = sidechain.Params()
params.interactive = True
interactive.set_hooks_dir(params.hooks_dir)
if err_str := params.check_error():
eprint(err_str)
sys.exit(1)
if params.verbose:
print("eprint enabled")
else:
disable_eprint()
if params.standalone:
sidechain.standalone_interactive_repl(params)
else:
sidechain.multinode_interactive_repl(params)
if __name__ == '__main__':
main()

193
bin/sidechain/python/ripple_client.py Normal file
View File

@@ -0,0 +1,193 @@
import asyncio
import datetime
import json
import os
from os.path import expanduser
import subprocess
import sys
from typing import Callable, List, Optional, Union
import time
import websockets
from command import Command, ServerInfo, SubscriptionCommand
from common import eprint
from config_file import ConfigFile
class RippleClient:
'''Client to send commands to the rippled server'''
def __init__(self,
*,
config: ConfigFile,
exe: str,
command_log: Optional[str] = None):
self.config = config
self.exe = exe
self.command_log = command_log
section = config.port_ws_admin_local
self.websocket_uri = f'{section.protocol}://{section.ip}:{section.port}'
self.subscription_websockets = []
self.tasks = []
self.pid = None
if command_log:
with open(self.command_log, 'w') as f:
f.write(f'# Start \n')
@property
def config_file_name(self):
return self.config.get_file_name()
def shutdown(self):
try:
group = asyncio.gather(*self.tasks, return_exceptions=True)
group.cancel()
asyncio.get_event_loop().run_until_complete(group)
for ws in self.subscription_websockets:
asyncio.get_event_loop().run_until_complete(ws.close())
except asyncio.CancelledError:
pass
def set_pid(self, pid: int):
self.pid = pid
def get_pid(self) -> Optional[int]:
return self.pid
def get_config(self) -> ConfigFile:
return self.config
# Get a dict of the server_state, validated_ledger_seq, and complete_ledgers
def get_brief_server_info(self) -> dict:
ret = {
'server_state': 'NA',
'ledger_seq': 'NA',
'complete_ledgers': 'NA'
}
if not self.pid or self.pid == -1:
return ret
r = self.send_command(ServerInfo())
if 'info' not in r:
return ret
r = r['info']
for f in ['server_state', 'complete_ledgers']:
if f in r:
ret[f] = r[f]
if 'validated_ledger' in r:
ret['ledger_seq'] = r['validated_ledger']['seq']
return ret
def _write_command_log_command(self, cmd: str, cmd_index: int) -> None:
if not self.command_log:
return
with open(self.command_log, 'a') as f:
f.write(f'\n\n# command {cmd_index}\n')
f.write(f'{cmd}')
def _write_command_log_result(self, result: str, cmd_index: int) -> None:
if not self.command_log:
return
with open(self.command_log, 'a') as f:
f.write(f'\n\n# result {cmd_index}\n')
f.write(f'{result}')
def _send_command_line_command(self, cmd_id: int, *args) -> dict:
'''Send the command to the rippled server using the command line interface'''
to_run = [self.exe, '-q', '--conf', self.config_file_name, '--']
to_run.extend(args)
self._write_command_log_command(to_run, cmd_id)
max_retries = 4
for retry_count in range(0, max_retries + 1):
try:
r = subprocess.check_output(to_run)
self._write_command_log_result(r, cmd_id)
return json.loads(r.decode('utf-8'))['result']
except Exception as e:
if retry_count == max_retries:
raise
eprint(
f'Got exception: {str(e)}\nretrying..{retry_count+1} of {max_retries}'
)
time.sleep(1) # give process time to startup
async def _send_websock_command(
self,
cmd: Command,
conn: Optional[websockets.client.Connect] = None) -> dict:
assert self.websocket_uri
if conn is None:
async with websockets.connect(self.websocket_uri) as ws:
return await self._send_websock_command(cmd, ws)
to_send = json.dumps(cmd.get_websocket_dict())
self._write_command_log_command(to_send, cmd.cmd_id)
await conn.send(to_send)
r = await conn.recv()
self._write_command_log_result(r, cmd.cmd_id)
j = json.loads(r)
if not 'result' in j:
eprint(
f'Error sending websocket command: {json.dumps(cmd.get_websocket_dict(), indent=1)}'
)
eprint(f'Result: {json.dumps(j, indent=1)}')
raise ValueError('Error sending websocket command')
return j['result']
def send_command(self, cmd: Command) -> dict:
'''Send the command to the rippled server'''
if self.websocket_uri:
return asyncio.get_event_loop().run_until_complete(
self._send_websock_command(cmd))
return self._send_command_line_command(cmd.cmd_id,
*cmd.get_command_line_list())
# Need async version to close ledgers from async functions
async def async_send_command(self, cmd: Command) -> dict:
'''Send the command to the rippled server'''
if self.websocket_uri:
return await self._send_websock_command(cmd)
return self._send_command_line_command(cmd.cmd_id,
*cmd.get_command_line_list())
def send_subscribe_command(
self,
cmd: SubscriptionCommand,
callback: Optional[Callable[[dict], None]] = None) -> dict:
'''Send the command to the rippled server'''
assert self.websocket_uri
ws = cmd.websocket
if ws is None:
# subscribe
assert callback
ws = asyncio.get_event_loop().run_until_complete(
websockets.connect(self.websocket_uri))
self.subscription_websockets.append(ws)
result = asyncio.get_event_loop().run_until_complete(
self._send_websock_command(cmd, ws))
if cmd.websocket is not None:
# unsubscribed. close the websocket
self.subscription_websockets.remove(cmd.websocket)
cmd.websocket.close()
cmd.websocket = None
else:
# setup a task to read the websocket
cmd.websocket = ws # must be set after the _send_websock_command or will unsubscribe
async def subscribe_callback(ws: websockets.client.Connect,
cb: Callable[[dict], None]):
while True:
r = await ws.recv()
d = json.loads(r)
cb(d)
task = asyncio.get_event_loop().create_task(
subscribe_callback(cmd.websocket, callback))
self.tasks.append(task)
return result
def stop(self):
'''Stop the server'''
return self.send_command(Stop())
def set_log_level(self, severity: str, *, partition: Optional[str] = None):
'''Set the server log level'''
return self.send_command(LogLevel(severity, parition=parition))

576
bin/sidechain/python/sidechain.py Executable file
View File

@@ -0,0 +1,576 @@
#!/usr/bin/env python3
'''
Script to test and debug sidechains.
The mainchain exe location can be set through the command line or
the environment variable RIPPLED_MAINCHAIN_EXE
The sidechain exe location can be set through the command line or
the environment variable RIPPLED_SIDECHAIN_EXE
The configs_dir (generated with create_config_files.py) can be set through the command line
or the environment variable RIPPLED_SIDECHAIN_CFG_DIR
'''
import argparse
import json
from multiprocessing import Process, Value
import os
import sys
import time
from typing import Callable, Dict, List, Optional
from app import App, single_client_app, testnet_app, configs_for_testnet
from command import AccountInfo, AccountTx, LedgerAccept, LogLevel, Subscribe
from common import Account, Asset, eprint, disable_eprint, XRP
from config_file import ConfigFile
import interactive
from log_analyzer import convert_log
from test_utils import mc_wait_for_payment_detect, sc_wait_for_payment_detect, mc_connect_subscription, sc_connect_subscription
from transaction import AccountSet, Payment, SignerListSet, SetRegularKey, Ticket, Trust
def parse_args_helper(parser: argparse.ArgumentParser):
parser.add_argument(
'--debug_sidechain',
'-ds',
action='store_true',
help=('Mode to debug sidechain (prompt to run sidechain in gdb)'),
)
parser.add_argument(
'--debug_mainchain',
'-dm',
action='store_true',
help=('Mode to debug mainchain (prompt to run sidechain in gdb)'),
)
parser.add_argument(
'--exe_mainchain',
'-em',
help=('path to mainchain rippled executable'),
)
parser.add_argument(
'--exe_sidechain',
'-es',
help=('path to mainchain rippled executable'),
)
parser.add_argument(
'--cfgs_dir',
'-c',
help=
('path to configuration file dir (generated with create_config_files.py)'
),
)
parser.add_argument(
'--standalone',
'-a',
action='store_true',
help=('run standalone tests'),
)
parser.add_argument(
'--interactive',
'-i',
action='store_true',
help=('run interactive repl'),
)
parser.add_argument(
'--quiet',
'-q',
action='store_true',
help=('Disable printing errors (eprint disabled)'),
)
parser.add_argument(
'--verbose',
'-v',
action='store_true',
help=('Enable printing errors (eprint enabled)'),
)
# Pauses are use for attaching debuggers and looking at logs are know checkpoints
parser.add_argument(
'--with_pauses',
'-p',
action='store_true',
help=
('Add pauses at certain checkpoints in tests until "enter" key is hit'
),
)
parser.add_argument(
'--hooks_dir',
help=('path to hooks dir'),
)
def parse_args():
parser = argparse.ArgumentParser(description=('Test and debug sidechains'))
parse_args_helper(parser)
return parser.parse_known_args()[0]
class Params:
def __init__(self, *, configs_dir: Optional[str] = None):
args = parse_args()
self.debug_sidechain = False
if args.debug_sidechain:
self.debug_sidechain = args.debug_sidechain
self.debug_mainchain = False
if args.debug_mainchain:
self.debug_mainchain = arts.debug_mainchain
self.standalone = args.standalone
self.with_pauses = args.with_pauses
self.interactive = args.interactive
self.quiet = args.quiet
self.verbose = args.verbose
self.mainchain_exe = None
if 'RIPPLED_MAINCHAIN_EXE' in os.environ:
self.mainchain_exe = os.environ['RIPPLED_MAINCHAIN_EXE']
if args.exe_mainchain:
self.mainchain_exe = args.exe_mainchain
self.sidechain_exe = None
if 'RIPPLED_SIDECHAIN_EXE' in os.environ:
self.sidechain_exe = os.environ['RIPPLED_SIDECHAIN_EXE']
if args.exe_sidechain:
self.sidechain_exe = args.exe_sidechain
self.configs_dir = None
if 'RIPPLED_SIDECHAIN_CFG_DIR' in os.environ:
self.configs_dir = os.environ['RIPPLED_SIDECHAIN_CFG_DIR']
if args.cfgs_dir:
self.configs_dir = args.cfgs_dir
if configs_dir is not None:
self.configs_dir = configs_dir
self.hooks_dir = None
if 'RIPPLED_SIDECHAIN_HOOKS_DIR' in os.environ:
self.hooks_dir = os.environ['RIPPLED_SIDECHAIN_HOOKS_DIR']
if args.hooks_dir:
self.hooks_dir = args.hooks_dir
if not self.configs_dir:
self.mainchain_config = None
self.sidechain_config = None
self.sidechain_bootstrap_config = None
self.genesis_account = None
self.mc_door_account = None
self.user_account = None
self.sc_door_account = None
self.federators = None
return
if self.standalone:
self.mainchain_config = ConfigFile(
file_name=f'{self.configs_dir}/main.no_shards.dog/rippled.cfg')
self.sidechain_config = ConfigFile(
file_name=
f'{self.configs_dir}/main.no_shards.dog.sidechain/rippled.cfg')
self.sidechain_bootstrap_config = ConfigFile(
file_name=
f'{self.configs_dir}/main.no_shards.dog.sidechain/sidechain_bootstrap.cfg'
)
else:
self.mainchain_config = ConfigFile(
file_name=
f'{self.configs_dir}/sidechain_testnet/main.no_shards.mainchain_0/rippled.cfg'
)
self.sidechain_config = ConfigFile(
file_name=
f'{self.configs_dir}/sidechain_testnet/sidechain_0/rippled.cfg'
)
self.sidechain_bootstrap_config = ConfigFile(
file_name=
f'{self.configs_dir}/sidechain_testnet/sidechain_0/sidechain_bootstrap.cfg'
)
self.genesis_account = Account(
account_id='rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh',
secret_key='masterpassphrase',
nickname='genesis')
self.mc_door_account = Account(
account_id=self.sidechain_config.sidechain.mainchain_account,
secret_key=self.sidechain_bootstrap_config.sidechain.
mainchain_secret,
nickname='door')
self.user_account = Account(
account_id='rJynXY96Vuq6B58pST9K5Ak5KgJ2JcRsQy',
secret_key='snVsJfrr2MbVpniNiUU6EDMGBbtzN',
nickname='alice')
self.sc_door_account = Account(
account_id='rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh',
secret_key='masterpassphrase',
nickname='door')
self.federators = [
l.split()[1].strip() for l in
self.sidechain_bootstrap_config.sidechain_federators.get_lines()
]
def check_error(self) -> str:
'''
Check for errors. Return `None` if no errors,
otherwise return a string describing the error
'''
if not self.mainchain_exe:
return 'Missing mainchain_exe location. Either set the env variable RIPPLED_MAINCHAIN_EXE or use the --exe_mainchain command line switch'
if not self.sidechain_exe:
return 'Missing sidechain_exe location. Either set the env variable RIPPLED_SIDECHAIN_EXE or use the --exe_sidechain command line switch'
if not self.configs_dir:
return 'Missing configs directory location. Either set the env variable RIPPLED_SIDECHAIN_CFG_DIR or use the --cfgs_dir command line switch'
if self.verbose and self.quiet:
return 'Cannot specify both verbose and quiet options at the same time'
mainDoorKeeper = 0
sideDoorKeeper = 1
updateSignerList = 2
def setup_mainchain(mc_app: App,
params: Params,
setup_user_accounts: bool = True):
mc_app.add_to_keymanager(params.mc_door_account)
if setup_user_accounts:
mc_app.add_to_keymanager(params.user_account)
mc_app(LogLevel('fatal'))
# Allow rippling through the genesis account
mc_app(AccountSet(account=params.genesis_account).set_default_ripple(True))
mc_app.maybe_ledger_accept()
# Create and fund the mc door account
mc_app(
Payment(account=params.genesis_account,
dst=params.mc_door_account,
amt=XRP(10_000)))
mc_app.maybe_ledger_accept()
# Create a trust line so USD/root account ious can be sent cross chain
mc_app(
Trust(account=params.mc_door_account,
limit_amt=Asset(value=1_000_000,
currency='USD',
issuer=params.genesis_account)))
# set the chain's signer list and disable the master key
divide = 4 * len(params.federators)
by = 5
quorum = (divide + by - 1) // by
mc_app(
SignerListSet(account=params.mc_door_account,
quorum=quorum,
keys=params.federators))
mc_app.maybe_ledger_accept()
r = mc_app(Ticket(account=params.mc_door_account, src_tag=mainDoorKeeper))
mc_app.maybe_ledger_accept()
mc_app(Ticket(account=params.mc_door_account, src_tag=sideDoorKeeper))
mc_app.maybe_ledger_accept()
mc_app(Ticket(account=params.mc_door_account, src_tag=updateSignerList))
mc_app.maybe_ledger_accept()
mc_app(AccountSet(account=params.mc_door_account).set_disable_master())
mc_app.maybe_ledger_accept()
if setup_user_accounts:
# Create and fund a regular user account
mc_app(
Payment(account=params.genesis_account,
dst=params.user_account,
amt=XRP(2_000)))
mc_app.maybe_ledger_accept()
def setup_sidechain(sc_app: App,
params: Params,
setup_user_accounts: bool = True):
sc_app.add_to_keymanager(params.sc_door_account)
if setup_user_accounts:
sc_app.add_to_keymanager(params.user_account)
sc_app(LogLevel('fatal'))
sc_app(LogLevel('trace', partition='SidechainFederator'))
# set the chain's signer list and disable the master key
divide = 4 * len(params.federators)
by = 5
quorum = (divide + by - 1) // by
sc_app(
SignerListSet(account=params.genesis_account,
quorum=quorum,
keys=params.federators))
sc_app.maybe_ledger_accept()
sc_app(Ticket(account=params.genesis_account, src_tag=mainDoorKeeper))
sc_app.maybe_ledger_accept()
sc_app(Ticket(account=params.genesis_account, src_tag=sideDoorKeeper))
sc_app.maybe_ledger_accept()
sc_app(Ticket(account=params.genesis_account, src_tag=updateSignerList))
sc_app.maybe_ledger_accept()
sc_app(AccountSet(account=params.genesis_account).set_disable_master())
sc_app.maybe_ledger_accept()
def _xchain_transfer(from_chain: App, to_chain: App, src: Account,
dst: Account, amt: Asset, from_chain_door: Account,
to_chain_door: Account):
memos = [{'Memo': {'MemoData': dst.account_id_str_as_hex()}}]
from_chain(Payment(account=src, dst=from_chain_door, amt=amt, memos=memos))
from_chain.maybe_ledger_accept()
if to_chain.standalone:
# from_chain (side chain) sends a txn, but won't close the to_chain (main chain) ledger
time.sleep(1)
to_chain.maybe_ledger_accept()
def main_to_side_transfer(mc_app: App, sc_app: App, src: Account, dst: Account,
amt: Asset, params: Params):
_xchain_transfer(mc_app, sc_app, src, dst, amt, params.mc_door_account,
params.sc_door_account)
def side_to_main_transfer(mc_app: App, sc_app: App, src: Account, dst: Account,
amt: Asset, params: Params):
_xchain_transfer(sc_app, mc_app, src, dst, amt, params.sc_door_account,
params.mc_door_account)
def simple_test(mc_app: App, sc_app: App, params: Params):
try:
bob = sc_app.create_account('bob')
main_to_side_transfer(mc_app, sc_app, params.user_account, bob,
XRP(200), params)
main_to_side_transfer(mc_app, sc_app, params.user_account, bob,
XRP(60), params)
if params.with_pauses:
_convert_log_files_to_json(
mc_app.get_configs() + sc_app.get_configs(),
'checkpoint1.json')
input(
"Pausing to check for main -> side txns (press enter to continue)"
)
side_to_main_transfer(mc_app, sc_app, bob, params.user_account, XRP(9),
params)
side_to_main_transfer(mc_app, sc_app, bob, params.user_account,
XRP(11), params)
if params.with_pauses:
input(
"Pausing to check for side -> main txns (press enter to continue)"
)
finally:
_convert_log_files_to_json(mc_app.get_configs() + sc_app.get_configs(),
'final.json')
def _rm_debug_log(config: ConfigFile):
try:
debug_log = config.debug_logfile.get_line()
if debug_log:
print(f'removing debug file: {debug_log}', flush=True)
os.remove(debug_log)
except:
pass
def _standalone_with_callback(params: Params,
callback: Callable[[App, App], None],
setup_user_accounts: bool = True):
if (params.debug_mainchain):
input("Start mainchain server and press enter to continue: ")
else:
_rm_debug_log(params.mainchain_config)
with single_client_app(config=params.mainchain_config,
exe=params.mainchain_exe,
standalone=True,
run_server=not params.debug_mainchain) as mc_app:
mc_connect_subscription(mc_app, params.mc_door_account)
setup_mainchain(mc_app, params, setup_user_accounts)
if (params.debug_sidechain):
input("Start sidechain server and press enter to continue: ")
else:
_rm_debug_log(params.sidechain_config)
with single_client_app(
config=params.sidechain_config,
exe=params.sidechain_exe,
standalone=True,
run_server=not params.debug_sidechain) as sc_app:
sc_connect_subscription(sc_app, params.sc_door_account)
setup_sidechain(sc_app, params, setup_user_accounts)
callback(mc_app, sc_app)
def _convert_log_files_to_json(to_convert: List[ConfigFile], suffix: str):
'''
Convert the log file to json
'''
for c in to_convert:
try:
debug_log = c.debug_logfile.get_line()
if not os.path.exists(debug_log):
continue
converted_log = f'{debug_log}.{suffix}'
if os.path.exists(converted_log):
os.remove(converted_log)
print(f'Converting log {debug_log} to {converted_log}', flush=True)
convert_log(debug_log, converted_log, pure_json=True)
except:
eprint(f'Exception converting log')
def _multinode_with_callback(params: Params,
callback: Callable[[App, App], None],
setup_user_accounts: bool = True):
mainchain_cfg = ConfigFile(
file_name=
f'{params.configs_dir}/sidechain_testnet/main.no_shards.mainchain_0/rippled.cfg'
)
_rm_debug_log(mainchain_cfg)
if params.debug_mainchain:
input("Start mainchain server and press enter to continue: ")
with single_client_app(config=mainchain_cfg,
exe=params.mainchain_exe,
standalone=True,
run_server=not params.debug_mainchain) as mc_app:
if params.with_pauses:
input("Pausing after mainchain start (press enter to continue)")
mc_connect_subscription(mc_app, params.mc_door_account)
setup_mainchain(mc_app, params, setup_user_accounts)
if params.with_pauses:
input("Pausing after mainchain setup (press enter to continue)")
testnet_configs = configs_for_testnet(
f'{params.configs_dir}/sidechain_testnet/sidechain_')
for c in testnet_configs:
_rm_debug_log(c)
run_server_list = [True] * len(testnet_configs)
if params.debug_sidechain:
run_server_list[0] = False
input(
f'Start testnet server {testnet_configs[0].get_file_name()} and press enter to continue: '
)
with testnet_app(exe=params.sidechain_exe,
configs=testnet_configs,
run_server=run_server_list) as n_app:
if params.with_pauses:
input("Pausing after testnet start (press enter to continue)")
sc_connect_subscription(n_app, params.sc_door_account)
setup_sidechain(n_app, params, setup_user_accounts)
if params.with_pauses:
input(
"Pausing after sidechain setup (press enter to continue)")
callback(mc_app, n_app)
def standalone_test(params: Params):
def callback(mc_app: App, sc_app: App):
simple_test(mc_app, sc_app, params)
_standalone_with_callback(params, callback)
def multinode_test(params: Params):
def callback(mc_app: App, sc_app: App):
simple_test(mc_app, sc_app, params)
_multinode_with_callback(params, callback)
# The mainchain runs in standalone mode. Most operations - like cross chain
# paymens - will automatically close ledgers. However, some operations, like
# refunds need an extra close. This loop automatically closes ledgers.
def close_mainchain_ledgers(stop_token: Value, params: Params, sleep_time=4):
with single_client_app(config=params.mainchain_config,
exe=params.mainchain_exe,
standalone=True,
run_server=False) as mc_app:
while stop_token.value != 0:
mc_app.maybe_ledger_accept()
time.sleep(sleep_time)
def standalone_interactive_repl(params: Params):
def callback(mc_app: App, sc_app: App):
# process will run while stop token is non-zero
stop_token = Value('i', 1)
p = None
if mc_app.standalone:
p = Process(target=close_mainchain_ledgers,
args=(stop_token, params))
p.start()
try:
interactive.repl(mc_app, sc_app)
finally:
if p:
stop_token.value = 0
p.join()
_standalone_with_callback(params, callback, setup_user_accounts=False)
def multinode_interactive_repl(params: Params):
def callback(mc_app: App, sc_app: App):
# process will run while stop token is non-zero
stop_token = Value('i', 1)
p = None
if mc_app.standalone:
p = Process(target=close_mainchain_ledgers,
args=(stop_token, params))
p.start()
try:
interactive.repl(mc_app, sc_app)
finally:
if p:
stop_token.value = 0
p.join()
_multinode_with_callback(params, callback, setup_user_accounts=False)
def main():
params = Params()
interactive.set_hooks_dir(params.hooks_dir)
if err_str := params.check_error():
eprint(err_str)
sys.exit(1)
if params.quiet:
print("Disabling eprint")
disable_eprint()
if params.interactive:
if params.standalone:
standalone_interactive_repl(params)
else:
multinode_interactive_repl(params)
elif params.standalone:
standalone_test(params)
else:
multinode_test(params)
if __name__ == '__main__':
main()

174
bin/sidechain/python/test_utils.py Normal file
View File

@@ -0,0 +1,174 @@
import asyncio
import collections
from contextlib import contextmanager
import json
import logging
import pprint
import time
from typing import Callable, Dict, List, Optional
from app import App, balances_dataframe
from common import Account, Asset, XRP, eprint
from command import Subscribe
MC_SUBSCRIBE_QUEUE = []
SC_SUBSCRIBE_QUEUE = []
def _mc_subscribe_callback(v: dict):
MC_SUBSCRIBE_QUEUE.append(v)
logging.info(f'mc subscribe_callback:\n{json.dumps(v, indent=1)}')
def _sc_subscribe_callback(v: dict):
SC_SUBSCRIBE_QUEUE.append(v)
logging.info(f'sc subscribe_callback:\n{json.dumps(v, indent=1)}')
def mc_connect_subscription(app: App, door_account: Account):
app(Subscribe(accounts=[door_account]), _mc_subscribe_callback)
def sc_connect_subscription(app: App, door_account: Account):
app(Subscribe(accounts=[door_account]), _sc_subscribe_callback)
# This pops elements off the subscribe_queue until the transaction is found
# It mofifies the queue in place.
async def async_wait_for_payment_detect(app: App, subscribe_queue: List[dict],
src: Account, dst: Account,
amt_asset: Asset):
logging.info(
f'Wait for payment {src.account_id = } {dst.account_id = } {amt_asset = }'
)
n_txns = 10 # keep this many txn in a circular buffer.
# If the payment is not detected, write them to the log.
last_n_paytxns = collections.deque(maxlen=n_txns)
for i in range(30):
while subscribe_queue:
d = subscribe_queue.pop(0)
if 'transaction' not in d:
continue
txn = d['transaction']
if txn['TransactionType'] != 'Payment':
continue
txn_asset = Asset(from_rpc_result=txn['Amount'])
if txn['Account'] == src.account_id and txn[
'Destination'] == dst.account_id and txn_asset == amt_asset:
if d['engine_result_code'] == 0:
logging.info(
f'Found payment {src.account_id = } {dst.account_id = } {amt_asset = }'
)
return
else:
logging.error(
f'Expected payment failed {src.account_id = } {dst.account_id = } {amt_asset = }'
)
raise ValueError(
f'Expected payment failed {src.account_id = } {dst.account_id = } {amt_asset = }'
)
else:
last_n_paytxns.append(txn)
if i > 0 and not (i % 5):
logging.warning(
f'Waiting for txn detect {src.account_id = } {dst.account_id = } {amt_asset = }'
)
# side chain can send transactions to the main chain, but won't close the ledger
# We don't know when the transaction will be sent, so may need to close the ledger here
await app.async_maybe_ledger_accept()
await asyncio.sleep(2)
logging.warning(
f'Last {len(last_n_paytxns)} pay txns while waiting for payment detect'
)
for t in last_n_paytxns:
logging.warning(
f'Detected pay transaction while waiting for payment: {t}')
logging.error(
f'Expected txn detect {src.account_id = } {dst.account_id = } {amt_asset = }'
)
raise ValueError(
f'Expected txn detect {src.account_id = } {dst.account_id = } {amt_asset = }'
)
def mc_wait_for_payment_detect(app: App, src: Account, dst: Account,
amt_asset: Asset):
logging.info(f'mainchain waiting for payment detect')
return asyncio.get_event_loop().run_until_complete(
async_wait_for_payment_detect(app, MC_SUBSCRIBE_QUEUE, src, dst,
amt_asset))
def sc_wait_for_payment_detect(app: App, src: Account, dst: Account,
amt_asset: Asset):
logging.info(f'sidechain waiting for payment detect')
return asyncio.get_event_loop().run_until_complete(
async_wait_for_payment_detect(app, SC_SUBSCRIBE_QUEUE, src, dst,
amt_asset))
def wait_for_balance_change(app: App,
acc: Account,
pre_balance: Asset,
final_diff: Optional[Asset] = None):
logging.info(
f'waiting for balance change {acc.account_id = } {pre_balance = } {final_diff = }'
)
for i in range(30):
new_bal = app.get_balance(acc, pre_balance(0))
diff = new_bal - pre_balance
if new_bal != pre_balance:
logging.info(
f'Balance changed {acc.account_id = } {pre_balance = } {new_bal = } {diff = } {final_diff = }'
)
if final_diff is None or diff == final_diff:
return
app.maybe_ledger_accept()
time.sleep(2)
if i > 0 and not (i % 5):
logging.warning(
f'Waiting for balance to change {acc.account_id = } {pre_balance = }'
)
logging.error(
f'Expected balance to change {acc.account_id = } {pre_balance = } {new_bal = } {diff = } {final_diff = }'
)
raise ValueError(
f'Expected balance to change {acc.account_id = } {pre_balance = } {new_bal = } {diff = } {final_diff = }'
)
def log_chain_state(mc_app, sc_app, log, msg='Chain State'):
chains = [mc_app, sc_app]
chain_names = ['mainchain', 'sidechain']
balances = balances_dataframe(chains, chain_names)
df_as_str = balances.to_string(float_format=lambda x: f'{x:,.6f}')
log(f'{msg} Balances: \n{df_as_str}')
federator_info = sc_app.federator_info()
log(f'{msg} Federator Info: \n{pprint.pformat(federator_info)}')
# Tests can set this to True to help debug test failures by showing account
# balances in the log before the test runs
test_context_verbose_logging = False
@contextmanager
def test_context(mc_app, sc_app, verbose_logging: Optional[bool] = None):
'''Write extra context info to the log on test failure'''
global test_context_verbose_logging
if verbose_logging is None:
verbose_logging = test_context_verbose_logging
try:
if verbose_logging:
log_chain_state(mc_app, sc_app, logging.info)
start_time = time.monotonic()
yield
except:
log_chain_state(mc_app, sc_app, logging.error)
raise
finally:
elapased_time = time.monotonic() - start_time
logging.info(f'Test elapsed time: {elapased_time}')
if verbose_logging:
log_chain_state(mc_app, sc_app, logging.info)

206
bin/sidechain/python/testnet.py Normal file
View File

@@ -0,0 +1,206 @@
'''
Bring up a rippled testnetwork from a set of config files with fixed ips.
'''
from contextlib import contextmanager
import glob
import os
import subprocess
import time
from typing import Callable, List, Optional, Set, Union
from command import ServerInfo
from config_file import ConfigFile
from ripple_client import RippleClient
class Network:
# If run_server is None, run all the servers.
# This is useful to help debugging
def __init__(self,
exe: str,
configs: List[ConfigFile],
*,
command_logs: Optional[List[str]] = None,
run_server: Optional[List[bool]] = None,
extra_args: Optional[List[List[str]]] = None):
if not configs:
raise ValueError(f'Must specify at least one config')
if run_server and len(run_server) != len(configs):
raise ValueError(
f'run_server length must match number of configs (or be None): {len(configs) = } {len(run_server) = }'
)
self.configs = configs
self.clients = []
self.running_server_indexes = set()
self.processes = {}
if not run_server:
run_server = []
run_server += [True] * (len(configs) - len(run_server))
self.run_server = run_server
if not command_logs:
command_logs = []
command_logs += [None] * (len(configs) - len(command_logs))
self.command_logs = command_logs
# remove the old database directories.
# we want tests to start from the same empty state every time
for config in self.configs:
db_path = config.database_path.get_line()
if db_path and os.path.isdir(db_path):
files = glob.glob(f'{db_path}/**', recursive=True)
for f in files:
if os.path.isdir(f):
continue
os.unlink(f)
for config, log in zip(self.configs, self.command_logs):
client = RippleClient(config=config, command_log=log, exe=exe)
self.clients.append(client)
self.servers_start(extra_args=extra_args)
def shutdown(self):
for a in self.clients:
a.shutdown()
self.servers_stop()
def num_clients(self) -> int:
return len(self.clients)
def get_client(self, i: int) -> RippleClient:
return self.clients[i]
def get_configs(self) -> List[ConfigFile]:
return [c.config for c in self.clients]
def get_pids(self) -> List[int]:
return [c.get_pid() for c in self.clients if c.get_pid() is not None]
# Get a dict of the server_state, validated_ledger_seq, and complete_ledgers
def get_brief_server_info(self) -> dict:
ret = {'server_state': [], 'ledger_seq': [], 'complete_ledgers': []}
for c in self.clients:
r = c.get_brief_server_info()
for (k, v) in r.items():
ret[k].append(v)
return ret
# returns true if the server is running, false if not. Note, this relies on
# servers being shut down through the `servers_stop` interface. If a server
# crashes, or is started or stopped through other means, an incorrect status
# may be reported.
def get_running_status(self) -> List[bool]:
return [
i in self.running_server_indexes for i in range(len(self.clients))
]
def is_running(self, index: int) -> bool:
return index in self.running_server_indexes
def wait_for_validated_ledger(self, server_index: Optional[int] = None):
'''
Don't return until the network has at least one validated ledger
'''
if server_index is None:
for i in range(len(self.configs)):
self.wait_for_validated_ledger(i)
return
client = self.clients[server_index]
for i in range(600):
r = client.send_command(ServerInfo())
state = None
if 'info' in r:
state = r['info']['server_state']
if state == 'proposing':
print(f'Synced: {server_index} : {state}', flush=True)
break
if not i % 10:
print(f'Waiting for sync: {server_index} : {state}',
flush=True)
time.sleep(1)
for i in range(600):
r = client.send_command(ServerInfo())
state = None
if 'info' in r:
complete_ledgers = r['info']['complete_ledgers']
if complete_ledgers and complete_ledgers != 'empty':
print(f'Have complete ledgers: {server_index} : {state}',
flush=True)
return
if not i % 10:
print(
f'Waiting for complete_ledgers: {server_index} : {complete_ledgers}',
flush=True)
time.sleep(1)
raise ValueError('Could not sync server {client.config_file_name}')
def servers_start(self,
server_indexes: Optional[Union[Set[int],
List[int]]] = None,
*,
extra_args: Optional[List[List[str]]] = None):
if server_indexes is None:
server_indexes = [i for i in range(len(self.clients))]
if extra_args is None:
extra_args = []
extra_args += [list()] * (len(self.configs) - len(extra_args))
for i in server_indexes:
if i in self.running_server_indexes or not self.run_server[i]:
continue
client = self.clients[i]
to_run = [client.exe, '--conf', client.config_file_name]
print(f'Starting server {client.config_file_name}')
fout = open(os.devnull, 'w')
p = subprocess.Popen(to_run + extra_args[i],
stdout=fout,
stderr=subprocess.STDOUT)
client.set_pid(p.pid)
print(
f'started rippled: config: {client.config_file_name} PID: {p.pid}',
flush=True)
self.running_server_indexes.add(i)
self.processes[i] = p
time.sleep(2) # give servers time to start
def servers_stop(self,
server_indexes: Optional[Union[Set[int],
List[int]]] = None):
if server_indexes is None:
server_indexes = self.running_server_indexes.copy()
if 0 in server_indexes:
print(
f'WARNING: Server 0 is being stopped. RPC commands cannot be sent until this is restarted.'
)
for i in server_indexes:
if i not in self.running_server_indexes:
continue
client = self.clients[i]
to_run = [client.exe, '--conf', client.config_file_name]
fout = open(os.devnull, 'w')
subprocess.Popen(to_run + ['stop'],
stdout=fout,
stderr=subprocess.STDOUT)
self.running_server_indexes.discard(i)
for i in server_indexes:
self.processes[i].wait()
del self.processes[i]
self.get_client(i).set_pid(-1)

64
bin/sidechain/python/tests/conftest.py Normal file
View File

@@ -0,0 +1,64 @@
# Add parent directory to module path
import os, sys
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from common import Account, Asset, XRP
import create_config_files
import sidechain
import pytest
'''
Sidechains uses argparse.ArgumentParser to add command line options.
The function call to add an argument is `add_argument`. pytest uses `addoption`.
This wrapper class changes calls from `add_argument` to calls to `addoption`.
To avoid conflicts between pytest and sidechains, all sidechain arguments have
the suffix `_sc` appended to them. I.e. `--verbose` is for pytest, `--verbose_sc`
is for sidechains.
'''
class ArgumentParserWrapper:
def __init__(self, wrapped):
self.wrapped = wrapped
def add_argument(self, *args, **kwargs):
for a in args:
if not a.startswith('--'):
continue
a = a + '_sc'
self.wrapped.addoption(a, **kwargs)
def pytest_addoption(parser):
wrapped = ArgumentParserWrapper(parser)
sidechain.parse_args_helper(wrapped)
def _xchain_assets(ratio: int = 1):
assets = {}
assets['xrp_xrp_sidechain_asset'] = create_config_files.XChainAsset(
XRP(0), XRP(0), 1, 1 * ratio, 200, 200 * ratio)
root_account = Account(account_id="rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh")
main_iou_asset = Asset(value=0, currency='USD', issuer=root_account)
side_iou_asset = Asset(value=0, currency='USD', issuer=root_account)
assets['iou_iou_sidechain_asset'] = create_config_files.XChainAsset(
main_iou_asset, side_iou_asset, 1, 1 * ratio, 0.02, 0.02 * ratio)
return assets
# Diction of config dirs. Key is ratio
_config_dirs = None
@pytest.fixture
def configs_dirs_dict(tmp_path):
global _config_dirs
if not _config_dirs:
params = create_config_files.Params()
_config_dirs = {}
for ratio in (1, 2):
params.configs_dir = str(tmp_path / f'test_config_files_{ratio}')
create_config_files.main(params, _xchain_assets(ratio))
_config_dirs[ratio] = params.configs_dir
return _config_dirs

191
bin/sidechain/python/tests/simple_xchain_transfer_test.py Normal file
View File

@@ -0,0 +1,191 @@
import logging
import pprint
import pytest
from multiprocessing import Process, Value
from typing import Dict
import sys
from app import App
from common import Asset, eprint, disable_eprint, XRP
import interactive
from sidechain import Params
import sidechain
import test_utils
import time
from transaction import Payment, Trust
def simple_xrp_test(mc_app: App, sc_app: App, params: Params):
alice = mc_app.account_from_alias('alice')
adam = sc_app.account_from_alias('adam')
# main to side
# First txn funds the side chain account
with test_utils.test_context(mc_app, sc_app):
to_send_asset = XRP(1000)
pre_bal = sc_app.get_balance(adam, to_send_asset)
sidechain.main_to_side_transfer(mc_app, sc_app, alice, adam,
to_send_asset, params)
test_utils.wait_for_balance_change(sc_app, adam, pre_bal,
to_send_asset)
for i in range(2):
# even amounts for main to side
for value in range(10, 20, 2):
with test_utils.test_context(mc_app, sc_app):
to_send_asset = XRP(value)
pre_bal = sc_app.get_balance(adam, to_send_asset)
sidechain.main_to_side_transfer(mc_app, sc_app, alice, adam,
to_send_asset, params)
test_utils.wait_for_balance_change(sc_app, adam, pre_bal,
to_send_asset)
# side to main
# odd amounts for side to main
for value in range(9, 19, 2):
with test_utils.test_context(mc_app, sc_app):
to_send_asset = XRP(value)
pre_bal = mc_app.get_balance(alice, to_send_asset)
sidechain.side_to_main_transfer(mc_app, sc_app, adam, alice,
to_send_asset, params)
test_utils.wait_for_balance_change(mc_app, alice, pre_bal,
to_send_asset)
def simple_iou_test(mc_app: App, sc_app: App, params: Params):
alice = mc_app.account_from_alias('alice')
adam = sc_app.account_from_alias('adam')
mc_asset = Asset(value=0,
currency='USD',
issuer=mc_app.account_from_alias('root'))
sc_asset = Asset(value=0,
currency='USD',
issuer=sc_app.account_from_alias('door'))
mc_app.add_asset_alias(mc_asset, 'mcd') # main chain dollar
sc_app.add_asset_alias(sc_asset, 'scd') # side chain dollar
mc_app(Trust(account=alice, limit_amt=mc_asset(1_000_000)))
## make sure adam account on the side chain exists and set the trust line
with test_utils.test_context(mc_app, sc_app):
sidechain.main_to_side_transfer(mc_app, sc_app, alice, adam, XRP(300),
params)
# create a trust line to alice and pay her USD/root
mc_app(Trust(account=alice, limit_amt=mc_asset(1_000_000)))
mc_app.maybe_ledger_accept()
mc_app(
Payment(account=mc_app.account_from_alias('root'),
dst=alice,
amt=mc_asset(10_000)))
mc_app.maybe_ledger_accept()
# create a trust line for adam
sc_app(Trust(account=adam, limit_amt=sc_asset(1_000_000)))
for i in range(2):
# even amounts for main to side
for value in range(10, 20, 2):
with test_utils.test_context(mc_app, sc_app):
to_send_asset = mc_asset(value)
rcv_asset = sc_asset(value)
pre_bal = sc_app.get_balance(adam, rcv_asset)
sidechain.main_to_side_transfer(mc_app, sc_app, alice, adam,
to_send_asset, params)
test_utils.wait_for_balance_change(sc_app, adam, pre_bal,
rcv_asset)
# side to main
# odd amounts for side to main
for value in range(9, 19, 2):
with test_utils.test_context(mc_app, sc_app):
to_send_asset = sc_asset(value)
rcv_asset = mc_asset(value)
pre_bal = mc_app.get_balance(alice, to_send_asset)
sidechain.side_to_main_transfer(mc_app, sc_app, adam, alice,
to_send_asset, params)
test_utils.wait_for_balance_change(mc_app, alice, pre_bal,
rcv_asset)
def run(mc_app: App, sc_app: App, params: Params):
# process will run while stop token is non-zero
stop_token = Value('i', 1)
p = None
if mc_app.standalone:
p = Process(target=sidechain.close_mainchain_ledgers,
args=(stop_token, params))
p.start()
try:
# TODO: Tests fail without this sleep. Fix this bug.
time.sleep(10)
setup_accounts(mc_app, sc_app, params)
simple_xrp_test(mc_app, sc_app, params)
simple_iou_test(mc_app, sc_app, params)
finally:
if p:
stop_token.value = 0
p.join()
sidechain._convert_log_files_to_json(
mc_app.get_configs() + sc_app.get_configs(), 'final.json')
def standalone_test(params: Params):
def callback(mc_app: App, sc_app: App):
run(mc_app, sc_app, params)
sidechain._standalone_with_callback(params,
callback,
setup_user_accounts=False)
def setup_accounts(mc_app: App, sc_app: App, params: Params):
# Setup a funded user account on the main chain, and add an unfunded account.
# Setup address book and add a funded account on the mainchain.
# Typical female names are addresses on the mainchain.
# The first account is funded.
alice = mc_app.create_account('alice')
beth = mc_app.create_account('beth')
carol = mc_app.create_account('carol')
deb = mc_app.create_account('deb')
ella = mc_app.create_account('ella')
mc_app(Payment(account=params.genesis_account, dst=alice, amt=XRP(20_000)))
mc_app.maybe_ledger_accept()
# Typical male names are addresses on the sidechain.
# All accounts are initially unfunded
adam = sc_app.create_account('adam')
bob = sc_app.create_account('bob')
charlie = sc_app.create_account('charlie')
dan = sc_app.create_account('dan')
ed = sc_app.create_account('ed')
def multinode_test(params: Params):
def callback(mc_app: App, sc_app: App):
run(mc_app, sc_app, params)
sidechain._multinode_with_callback(params,
callback,
setup_user_accounts=False)
def test_simple_xchain(configs_dirs_dict: Dict[int, str]):
params = sidechain.Params(configs_dir=configs_dirs_dict[1])
if err_str := params.check_error():
eprint(err_str)
sys.exit(1)
if params.verbose:
print("eprint enabled")
else:
disable_eprint()
# Set to true to help debug tests
test_utils.test_context_verbose_logging = True
if params.standalone:
standalone_test(params)
else:
multinode_test(params)

366
bin/sidechain/python/transaction.py Normal file
View File

@@ -0,0 +1,366 @@
import datetime
import json
from typing import Dict, List, Optional, Union
from command import Command
from common import Account, Asset, Path, PathList, to_rippled_epoch
class Transaction(Command):
'''Interface for all transactions'''
def __init__(
self,
*,
account: Account,
flags: Optional[int] = None,
fee: Optional[Union[Asset, int]] = None,
sequence: Optional[int] = None,
account_txn_id: Optional[str] = None,
last_ledger_sequence: Optional[int] = None,
src_tag: Optional[int] = None,
memos: Optional[List[Dict[str, dict]]] = None,
):
super().__init__()
self.account = account
# set even if None
self.flags = flags
self.fee = fee
self.sequence = sequence
self.account_txn_id = account_txn_id
self.last_ledger_sequence = last_ledger_sequence
self.src_tag = src_tag
self.memos = memos
def cmd_name(self) -> str:
return 'submit'
def set_seq_and_fee(self, seq: int, fee: Union[Asset, int]):
self.sequence = seq
self.fee = fee
def to_cmd_obj(self) -> dict:
txn = {
'Account': self.account.account_id,
}
if self.flags is not None:
txn['Flags'] = flags
if self.fee is not None:
if isinstance(self.fee, int):
txn['Fee'] = f'{self.fee}' # must be a string
else:
txn['Fee'] = self.fee.to_cmd_obj()
if self.sequence is not None:
txn['Sequence'] = self.sequence
if self.account_txn_id is not None:
txn['AccountTxnID'] = self.account_txn_id
if self.last_ledger_sequence is not None:
txn['LastLedgerSequence'] = self.last_ledger_sequence
if self.src_tag is not None:
txn['SourceTag'] = self.src_tag
if self.memos is not None:
txn['Memos'] = self.memos
return txn
class Payment(Transaction):
'''A payment transaction'''
def __init__(self,
*,
dst: Account,
amt: Asset,
send_max: Optional[Asset] = None,
paths: Optional[PathList] = None,
dst_tag: Optional[int] = None,
deliver_min: Optional[Asset] = None,
**rest):
super().__init__(**rest)
self.dst = dst
self.amt = amt
self.send_max = send_max
if paths is not None and isinstance(paths, Path):
# allow paths = Path([...]) special case
self.paths = PathList([paths])
else:
self.paths = paths
self.dst_tag = dst_tag
self.deliver_min = deliver_min
def set_partial_payment(self, value: bool = True):
'''Set or clear the partial payment flag'''
self._set_flag(0x0002_0000, value)
def to_cmd_obj(self) -> dict:
'''convert to transaction form (suitable for using json.dumps or similar)'''
txn = super().to_cmd_obj()
txn = {
**txn,
'TransactionType': 'Payment',
'Destination': self.dst.account_id,
'Amount': self.amt.to_cmd_obj(),
}
if self.paths is not None:
txn['Paths'] = self.paths.to_cmd_obj()
if self.send_max is not None:
txn['SendMax'] = self.send_max.to_cmd_obj()
if self.dst_tag is not None:
txn['DestinationTag'] = self.dst_tag
if self.deliver_min is not None:
txn['DeliverMin'] = self.deliver_min
return txn
class Trust(Transaction):
'''A trust set transaction'''
def __init__(self,
*,
limit_amt: Optional[Asset] = None,
qin: Optional[int] = None,
qout: Optional[int] = None,
**rest):
super().__init__(**rest)
self.limit_amt = limit_amt
self.qin = qin
self.qout = qout
def set_auth(self):
'''Set the auth flag (cannot be cleared)'''
self._set_flag(0x00010000)
return self
def set_no_ripple(self, value: bool = True):
'''Set or clear the noRipple flag'''
self._set_flag(0x0002_0000, value)
self._set_flag(0x0004_0000, not value)
return self
def set_freeze(self, value: bool = True):
'''Set or clear the freeze flag'''
self._set_flag(0x0020_0000, value)
self._set_flag(0x0040_0000, not value)
return self
def to_cmd_obj(self) -> dict:
'''convert to transaction form (suitable for using json.dumps or similar)'''
result = super().to_cmd_obj()
result = {
**result,
'TransactionType': 'TrustSet',
'LimitAmount': self.limit_amt.to_cmd_obj(),
}
if self.qin is not None:
result['QualityIn'] = self.qin
if self.qout is not None:
result['QualityOut'] = self.qout
return result
class SetRegularKey(Transaction):
'''A SetRegularKey transaction'''
def __init__(self, *, key: str, **rest):
super().__init__(**rest)
self.key = key
def to_cmd_obj(self) -> dict:
'''convert to transaction form (suitable for using json.dumps or similar)'''
result = super().to_cmd_obj()
result = {
**result,
'TransactionType': 'SetRegularKey',
'RegularKey': self.key,
}
return result
class SignerListSet(Transaction):
'''A SignerListSet transaction'''
def __init__(self,
*,
keys: List[str],
weights: Optional[List[int]] = None,
quorum: int,
**rest):
super().__init__(**rest)
self.keys = keys
self.quorum = quorum
if weights:
if len(weights) != len(keys):
raise ValueError(
f'SignerSetList number of weights must equal number of keys (or be empty). Weights: {weights} Keys: {keys}'
)
self.weights = weights
else:
self.weights = [1] * len(keys)
def to_cmd_obj(self) -> dict:
'''convert to transaction form (suitable for using json.dumps or similar)'''
result = super().to_cmd_obj()
result = {
**result,
'TransactionType': 'SignerListSet',
'SignerQuorum': self.quorum,
}
entries = []
for k, w in zip(self.keys, self.weights):
entries.append({'SignerEntry': {'Account': k, 'SignerWeight': w}})
result['SignerEntries'] = entries
return result
class AccountSet(Transaction):
'''An account set transaction'''
def __init__(self, account: Account, **rest):
super().__init__(account=account, **rest)
self.clear_flag = None
self.set_flag = None
self.transfer_rate = None
self.tick_size = None
def _set_account_flag(self, flag_id: int, value):
if value:
self.set_flag = flag_id
else:
self.clear_flag = flag_id
return self
def set_account_txn_id(self, value: bool = True):
'''Set or clear the asfAccountTxnID flag'''
return self._set_account_flag(5, value)
def set_default_ripple(self, value: bool = True):
'''Set or clear the asfDefaultRipple flag'''
return self._set_account_flag(8, value)
def set_deposit_auth(self, value: bool = True):
'''Set or clear the asfDepositAuth flag'''
return self._set_account_flag(9, value)
def set_disable_master(self, value: bool = True):
'''Set or clear the asfDisableMaster flag'''
return self._set_account_flag(4, value)
def set_disallow_xrp(self, value: bool = True):
'''Set or clear the asfDisallowXRP flag'''
return self._set_account_flag(3, value)
def set_global_freeze(self, value: bool = True):
'''Set or clear the asfGlobalFreeze flag'''
return self._set_account_flag(7, value)
def set_no_freeze(self, value: bool = True):
'''Set or clear the asfNoFreeze flag'''
return self._set_account_flag(6, value)
def set_require_auth(self, value: bool = True):
'''Set or clear the asfRequireAuth flag'''
return self._set_account_flag(2, value)
def set_require_dest(self, value: bool = True):
'''Set or clear the asfRequireDest flag'''
return self._set_account_flag(1, value)
def set_transfer_rate(self, value: int):
'''Set the fee to change when users transfer this account's issued currencies'''
self.transfer_rate = value
return self
def set_tick_size(self, value: int):
'''Tick size to use for offers involving a currency issued by this address'''
self.tick_size = value
return self
def to_cmd_obj(self) -> dict:
'''convert to transaction form (suitable for using json.dumps or similar)'''
result = super().to_cmd_obj()
result = {
**result,
'TransactionType': 'AccountSet',
}
if self.clear_flag is not None:
result['ClearFlag'] = self.clear_flag
if self.set_flag is not None:
result['SetFlag'] = self.set_flag
if self.transfer_rate is not None:
result['TransferRate'] = self.transfer_rate
if self.tick_size is not None:
result['TickSize'] = self.tick_size
return result
class Offer(Transaction):
'''An offer transaction'''
def __init__(self,
*,
taker_pays: Asset,
taker_gets: Asset,
expiration: Optional[int] = None,
offer_sequence: Optional[int] = None,
**rest):
super().__init__(**rest)
self.taker_pays = taker_pays
self.taker_gets = taker_gets
self.expiration = expiration
self.offer_sequence = offer_sequence
def set_passive(self, value: bool = True):
return self._set_flag(0x0001_0000, value)
def set_immediate_or_cancel(self, value: bool = True):
return self._set_flag(0x0002_0000, value)
def set_fill_or_kill(self, value: bool = True):
return self._set_flag(0x0004_0000, value)
def set_sell(self, value: bool = True):
return self._set_flag(0x0008_0000, value)
def to_cmd_obj(self) -> dict:
txn = super().to_cmd_obj()
txn = {
**txn,
'TransactionType': 'OfferCreate',
'TakerPays': self.taker_pays.to_cmd_obj(),
'TakerGets': self.taker_gets.to_cmd_obj(),
}
if self.expiration is not None:
txn['Expiration'] = self.expiration
if self.offer_sequence is not None:
txn['OfferSequence'] = self.offer_sequence
return txn
class Ticket(Transaction):
'''A ticket create transaction'''
def __init__(self, *, count: int = 1, **rest):
super().__init__(**rest)
self.count = count
def to_cmd_obj(self) -> dict:
txn = super().to_cmd_obj()
txn = {
**txn,
'TransactionType': 'TicketCreate',
'TicketCount': self.count,
}
return txn
class SetHook(Transaction):
'''A SetHook transaction for the experimental hook amendment'''
def __init__(self,
*,
create_code: str,
hook_on: str = '0000000000000000',
**rest):
super().__init__(**rest)
self.create_code = create_code
self.hook_on = hook_on
def to_cmd_obj(self) -> dict:
txn = super().to_cmd_obj()
txn = {
**txn,
'TransactionType': 'SetHook',
'CreateCode': self.create_code,
'HookOn': self.hook_on,
}
return txn