diff --git a/bin/ripple/ledger/Args.py b/bin/ripple/ledger/Args.py index 1845a9aabe..9c1b04d341 100644 --- a/bin/ripple/ledger/Args.py +++ b/bin/ripple/ledger/Args.py @@ -13,6 +13,7 @@ from ripple.util.Function import Function NAME = 'LedgerTool' VERSION = '0.1' +NONE = '(none)' _parser = argparse.ArgumentParser( prog=NAME, @@ -57,7 +58,14 @@ _parser.add_argument( ) _parser.add_argument( - '--display', '-d', + '--database', '-d', + nargs='*', + default=NONE, + help='Specify a database.', + ) + +_parser.add_argument( + '--display', help='Specify a function to display ledgers.', ) @@ -103,6 +111,12 @@ _parser.add_argument( help='If true, display times in UTC rather than local time.', ) +_parser.add_argument( + '--validations', + default=3, + help='The number of validations needed before considering a ledger valid.', + ) + _parser.add_argument( '--version', action='version', @@ -131,6 +145,7 @@ _parser.add_argument( # Read the arguments from the command line. ARGS = _parser.parse_args() +ARGS.NONE = NONE Log.VERBOSE = ARGS.verbose @@ -162,10 +177,11 @@ if ARGS.window < 0: PrettyPrint.INDENT = (ARGS.indent * ' ') -_loaders = bool(ARGS.server) + bool(ARGS.rippled) +_loaders = (ARGS.database != NONE) + bool(ARGS.rippled) + bool(ARGS.server) if not _loaders: ARGS.rippled = 'rippled' elif _loaders > 1: - raise ValueError('At most one of --rippled and --server must be specified') + raise ValueError('At most one of --database, --rippled and --server ' + 'may be specified') diff --git a/bin/ripple/ledger/DatabaseReader.py b/bin/ripple/ledger/DatabaseReader.py new file mode 100644 index 0000000000..fe8b52e8c1 --- /dev/null +++ b/bin/ripple/ledger/DatabaseReader.py @@ -0,0 +1,78 @@ +from __future__ import absolute_import, division, print_function, unicode_literals + +import json +import os +import subprocess + +from ripple.ledger.Args import ARGS +from ripple.util import ConfigFile +from ripple.util import Database +from ripple.util import File +from ripple.util import Log +from ripple.util import Range + +LEDGER_QUERY = """ +SELECT + L.*, count(1) validations +FROM + (select LedgerHash, LedgerSeq from Ledgers ORDER BY LedgerSeq DESC) L + JOIN Validations V + ON (V.LedgerHash = L.LedgerHash) + GROUP BY L.LedgerHash + HAVING validations >= {validation_quorum} + ORDER BY 2; +""" + +COMPLETE_QUERY = """ +SELECT + L.LedgerSeq, count(*) validations +FROM + (select LedgerHash, LedgerSeq from Ledgers ORDER BY LedgerSeq) L + JOIN Validations V + ON (V.LedgerHash = L.LedgerHash) + GROUP BY L.LedgerHash + HAVING validations >= :validation_quorum + ORDER BY 2; +""" + +_DATABASE_NAME = 'ledger.db' + +USE_PLACEHOLDERS = False + +class DatabaseReader(object): + def __init__(self, config): + assert ARGS.database != ARGS.NONE + database = ARGS.database or config['database_path'] + if not database.endswith(_DATABASE_NAME): + database = os.path.join(database, _DATABASE_NAME) + if USE_PLACEHOLDERS: + cursor = Database.fetchall( + database, COMPLETE_QUERY, config) + else: + cursor = Database.fetchall( + database, LEDGER_QUERY.format(**config), {}) + self.complete = [c[1] for c in cursor] + + def name_to_ledger_index(self, ledger_name, is_full=False): + if not self.complete: + return None + if ledger_name == 'closed': + return self.complete[-1] + if ledger_name == 'current': + return None + if ledger_name == 'validated': + return self.complete[-1] + + def get_ledger(self, name, is_full=False): + cmd = ['ledger', str(name)] + if is_full: + cmd.append('full') + response = self._command(*cmd) + result = response.get('ledger') + if result: + return result + error = response['error'] + etext = _ERROR_TEXT.get(error) + if etext: + error = '%s (%s)' % (etext, error) + Log.fatal(_ERROR_TEXT.get(error, error)) diff --git a/bin/ripple/ledger/RippledReader.py b/bin/ripple/ledger/RippledReader.py index b79df532d6..5c970cc072 100644 --- a/bin/ripple/ledger/RippledReader.py +++ b/bin/ripple/ledger/RippledReader.py @@ -22,13 +22,13 @@ _ERROR_TEXT = { _DEFAULT_ERROR_ = "Couldn't connect to server." class RippledReader(object): - def __init__(self): + def __init__(self, config): fname = File.normalize(ARGS.rippled) if not os.path.exists(fname): raise Exception('No rippled found at %s.' % fname) self.cmd = [fname] if ARGS.config: - self.cmd.extend(['--conf', _normalize(ARGS.config)]) + self.cmd.extend(['--conf', File.normalize(ARGS.config)]) self.info = self._command('server_info')['info'] c = self.info.get('complete_ledgers') if c == 'empty': diff --git a/bin/ripple/ledger/Server.py b/bin/ripple/ledger/Server.py index 57966e3379..7a489d5465 100644 --- a/bin/ripple/ledger/Server.py +++ b/bin/ripple/ledger/Server.py @@ -3,17 +3,21 @@ from __future__ import absolute_import, division, print_function, unicode_litera import json import os -from ripple.ledger import RippledReader, ServerReader +from ripple.ledger import DatabaseReader, RippledReader from ripple.ledger.Args import ARGS from ripple.util.FileCache import FileCache +from ripple.util import ConfigFile +from ripple.util import File from ripple.util import Range class Server(object): def __init__(self): - if ARGS.rippled: - reader = RippledReader.RippledReader() + cfg_file = File.normalize(ARGS.config or 'rippled.cfg') + self.config = ConfigFile.read(open(cfg_file)) + if ARGS.database != ARGS.NONE: + reader = DatabaseReader.DatabaseReader(self.config) else: - reader = ServerReader.ServerReader() + reader = RippledReader.RippledReader(self.config) self.reader = reader self.complete = reader.complete @@ -23,8 +27,7 @@ class Server(object): 'current': reader.name_to_ledger_index('current'), 'validated': reader.name_to_ledger_index('validated'), 'first': self.complete[0] if self.complete else None, - 'last': self.complete[-1] if self.complete else None -, + 'last': self.complete[-1] if self.complete else None, } self.__dict__.update(names) self.ledgers = sorted(Range.join_ranges(*ARGS.ledgers, **names)) diff --git a/bin/ripple/ledger/ServerReader.py b/bin/ripple/ledger/ServerReader.py index 1cc271707c..161560ad6f 100644 --- a/bin/ripple/ledger/ServerReader.py +++ b/bin/ripple/ledger/ServerReader.py @@ -1,5 +1,5 @@ from __future__ import absolute_import, division, print_function, unicode_literals class ServerReader(object): - def __init__(self, server): + def __init__(self, config): raise ValueError('Direct server connections are not yet implemented.') diff --git a/bin/ripple/ledger/commands/Info.py b/bin/ripple/ledger/commands/Info.py index 43cc33ca85..cf1d5685e2 100644 --- a/bin/ripple/ledger/commands/Info.py +++ b/bin/ripple/ledger/commands/Info.py @@ -10,8 +10,8 @@ SAFE = True HELP = 'info - return server_info' def info(server): - Log.out('first = ', server.first) - Log.out('last = ', server.last) + Log.out('first =', server.first) + Log.out('last =', server.last) Log.out('closed =', server.closed) Log.out('current =', server.current) Log.out('validated =', server.validated) diff --git a/bin/ripple/util/ConfigFile.py b/bin/ripple/util/ConfigFile.py new file mode 100644 index 0000000000..2816216593 --- /dev/null +++ b/bin/ripple/util/ConfigFile.py @@ -0,0 +1,54 @@ +from __future__ import absolute_import, division, print_function, unicode_literals + +import json + +"""Ripple has a proprietary format for their .cfg files, so we need a reader for +them.""" + +def read(lines): + sections = [] + section = [] + for line in lines: + line = line.strip() + if (not line) or line[0] == '#': + continue + if line.startswith('['): + if section: + sections.append(section) + section = [] + section.append(line) + if section: + sections.append(section) + + result = {} + for section in sections: + option = section.pop(0) + assert section, ('No value for option "%s".' % option) + assert option.startswith('[') and option.endswith(']'), ( + 'No option name in block "%s"' % p[0]) + option = option[1:-1] + assert option not in result, 'Duplicate option "%s".' % option + + subdict = {} + items = [] + for part in section: + if '=' in part: + assert not items, 'Dictionary mixed with list.' + k, v = part.split('=', 1) + assert k not in subdict, 'Repeated dictionary entry ' + k + subdict[k] = v + else: + assert not subdict, 'List mixed with dictionary.' + if part.startswith('{'): + items.append(json.loads(part)) + else: + words = part.split() + if len(words) > 1: + items.append(words) + else: + items.append(part) + if len(items) == 1: + result[option] = items[0] + else: + result[option] = items or subdict + return result diff --git a/bin/ripple/util/Database.py b/bin/ripple/util/Database.py new file mode 100644 index 0000000000..689b058710 --- /dev/null +++ b/bin/ripple/util/Database.py @@ -0,0 +1,12 @@ +from __future__ import absolute_import, division, print_function, unicode_literals + +import sqlite3 + +def fetchall(database, query, kwds): + conn = sqlite3.connect(database) + try: + cursor = conn.execute(query, kwds) + return cursor.fetchall() + + finally: + conn.close() diff --git a/bin/ripple/util/Function.py b/bin/ripple/util/Function.py index 8e33820210..d6fe864533 100644 --- a/bin/ripple/util/Function.py +++ b/bin/ripple/util/Function.py @@ -20,7 +20,10 @@ REMAPPINGS = { } def eval_arguments(args): - tokens = tokenize.generate_tokens(StringIO(args or '()').readline) + args = args.strip() + if not args or (args == '()'): + return () + tokens = list(tokenize.generate_tokens(StringIO(args).readline)) def remap(): for type, name, _, _, _ in tokens: if type == tokenize.NAME and name not in REMAPPINGS: @@ -30,7 +33,11 @@ def eval_arguments(args): untok = tokenize.untokenize(remap()) if untok[1:-1].strip(): untok = untok[:-1] + ',)' # Force a tuple. - return eval(untok, REMAPPINGS) + try: + return eval(untok, REMAPPINGS) + except Exception as e: + raise ValueError('Couldn\'t evaluate expression "%s" (became "%s"), ' + 'error "%s"' % (args, untok, str(e))) class Function(object): def __init__(self, desc='', default_path=''): diff --git a/bin/ripple/util/test_ConfigFile.py b/bin/ripple/util/test_ConfigFile.py new file mode 100644 index 0000000000..8b6d040505 --- /dev/null +++ b/bin/ripple/util/test_ConfigFile.py @@ -0,0 +1,163 @@ +from __future__ import absolute_import, division, print_function, unicode_literals + +from ripple.util import ConfigFile + +from unittest import TestCase + +class test_ConfigFile(TestCase): + def test_trivial(self): + self.assertEquals(ConfigFile.read(''), {}) + + def test_full(self): + self.assertEquals(ConfigFile.read(FULL.splitlines()), RESULT) + +RESULT = { + 'websocket_port': '6206', + 'database_path': '/development/alpha/db', + 'sntp_servers': + ['time.windows.com', 'time.apple.com', 'time.nist.gov', 'pool.ntp.org'], + 'validation_seed': 'sh1T8T9yGuV7Jb6DPhqSzdU2s5LcV', + 'node_size': 'medium', + 'rpc_startup': { + 'command': 'log_level', + 'severity': 'debug'}, + 'ips': ['r.ripple.com', '51235'], + 'node_db': { + 'file_size_mult': '2', + 'file_size_mb': '8', + 'cache_mb': '256', + 'path': '/development/alpha/db/rocksdb', + 'open_files': '2000', + 'type': 'RocksDB', + 'filter_bits': '12'}, + 'peer_port': '53235', + 'ledger_history': 'full', + 'rpc_ip': '127.0.0.1', + 'websocket_public_ip': '0.0.0.0', + 'rpc_allow_remote': '0', + 'validators': + [['n949f75evCHwgyP4fPVgaHqNHxUVN15PsJEZ3B3HnXPcPjcZAoy7', 'RL1'], + ['n9MD5h24qrQqiyBC8aeqqCWvpiBiYQ3jxSr91uiDvmrkyHRdYLUj', 'RL2'], + ['n9L81uNCaPgtUJfaHh89gmdvXKAmSt5Gdsw2g1iPWaPkAHW5Nm4C', 'RL3'], + ['n9KiYM9CgngLvtRCQHZwgC2gjpdaZcCcbt3VboxiNFcKuwFVujzS', 'RL4'], + ['n9LdgEtkmGB9E2h3K4Vp7iGUaKuq23Zr32ehxiU8FWY7xoxbWTSA', 'RL5']], + 'debug_logfile': '/development/alpha/debug.log', + 'websocket_public_port': '5206', + 'peer_ip': '0.0.0.0', + 'rpc_port': '5205', + 'validation_quorum': '3', + 'websocket_ip': '127.0.0.1'} + +FULL = """ +[ledger_history] +full + +# Allow other peers to connect to this server. +# +[peer_ip] +0.0.0.0 + +[peer_port] +53235 + +# Allow untrusted clients to connect to this server. +# +[websocket_public_ip] +0.0.0.0 + +[websocket_public_port] +5206 + +# Provide trusted websocket ADMIN access to the localhost. +# +[websocket_ip] +127.0.0.1 + +[websocket_port] +6206 + +# Provide trusted json-rpc ADMIN access to the localhost. +# +[rpc_ip] +127.0.0.1 + +[rpc_port] +5205 + +[rpc_allow_remote] +0 + +[node_size] +medium + +# This is primary persistent datastore for rippled. This includes transaction +# metadata, account states, and ledger headers. Helpful information can be +# found here: https://ripple.com/wiki/NodeBackEnd +[node_db] +type=RocksDB +path=/development/alpha/db/rocksdb +open_files=2000 +filter_bits=12 +cache_mb=256 +file_size_mb=8 +file_size_mult=2 + +[database_path] +/development/alpha/db + +# This needs to be an absolute directory reference, not a relative one. +# Modify this value as required. +[debug_logfile] +/development/alpha/debug.log + +[sntp_servers] +time.windows.com +time.apple.com +time.nist.gov +pool.ntp.org + +# Where to find some other servers speaking the Ripple protocol. +# +[ips] +r.ripple.com 51235 + +# The latest validators can be obtained from +# https://ripple.com/ripple.txt +# +[validators] +n949f75evCHwgyP4fPVgaHqNHxUVN15PsJEZ3B3HnXPcPjcZAoy7 RL1 +n9MD5h24qrQqiyBC8aeqqCWvpiBiYQ3jxSr91uiDvmrkyHRdYLUj RL2 +n9L81uNCaPgtUJfaHh89gmdvXKAmSt5Gdsw2g1iPWaPkAHW5Nm4C RL3 +n9KiYM9CgngLvtRCQHZwgC2gjpdaZcCcbt3VboxiNFcKuwFVujzS RL4 +n9LdgEtkmGB9E2h3K4Vp7iGUaKuq23Zr32ehxiU8FWY7xoxbWTSA RL5 + +# Ditto. +[validation_quorum] +3 + +[validation_seed] +sh1T8T9yGuV7Jb6DPhqSzdU2s5LcV + +# Turn down default logging to save disk space in the long run. +# Valid values here are trace, debug, info, warning, error, and fatal +[rpc_startup] +{ "command": "log_level", "severity": "debug" } + +# Configure SSL for WebSockets. Not enabled by default because not everybody +# has an SSL cert on their server, but if you uncomment the following lines and +# set the path to the SSL certificate and private key the WebSockets protocol +# will be protected by SSL/TLS. +#[websocket_secure] +#1 + +#[websocket_ssl_cert] +#/etc/ssl/certs/server.crt + +#[websocket_ssl_key] +#/etc/ssl/private/server.key + +# Defaults to 0 ("no") so that you can use self-signed SSL certificates for +# development, or internally. +#[ssl_verify] +#0 +""".strip()