From 9160b46c1edee49e3b60d3afc936870c3e2e9fc0 Mon Sep 17 00:00:00 2001 From: Tom Ritchford Date: Tue, 26 Aug 2014 10:33:35 -0400 Subject: [PATCH] Bug fixes and new features for LedgerTool: * Fix RIPD-509, RIPD-514, RIPD-519, RIPD-525, RIPD-527, RIPD-529, RIPD-530 and RIPD-531. * Protect people from ledger-spew and remove cruft. * Better error messages and handling. * Cache command lists or clears ledger cache. * Better ledger summaries. * Offline mode. --- bin/{lt => LT} | 0 bin/LedgerTool.py | 20 ++++++--- bin/ripple/ledger/Args.py | 30 +++++++++---- bin/ripple/ledger/Log.py | 19 --------- bin/ripple/ledger/RippledReader.py | 17 +++++++- bin/ripple/ledger/SearchLedgers.py | 2 +- bin/ripple/ledger/Server.py | 23 ++++++---- bin/ripple/ledger/commands/Cache.py | 34 +++++++++++++++ bin/ripple/ledger/commands/Info.py | 4 +- bin/ripple/ledger/displays/__init__.py | 58 +++++++++++++++++--------- bin/ripple/util/FileCache.py | 57 +++++++++++++++++-------- bin/ripple/util/Function.py | 48 ++++++++++++++++----- bin/ripple/util/Log.py | 43 +++++++------------ bin/ripple/util/Range.py | 2 +- bin/ripple/util/test_Function.py | 12 ++++++ 15 files changed, 250 insertions(+), 119 deletions(-) rename bin/{lt => LT} (100%) delete mode 100644 bin/ripple/ledger/Log.py create mode 100644 bin/ripple/ledger/commands/Cache.py diff --git a/bin/lt b/bin/LT similarity index 100% rename from bin/lt rename to bin/LT diff --git a/bin/LedgerTool.py b/bin/LedgerTool.py index 44cbffdaec..4cc79e4d26 100755 --- a/bin/LedgerTool.py +++ b/bin/LedgerTool.py @@ -2,15 +2,23 @@ from __future__ import absolute_import, division, print_function, unicode_literals +import sys +import traceback + from ripple.ledger import Server +from ripple.ledger.commands import Cache, Info, Print from ripple.ledger.Args import ARGS +from ripple.util import Log from ripple.util.CommandList import CommandList -from ripple.ledger.commands import Info, Print - -_COMMANDS = CommandList(Info, Print) +_COMMANDS = CommandList(Cache, Info, Print) if __name__ == '__main__': - server = Server.Server() - args = list(ARGS.command) - _COMMANDS.run_safe(args.pop(0), server, *args) + try: + server = Server.Server() + args = list(ARGS.command) + _COMMANDS.run_safe(args.pop(0), server, *args) + except Exception as e: + if ARGS.verbose: + print(traceback.format_exc(), sys.stderr) + Log.error(e) diff --git a/bin/ripple/ledger/Args.py b/bin/ripple/ledger/Args.py index 9f078123ec..29c27add58 100644 --- a/bin/ripple/ledger/Args.py +++ b/bin/ripple/ledger/Args.py @@ -6,8 +6,9 @@ import os from ripple.ledger import LedgerNumber from ripple.util import File -from ripple.util.Function import Function +from ripple.util import Log from ripple.util import Range +from ripple.util.Function import Function NAME = 'LedgerTool' VERSION = '0.1' @@ -26,6 +27,12 @@ _parser.add_argument( ) # Flag arguments. +_parser.add_argument( + '--binary', + action='store_true', + help='If true, searches are binary - by default linear search is used.', + ) + _parser.add_argument( '--cache', default='~/.local/share/ripple/ledger', @@ -40,7 +47,6 @@ _parser.add_argument( _parser.add_argument( '--condition', '-c', - default='all_ledgers', help='The name of a condition function used to match ledgers.', ) @@ -51,7 +57,6 @@ _parser.add_argument( _parser.add_argument( '--display', '-d', - default='ledger_number', help='Specify a function to display ledgers.', ) @@ -69,9 +74,9 @@ _parser.add_argument( ) _parser.add_argument( - '--binary', + '--offline', '-o', action='store_true', - help='If true, searches are binary - by default linear search is used.', + help='If true, work entirely from cache, do not try to contact the server.', ) _parser.add_argument( @@ -126,6 +131,8 @@ _parser.add_argument( # Read the arguments from the command line. ARGS = _parser.parse_args() +Log.VERBOSE = ARGS.verbose + # Now remove any items that look like ledger numbers from the command line. _command = ARGS.command _parts = (ARGS.command, ARGS.ledgers) = ([], []) @@ -136,8 +143,17 @@ for c in _command: ARGS.command = ARGS.command or ['print' if ARGS.ledgers else 'info'] ARGS.cache = File.normalize(ARGS.cache) -ARGS.condition = Function(ARGS.condition, 'ripple.ledger.conditions') -ARGS.display = Function(ARGS.display, 'ripple.ledger.displays') + +if not ARGS.ledgers: + if ARGS.condition: + Log.warn('--condition needs a range of ledgers') + if ARGS.display: + Log.warn('--display needs a range of ledgers') + +ARGS.condition = Function( + ARGS.condition or 'all_ledgers', 'ripple.ledger.conditions') +ARGS.display = Function( + ARGS.display or 'ledger_number', 'ripple.ledger.displays') if ARGS.window < 0: raise ValueError('Window cannot be negative: --window=%d' % diff --git a/bin/ripple/ledger/Log.py b/bin/ripple/ledger/Log.py deleted file mode 100644 index 866afb2c0b..0000000000 --- a/bin/ripple/ledger/Log.py +++ /dev/null @@ -1,19 +0,0 @@ -from __future__ import absolute_import, division, print_function, unicode_literals - -import sys - -from ripple.ledger.Args import ARGS - -def out(*args, **kwds): - kwds.get('print', print)(*args, file=sys.stdout, **kwds) - -def info(*args, **kwds): - if ARGS.verbose: - out(*args, **kwds) - -def error(*args, **kwds): - out('ERROR', *args, **kwds) - -def fatal(*args, **kwds): - out('FATAL', *args, **kwds) - raise Exception('FATAL: ' + ' '.join(str(a) for a in args)) diff --git a/bin/ripple/ledger/RippledReader.py b/bin/ripple/ledger/RippledReader.py index 01cbb55db9..b79df532d6 100644 --- a/bin/ripple/ledger/RippledReader.py +++ b/bin/ripple/ledger/RippledReader.py @@ -6,12 +6,19 @@ import subprocess from ripple.ledger.Args import ARGS from ripple.util import File +from ripple.util import Log from ripple.util import Range _ERROR_CODE_REASON = { 62: 'No rippled server is running.', } +_ERROR_TEXT = { + 'lgrNotFound': 'The ledger you requested was not found.', + 'noCurrent': 'The server has no current ledger.', + 'noNetwork': 'The server did not respond to your request.', +} + _DEFAULT_ERROR_ = "Couldn't connect to server." class RippledReader(object): @@ -36,7 +43,15 @@ class RippledReader(object): cmd = ['ledger', str(name)] if is_full: cmd.append('full') - return self._command(*cmd)['ledger'] + response = self._command(*cmd) + result = response.get('ledger') + if result: + return result + error = response['error'] + etext = _ERROR_TEXT.get(error) + if etext: + error = '%s (%s)' % (etext, error) + Log.fatal(_ERROR_TEXT.get(error, error)) def _command(self, *cmds): cmd = self.cmd + list(cmds) diff --git a/bin/ripple/ledger/SearchLedgers.py b/bin/ripple/ledger/SearchLedgers.py index 48f5de5b2f..8200366281 100644 --- a/bin/ripple/ledger/SearchLedgers.py +++ b/bin/ripple/ledger/SearchLedgers.py @@ -3,7 +3,7 @@ from __future__ import absolute_import, division, print_function, unicode_litera import sys from ripple.ledger.Args import ARGS -from ripple.ledger import Log +from ripple.util import Log from ripple.util import Range from ripple.util import Search diff --git a/bin/ripple/ledger/Server.py b/bin/ripple/ledger/Server.py index 8c1adca0f7..57966e3379 100644 --- a/bin/ripple/ledger/Server.py +++ b/bin/ripple/ledger/Server.py @@ -5,7 +5,7 @@ import os from ripple.ledger import RippledReader, ServerReader from ripple.ledger.Args import ARGS -from ripple.util.FileCache import file_cache +from ripple.util.FileCache import FileCache from ripple.util import Range class Server(object): @@ -16,15 +16,15 @@ class Server(object): reader = ServerReader.ServerReader() self.reader = reader - self.complete = reader.complete names = { 'closed': reader.name_to_ledger_index('closed'), 'current': reader.name_to_ledger_index('current'), 'validated': reader.name_to_ledger_index('validated'), - 'first': self.complete[0], - 'last': self.complete[-1], + 'first': self.complete[0] if self.complete else None, + 'last': self.complete[-1] if self.complete else None +, } self.__dict__.update(names) self.ledgers = sorted(Range.join_ranges(*ARGS.ledgers, **names)) @@ -33,11 +33,20 @@ class Server(object): name = 'full' if is_full else 'summary' filepath = os.path.join(ARGS.cache, name) creator = lambda n: reader.get_ledger(n, is_full) - return file_cache(filepath, creator) - self.caches = [make_cache(False), make_cache(True)] + return FileCache(filepath, creator) + self._caches = [make_cache(False), make_cache(True)] def info(self): return self.reader.info + def cache(self, is_full): + return self._caches[is_full] + def get_ledger(self, number, is_full=False): - return self.caches[is_full](number, int(number) in self.complete) + num = int(number) + save_in_cache = num in self.complete + can_create = (not ARGS.offline and + self.complete and + self.complete[0] <= num - 1) + cache = self.cache(is_full) + return cache.get_data(number, save_in_cache, can_create) diff --git a/bin/ripple/ledger/commands/Cache.py b/bin/ripple/ledger/commands/Cache.py new file mode 100644 index 0000000000..9dd6756202 --- /dev/null +++ b/bin/ripple/ledger/commands/Cache.py @@ -0,0 +1,34 @@ +from __future__ import absolute_import, division, print_function, unicode_literals + +from ripple.ledger.Args import ARGS +from ripple.ledger.PrettyPrint import pretty_print +from ripple.util import Log +from ripple.util import Range + +SAFE = True + +HELP = """cache +return server_info""" + +def cache(server, clear=False): + cache = server.cache(ARGS.full) + name = ['summary', 'full'][ARGS.full] + files = cache.file_count() + if not files: + Log.error('No files in %s cache.' % name) + + elif clear: + if not clear.strip() == 'clear': + raise Exception("Don't understand 'clear %s'." % clear) + if not ARGS.yes: + yes = raw_input('OK to clear %s cache? (y/N) ' % name) + if not yes.lower().startswith('y'): + Log.out('Cancelled.') + return + cache.clear(ARGS.full) + Log.out('%s cache cleared - %d file%s deleted.' % + (name.capitalize(), files, '' if files == 1 else 's')) + + else: + caches = (int(c) for c in cache.cache_list()) + Log.out(Range.to_string(caches)) diff --git a/bin/ripple/ledger/commands/Info.py b/bin/ripple/ledger/commands/Info.py index fb7e7dd079..76bc712b28 100644 --- a/bin/ripple/ledger/commands/Info.py +++ b/bin/ripple/ledger/commands/Info.py @@ -1,8 +1,8 @@ from __future__ import absolute_import, division, print_function, unicode_literals from ripple.ledger.Args import ARGS -from ripple.ledger import Log from ripple.ledger.PrettyPrint import pretty_print +from ripple.util import Log from ripple.util import Range SAFE = True @@ -10,6 +10,8 @@ SAFE = True HELP = 'info - return server_info' def info(server): + Log.out('first = ', server.first) + Log.out('last = ', server.last) Log.out('closed =', server.closed) Log.out('current =', server.current) Log.out('validated =', server.validated) diff --git a/bin/ripple/ledger/displays/__init__.py b/bin/ripple/ledger/displays/__init__.py index 362ec06cf4..852a794468 100644 --- a/bin/ripple/ledger/displays/__init__.py +++ b/bin/ripple/ledger/displays/__init__.py @@ -6,9 +6,32 @@ import jsonpath_rw from ripple.ledger.Args import ARGS from ripple.ledger.PrettyPrint import pretty_print -from ripple.util.Decimal import Decimal from ripple.util import Dict +from ripple.util import Log from ripple.util import Range +from ripple.util.Decimal import Decimal + +TRANSACT_FIELDS = ( + 'accepted', + 'close_time_human', + 'closed', + 'ledger_index', + 'total_coins', + 'transactions', +) + +LEDGER_FIELDS = ( + 'accepted', + 'accountState', + 'close_time_human', + 'closed', + 'ledger_index', + 'total_coins', + 'transactions', +) + +def _dict_filter(d, keys): + return dict((k, v) for (k, v) in d.items() if k in keys) def ledger_number(server, numbers): yield Range.to_string(numbers) @@ -19,7 +42,8 @@ def display(f): def wrapper(server, numbers, *args, **kwds): for number in numbers: ledger = server.get_ledger(number, ARGS.full) - yield pretty_print(f(ledger, *args, **kwds)) + if ledger: + yield pretty_print(f(ledger, *args, **kwds)) return wrapper def json(f): @@ -33,31 +57,27 @@ def json(f): for number in numbers: ledger = server.get_ledger(number, ARGS.full) - finds = path_expr.find(ledger) - yield pretty_print(f(finds, *args, **kwds)) + if ledger: + finds = path_expr.find(ledger) + yield pretty_print(f(finds, *args, **kwds)) return wrapper +@display +def ledger(ledger, full=False): + if ARGS.full: + if full: + return ledger + ledger = Dict.prune(ledger, 1, False) + + return _dict_filter(ledger, LEDGER_FIELDS) @display -def ledger(led): - return led - -@display -def prune(ledger, level=2): +def prune(ledger, level=1): return Dict.prune(ledger, level, False) -TRANSACT_FIELDS = ( - 'accepted', - 'close_time_human', - 'closed', - 'ledger_index', - 'total_coins', - 'transactions', -) - @display def transact(ledger): - return dict((f, ledger[f]) for f in TRANSACT_FIELDS) + return _dict_filter(ledger, TRANSACT_FIELDS) @json def extract(finds): diff --git a/bin/ripple/util/FileCache.py b/bin/ripple/util/FileCache.py index d18d4e7b15..259e42020f 100644 --- a/bin/ripple/util/FileCache.py +++ b/bin/ripple/util/FileCache.py @@ -1,33 +1,56 @@ from __future__ import absolute_import, division, print_function, unicode_literals -import json import gzip +import json import os _NONE = object() -def file_cache(filename_prefix, creator, open=gzip.open, suffix='.gz'): +class FileCache(object): """A two-level cache, which stores expensive results in memory and on disk. """ - cached_data = {} - if not os.path.exists(filename_prefix): - os.makedirs(filename_prefix) + def __init__(self, cache_directory, creator, open=gzip.open, suffix='.gz'): + self.cache_directory = cache_directory + self.creator = creator + self.open = open + self.suffix = suffix + self.cached_data = {} + if not os.path.exists(self.cache_directory): + os.makedirs(self.cache_directory) - def get_file_data(name): - filename = os.path.join(filename_prefix, str(name)) + suffix + def get_file_data(self, name): if os.path.exists(filename): - return json.load(open(filename)) + return json.load(self.open(filename)) - result = creator(name) - json.dump(result, open(filename, 'w')) + result = self.creator(name) return result - def get_data(name, use_file_cache=True): - result = cached_data.get(name, _NONE) + def get_data(self, name, save_in_cache, can_create, default=None): + name = str(name) + result = self.cached_data.get(name, _NONE) if result is _NONE: - maker = get_file_data if use_file_cache else creator - result = maker(name) - cached_data[name] = result - return result + filename = os.path.join(self.cache_directory, name) + self.suffix + if os.path.exists(filename): + result = json.load(self.open(filename)) or _NONE + if result is _NONE and can_create: + result = self.creator(name) + if save_in_cache: + json.dump(result, self.open(filename, 'w')) + return default if result is _NONE else result - return get_data + def _files(self): + return os.listdir(self.cache_directory) + + def cache_list(self): + for f in self._files(): + if f.endswith(self.suffix): + yield f[:-len(self.suffix)] + + def file_count(self): + return len(self._files()) + + def clear(self): + """Clears both local files and memory.""" + self.cached_data = {} + for f in self._files(): + os.remove(os.path.join(self.cache_directory, f)) diff --git a/bin/ripple/util/Function.py b/bin/ripple/util/Function.py index 4e168732c0..73fd75cd34 100644 --- a/bin/ripple/util/Function.py +++ b/bin/ripple/util/Function.py @@ -4,18 +4,33 @@ from __future__ import absolute_import, division, print_function, unicode_litera import importlib import re +import tokenize + +from StringIO import StringIO MATCHER = re.compile(r'([\w.]+)(.*)') -def _split_function(desc): - m = MATCHER.match(desc) - if not m: - raise ValueError('"%s" is not a function' % desc) - name, args = (g.strip() for g in m.groups()) - args = eval(args or '()') # Yes, really eval()! - if not isinstance(args, tuple): - args = (args,) - return name, args +REMAPPINGS = { + 'false': False, + 'true': True, + 'null': None, + 'False': False, + 'True': True, + 'None': None, +} + +def eval_arguments(args): + tokens = tokenize.generate_tokens(StringIO(args or '()').readline) + def remap(): + for type, name, _, _, _ in tokens: + if type == tokenize.NAME and name not in REMAPPINGS: + yield tokenize.STRING, '"%s"' % name + else: + yield type, name + untok = tokenize.untokenize(remap()) + if untok[1:-1].strip(): + untok = untok[:-1] + ',)' # Force a tuple. + return eval(untok, REMAPPINGS) class Function(object): def __init__(self, desc='', default_path=''): @@ -26,7 +41,12 @@ class Function(object): self.function = lambda *args, **kwds: None return - self.function, self.args = _split_function(self.desc) + m = MATCHER.match(desc) + if not m: + raise ValueError('"%s" is not a function' % desc) + self.function, self.args = (g.strip() for g in m.groups()) + self.args = eval_arguments(self.args) + if '.' not in self.function: if default_path and not default_path.endswith('.'): default_path += '.' @@ -40,10 +60,16 @@ class Function(object): try: self.function = getattr(mod, m) except: - raise ValueError('No function "%s" in module "%s"' % (p, m)) + raise ValueError('No function "%s" in module "%s"' % (m, p)) def __str__(self): return self.desc def __call__(self, *args, **kwds): return self.function(*(args + self.args), **kwds) + + def __eq__(self, other): + return self.function == other.function and self.args == other.args + + def __ne__(self, other): + return not (self == other) diff --git a/bin/ripple/util/Log.py b/bin/ripple/util/Log.py index 8cb2209e3c..8283f709d2 100644 --- a/bin/ripple/util/Log.py +++ b/bin/ripple/util/Log.py @@ -1,36 +1,21 @@ from __future__ import absolute_import, division, print_function, unicode_literals -def count_all_subitems(x): - """Count the subitems of a Python object, including the object itself.""" - if isinstance(x, list): - return 1 + sum(count_all_subitems(i) for i in x) - if isinstance(x, dict): - return 1 + sum(count_all_subitems(i) for i in x.values()) - return 1 +import sys -def prune(item, level, count_recursively=True): - def subitems(x): - if count_recursively: - i = count_all_subitems(x) - 1 - else: - i = len(x) - return '1 subitem' if (i == 1) else '%d subitems' % i +VERBOSE = False - assert level >= 0 - if not item: - return item +def out(*args, **kwds): + kwds.get('print', print)(*args, file=sys.stdout, **kwds) - if isinstance(item, list): - if level: - return [prune(i, level - 1, count_recursively) for i in item] - else: - return '[list with %s]' % subitems(item) +def info(*args, **kwds): + if VERBOSE: + out(*args, **kwds) - if isinstance(item, dict): - if level: - return dict((k, prune(v, level - 1, count_recursively)) - for k, v in item.iteritems()) - else: - return '{dict with %s}' % subitems(item) +def warn(*args, **kwds): + out('WARNING:', *args, **kwds) - return item +def error(*args, **kwds): + out('ERROR:', *args, **kwds) + +def fatal(*args, **kwds): + raise Exception('FATAL: ' + ' '.join(str(a) for a in args)) diff --git a/bin/ripple/util/Range.py b/bin/ripple/util/Range.py index 4fae61f3b8..b3ea709679 100644 --- a/bin/ripple/util/Range.py +++ b/bin/ripple/util/Range.py @@ -13,7 +13,7 @@ def from_string(desc, **aliases): return [] result = set() for d in desc.split(','): - nums = [aliases.get(x, None) or int(x) for x in d.split('-')] + nums = [int(aliases.get(x) or x) for x in d.split('-')] if len(nums) == 1: result.add(nums[0]) elif len(nums) == 2: diff --git a/bin/ripple/util/test_Function.py b/bin/ripple/util/test_Function.py index 73769d4dac..9da880a7ef 100644 --- a/bin/ripple/util/test_Function.py +++ b/bin/ripple/util/test_Function.py @@ -18,8 +18,20 @@ class test_Function(TestCase): def test_empty_function(self): self.assertEquals(Function()(), None) + def test_empty_args(self): + f = Function('ripple.util.test_Function.FN()') + self.assertEquals(f(), ((), {})) + def test_function(self): f = Function('ripple.util.test_Function.FN(True, {1: 2}, None)') self.assertEquals(f(), ((True, {1: 2}, None), {})) self.assertEquals(f('hello', foo='bar'), (('hello', True, {1: 2}, None), {'foo':'bar'})) + self.assertEquals( + f, Function('ripple.util.test_Function.FN(true, {1: 2}, null)')) + + def test_quoting(self): + f = Function('ripple.util.test_Function.FN(testing)') + self.assertEquals(f(), (('testing',), {})) + f = Function('ripple.util.test_Function.FN(testing, true, false, null)') + self.assertEquals(f(), (('testing', True, False, None), {}))