Bug fixes and new features for LedgerTool:

* Fix RIPD-509, RIPD-514, RIPD-519, RIPD-525, RIPD-527, RIPD-529,
  RIPD-530 and RIPD-531.
* Protect people from ledger-spew and remove cruft.
* Better error messages and handling.
* Cache command lists or clears ledger cache.
* Better ledger summaries.
* Offline mode.

bin/LedgerTool.py

@@ -2,15 +2,23 @@
 
 from __future__ import absolute_import, division, print_function, unicode_literals
 
+import sys
+import traceback
+
 from ripple.ledger import Server
+from ripple.ledger.commands import Cache, Info, Print
 from ripple.ledger.Args import ARGS
+from ripple.util import Log
 from ripple.util.CommandList import CommandList
 
-from ripple.ledger.commands import Info, Print
-
-_COMMANDS = CommandList(Info, Print)
+_COMMANDS = CommandList(Cache, Info, Print)
 
 if __name__ == '__main__':
-    server = Server.Server()
-    args = list(ARGS.command)
-    _COMMANDS.run_safe(args.pop(0), server, *args)
+    try:
+        server = Server.Server()
+        args = list(ARGS.command)
+        _COMMANDS.run_safe(args.pop(0), server, *args)
+    except Exception as e:
+        if ARGS.verbose:
+            print(traceback.format_exc(), sys.stderr)
+        Log.error(e)

bin/ripple/ledger/Args.py

@@ -6,8 +6,9 @@ import os
 
 from ripple.ledger import LedgerNumber
 from ripple.util import File
-from ripple.util.Function import Function
+from ripple.util import Log
 from ripple.util import Range
+from ripple.util.Function import Function
 
 NAME = 'LedgerTool'
 VERSION = '0.1'
@@ -26,6 +27,12 @@ _parser.add_argument(
 )
 
 # Flag arguments.
+_parser.add_argument(
+    '--binary',
+    action='store_true',
+    help='If true, searches are binary - by default linear search is used.',
+    )
+
 _parser.add_argument(
     '--cache',
     default='~/.local/share/ripple/ledger',
@@ -40,7 +47,6 @@ _parser.add_argument(
 
 _parser.add_argument(
     '--condition', '-c',
-    default='all_ledgers',
     help='The name of a condition function used to match ledgers.',
     )
 
@@ -51,7 +57,6 @@ _parser.add_argument(
 
 _parser.add_argument(
     '--display', '-d',
-    default='ledger_number',
     help='Specify a function to display ledgers.',
     )
 
@@ -69,9 +74,9 @@ _parser.add_argument(
     )
 
 _parser.add_argument(
-    '--binary',
+    '--offline', '-o',
     action='store_true',
-    help='If true, searches are binary - by default linear search is used.',
+    help='If true, work entirely from cache, do not try to contact the server.',
     )
 
 _parser.add_argument(
@@ -126,6 +131,8 @@ _parser.add_argument(
 # Read the arguments from the command line.
 ARGS = _parser.parse_args()
 
+Log.VERBOSE = ARGS.verbose
+
 # Now remove any items that look like ledger numbers from the command line.
 _command = ARGS.command
 _parts = (ARGS.command, ARGS.ledgers) = ([], [])
@@ -136,8 +143,17 @@ for c in _command:
 ARGS.command = ARGS.command or ['print' if ARGS.ledgers else 'info']
 
 ARGS.cache = File.normalize(ARGS.cache)
-ARGS.condition = Function(ARGS.condition, 'ripple.ledger.conditions')
-ARGS.display = Function(ARGS.display, 'ripple.ledger.displays')
+
+if not ARGS.ledgers:
+    if ARGS.condition:
+        Log.warn('--condition needs a range of ledgers')
+    if ARGS.display:
+        Log.warn('--display needs a range of ledgers')
+
+ARGS.condition = Function(
+    ARGS.condition or 'all_ledgers', 'ripple.ledger.conditions')
+ARGS.display = Function(
+    ARGS.display or 'ledger_number', 'ripple.ledger.displays')
 
 if ARGS.window < 0:
     raise ValueError('Window cannot be negative: --window=%d' %

bin/ripple/ledger/Log.py

@@ -1,19 +0,0 @@
-from __future__ import absolute_import, division, print_function, unicode_literals
-
-import sys
-
-from ripple.ledger.Args import ARGS
-
-def out(*args, **kwds):
-    kwds.get('print', print)(*args, file=sys.stdout, **kwds)
-
-def info(*args, **kwds):
-    if ARGS.verbose:
-        out(*args, **kwds)
-
-def error(*args, **kwds):
-    out('ERROR', *args, **kwds)
-
-def fatal(*args, **kwds):
-    out('FATAL', *args, **kwds)
-    raise Exception('FATAL: ' + ' '.join(str(a) for a in args))

bin/ripple/ledger/RippledReader.py

@@ -6,12 +6,19 @@ import subprocess
 
 from ripple.ledger.Args import ARGS
 from ripple.util import File
+from ripple.util import Log
 from ripple.util import Range
 
 _ERROR_CODE_REASON = {
     62: 'No rippled server is running.',
 }
 
+_ERROR_TEXT = {
+    'lgrNotFound': 'The ledger you requested was not found.',
+    'noCurrent': 'The server has no current ledger.',
+    'noNetwork': 'The server did not respond to your request.',
+}
+
 _DEFAULT_ERROR_ = "Couldn't connect to server."
 
 class RippledReader(object):
@@ -36,7 +43,15 @@ class RippledReader(object):
         cmd = ['ledger', str(name)]
         if is_full:
             cmd.append('full')
-        return self._command(*cmd)['ledger']
+        response = self._command(*cmd)
+        result = response.get('ledger')
+        if result:
+            return result
+        error = response['error']
+        etext = _ERROR_TEXT.get(error)
+        if etext:
+            error = '%s (%s)' % (etext, error)
+        Log.fatal(_ERROR_TEXT.get(error, error))
 
     def _command(self, *cmds):
         cmd = self.cmd + list(cmds)

bin/ripple/ledger/SearchLedgers.py

@@ -3,7 +3,7 @@ from __future__ import absolute_import, division, print_function, unicode_litera
 import sys
 
 from ripple.ledger.Args import ARGS
-from ripple.ledger import Log
+from ripple.util import Log
 from ripple.util import Range
 from ripple.util import Search
 

bin/ripple/ledger/Server.py

@@ -5,7 +5,7 @@ import os
 
 from ripple.ledger import RippledReader, ServerReader
 from ripple.ledger.Args import ARGS
-from ripple.util.FileCache import file_cache
+from ripple.util.FileCache import FileCache
 from ripple.util import Range
 
 class Server(object):
@@ -16,15 +16,15 @@ class Server(object):
             reader = ServerReader.ServerReader()
 
         self.reader = reader
-
         self.complete = reader.complete
 
         names = {
             'closed': reader.name_to_ledger_index('closed'),
             'current': reader.name_to_ledger_index('current'),
             'validated': reader.name_to_ledger_index('validated'),
-            'first': self.complete[0],
-            'last': self.complete[-1],
+            'first': self.complete[0] if self.complete else None,
+            'last': self.complete[-1] if self.complete else None
+,
         }
         self.__dict__.update(names)
         self.ledgers = sorted(Range.join_ranges(*ARGS.ledgers, **names))
@@ -33,11 +33,20 @@ class Server(object):
             name = 'full' if is_full else 'summary'
             filepath = os.path.join(ARGS.cache, name)
             creator = lambda n: reader.get_ledger(n, is_full)
-            return file_cache(filepath, creator)
-        self.caches = [make_cache(False), make_cache(True)]
+            return FileCache(filepath, creator)
+        self._caches = [make_cache(False), make_cache(True)]
 
     def info(self):
         return self.reader.info
 
+    def cache(self, is_full):
+        return self._caches[is_full]
+
     def get_ledger(self, number, is_full=False):
-        return self.caches[is_full](number, int(number) in self.complete)
+        num = int(number)
+        save_in_cache = num in self.complete
+        can_create = (not ARGS.offline and
+                      self.complete and
+                      self.complete[0] <= num - 1)
+        cache = self.cache(is_full)
+        return cache.get_data(number, save_in_cache, can_create)

bin/ripple/ledger/commands/Cache.py

@@ -0,0 +1,34 @@
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from ripple.ledger.Args import ARGS
+from ripple.ledger.PrettyPrint import pretty_print
+from ripple.util import Log
+from ripple.util import Range
+
+SAFE = True
+
+HELP = """cache
+return server_info"""
+
+def cache(server, clear=False):
+    cache = server.cache(ARGS.full)
+    name = ['summary', 'full'][ARGS.full]
+    files = cache.file_count()
+    if not files:
+        Log.error('No files in %s cache.' % name)
+
+    elif clear:
+        if not clear.strip() == 'clear':
+            raise Exception("Don't understand 'clear %s'." % clear)
+        if not ARGS.yes:
+            yes = raw_input('OK to clear %s cache? (y/N) ' % name)
+            if not yes.lower().startswith('y'):
+                Log.out('Cancelled.')
+                return
+        cache.clear(ARGS.full)
+        Log.out('%s cache cleared - %d file%s deleted.' %
+                (name.capitalize(), files, '' if files == 1 else 's'))
+
+    else:
+        caches = (int(c) for c in cache.cache_list())
+        Log.out(Range.to_string(caches))

bin/ripple/ledger/commands/Info.py

@@ -1,8 +1,8 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 from ripple.ledger.Args import ARGS
-from ripple.ledger import Log
 from ripple.ledger.PrettyPrint import pretty_print
+from ripple.util import Log
 from ripple.util import Range
 
 SAFE = True
@@ -10,6 +10,8 @@ SAFE = True
 HELP = 'info - return server_info'
 
 def info(server):
+    Log.out('first = ', server.first)
+    Log.out('last = ', server.last)
     Log.out('closed =', server.closed)
     Log.out('current =', server.current)
     Log.out('validated =', server.validated)

bin/ripple/ledger/displays/__init__.py

@@ -6,9 +6,32 @@ import jsonpath_rw
 
 from ripple.ledger.Args import ARGS
 from ripple.ledger.PrettyPrint import pretty_print
-from ripple.util.Decimal import Decimal
 from ripple.util import Dict
+from ripple.util import Log
 from ripple.util import Range
+from ripple.util.Decimal import Decimal
+
+TRANSACT_FIELDS = (
+    'accepted',
+    'close_time_human',
+    'closed',
+    'ledger_index',
+    'total_coins',
+    'transactions',
+)
+
+LEDGER_FIELDS = (
+    'accepted',
+    'accountState',
+    'close_time_human',
+    'closed',
+    'ledger_index',
+    'total_coins',
+    'transactions',
+)
+
+def _dict_filter(d, keys):
+    return dict((k, v) for (k, v) in d.items() if k in keys)
 
 def ledger_number(server, numbers):
     yield Range.to_string(numbers)
@@ -19,7 +42,8 @@ def display(f):
     def wrapper(server, numbers, *args, **kwds):
         for number in numbers:
             ledger = server.get_ledger(number, ARGS.full)
-            yield pretty_print(f(ledger, *args, **kwds))
+            if ledger:
+                yield pretty_print(f(ledger, *args, **kwds))
     return wrapper
 
 def json(f):
@@ -33,31 +57,27 @@ def json(f):
 
         for number in numbers:
             ledger = server.get_ledger(number, ARGS.full)
-            finds = path_expr.find(ledger)
-            yield pretty_print(f(finds, *args, **kwds))
+            if ledger:
+                finds = path_expr.find(ledger)
+                yield pretty_print(f(finds, *args, **kwds))
     return wrapper
 
+@display
+def ledger(ledger, full=False):
+    if ARGS.full:
+        if full:
+            return ledger
+        ledger = Dict.prune(ledger, 1, False)
+
+    return _dict_filter(ledger, LEDGER_FIELDS)
 
 @display
-def ledger(led):
-    return led
-
-@display
-def prune(ledger, level=2):
+def prune(ledger, level=1):
     return Dict.prune(ledger, level, False)
 
-TRANSACT_FIELDS = (
-    'accepted',
-    'close_time_human',
-    'closed',
-    'ledger_index',
-    'total_coins',
-    'transactions',
-)
-
 @display
 def transact(ledger):
-    return dict((f, ledger[f]) for f in TRANSACT_FIELDS)
+    return _dict_filter(ledger, TRANSACT_FIELDS)
 
 @json
 def extract(finds):

bin/ripple/util/FileCache.py

@@ -1,33 +1,56 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-import json
 import gzip
+import json
 import os
 
 _NONE = object()
 
-def file_cache(filename_prefix, creator, open=gzip.open, suffix='.gz'):
+class FileCache(object):
     """A two-level cache, which stores expensive results in memory and on disk.
     """
-    cached_data = {}
-    if not os.path.exists(filename_prefix):
-        os.makedirs(filename_prefix)
+    def __init__(self, cache_directory, creator, open=gzip.open, suffix='.gz'):
+        self.cache_directory = cache_directory
+        self.creator = creator
+        self.open = open
+        self.suffix = suffix
+        self.cached_data = {}
+        if not os.path.exists(self.cache_directory):
+            os.makedirs(self.cache_directory)
 
-    def get_file_data(name):
-        filename = os.path.join(filename_prefix, str(name)) + suffix
+    def get_file_data(self, name):
         if os.path.exists(filename):
-            return json.load(open(filename))
+            return json.load(self.open(filename))
 
-        result = creator(name)
-        json.dump(result, open(filename, 'w'))
+        result = self.creator(name)
         return result
 
-    def get_data(name, use_file_cache=True):
-        result = cached_data.get(name, _NONE)
+    def get_data(self, name, save_in_cache, can_create, default=None):
+        name = str(name)
+        result = self.cached_data.get(name, _NONE)
         if result is _NONE:
-            maker = get_file_data if use_file_cache else creator
-            result = maker(name)
-            cached_data[name] = result
-        return result
+            filename = os.path.join(self.cache_directory, name) + self.suffix
+            if os.path.exists(filename):
+                result = json.load(self.open(filename)) or _NONE
+            if result is _NONE and can_create:
+                result = self.creator(name)
+                if save_in_cache:
+                    json.dump(result, self.open(filename, 'w'))
+        return default if result is _NONE else result
 
-    return get_data
+    def _files(self):
+        return os.listdir(self.cache_directory)
+
+    def cache_list(self):
+        for f in self._files():
+            if f.endswith(self.suffix):
+                yield f[:-len(self.suffix)]
+
+    def file_count(self):
+        return len(self._files())
+
+    def clear(self):
+        """Clears both local files and memory."""
+        self.cached_data = {}
+        for f in self._files():
+            os.remove(os.path.join(self.cache_directory, f))

bin/ripple/util/Function.py

@@ -4,18 +4,33 @@ from __future__ import absolute_import, division, print_function, unicode_litera
 
 import importlib
 import re
+import tokenize
+
+from StringIO import StringIO
 
 MATCHER = re.compile(r'([\w.]+)(.*)')
 
-def _split_function(desc):
-    m = MATCHER.match(desc)
-    if not m:
-        raise ValueError('"%s" is not a function' % desc)
-    name, args = (g.strip() for g in m.groups())
-    args = eval(args or '()')  # Yes, really eval()!
-    if not isinstance(args, tuple):
-        args = (args,)
-    return name, args
+REMAPPINGS = {
+    'false': False,
+    'true': True,
+    'null': None,
+    'False': False,
+    'True': True,
+    'None': None,
+}
+
+def eval_arguments(args):
+    tokens = tokenize.generate_tokens(StringIO(args or '()').readline)
+    def remap():
+        for type, name, _, _, _ in tokens:
+            if type == tokenize.NAME and name not in REMAPPINGS:
+                yield tokenize.STRING, '"%s"' % name
+            else:
+                yield type, name
+    untok = tokenize.untokenize(remap())
+    if untok[1:-1].strip():
+        untok = untok[:-1] + ',)'  # Force a tuple.
+    return eval(untok, REMAPPINGS)
 
 class Function(object):
     def __init__(self, desc='', default_path=''):
@@ -26,7 +41,12 @@ class Function(object):
             self.function = lambda *args, **kwds: None
             return
 
-        self.function, self.args = _split_function(self.desc)
+        m = MATCHER.match(desc)
+        if not m:
+            raise ValueError('"%s" is not a function' % desc)
+        self.function, self.args = (g.strip() for g in m.groups())
+        self.args = eval_arguments(self.args)
+
         if '.' not in self.function:
             if default_path and not default_path.endswith('.'):
                 default_path += '.'
@@ -40,10 +60,16 @@ class Function(object):
         try:
             self.function = getattr(mod, m)
         except:
-            raise ValueError('No function "%s" in module "%s"' % (p, m))
+            raise ValueError('No function "%s" in module "%s"' % (m, p))
 
     def __str__(self):
         return self.desc
 
     def __call__(self, *args, **kwds):
         return self.function(*(args + self.args), **kwds)
+
+    def __eq__(self, other):
+        return self.function == other.function and self.args == other.args
+
+    def __ne__(self, other):
+        return not (self == other)

bin/ripple/util/Log.py

@@ -1,36 +1,21 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-def count_all_subitems(x):
-    """Count the subitems of a Python object, including the object itself."""
-    if isinstance(x, list):
-        return 1 + sum(count_all_subitems(i) for i in x)
-    if isinstance(x, dict):
-        return 1 + sum(count_all_subitems(i) for i in x.values())
-    return 1
+import sys
 
-def prune(item, level, count_recursively=True):
-    def subitems(x):
-        if count_recursively:
-            i = count_all_subitems(x) - 1
-        else:
-            i = len(x)
-        return '1 subitem' if (i == 1) else '%d subitems' % i
+VERBOSE = False
 
-    assert level >= 0
-    if not item:
-        return item
+def out(*args, **kwds):
+    kwds.get('print', print)(*args, file=sys.stdout, **kwds)
 
-    if isinstance(item, list):
-        if level:
-            return [prune(i, level - 1, count_recursively) for i in item]
-        else:
-            return '[list with %s]' % subitems(item)
+def info(*args, **kwds):
+    if VERBOSE:
+        out(*args, **kwds)
 
-    if isinstance(item, dict):
-        if level:
-            return dict((k, prune(v, level - 1, count_recursively))
-                        for k, v in item.iteritems())
-        else:
-           return '{dict with %s}' % subitems(item)
+def warn(*args, **kwds):
+    out('WARNING:', *args, **kwds)
 
-    return item
+def error(*args, **kwds):
+    out('ERROR:', *args, **kwds)
+
+def fatal(*args, **kwds):
+    raise Exception('FATAL: ' + ' '.join(str(a) for a in args))

bin/ripple/util/Range.py

@@ -13,7 +13,7 @@ def from_string(desc, **aliases):
         return []
     result = set()
     for d in desc.split(','):
-        nums = [aliases.get(x, None) or int(x) for x in d.split('-')]
+        nums = [int(aliases.get(x) or x) for x in d.split('-')]
         if len(nums) == 1:
             result.add(nums[0])
         elif len(nums) == 2:

bin/ripple/util/test_Function.py

@@ -18,8 +18,20 @@ class test_Function(TestCase):
     def test_empty_function(self):
         self.assertEquals(Function()(), None)
 
+    def test_empty_args(self):
+        f = Function('ripple.util.test_Function.FN()')
+        self.assertEquals(f(), ((), {}))
+
     def test_function(self):
         f = Function('ripple.util.test_Function.FN(True, {1: 2}, None)')
         self.assertEquals(f(), ((True, {1: 2}, None), {}))
         self.assertEquals(f('hello', foo='bar'),
                           (('hello', True, {1: 2}, None), {'foo':'bar'}))
+        self.assertEquals(
+            f, Function('ripple.util.test_Function.FN(true, {1: 2}, null)'))
+
+    def test_quoting(self):
+        f = Function('ripple.util.test_Function.FN(testing)')
+        self.assertEquals(f(), (('testing',), {}))
+        f = Function('ripple.util.test_Function.FN(testing, true, false, null)')
+        self.assertEquals(f(), (('testing', True, False, None), {}))