New DatabaseReader reads ledger numbers from database.

bin/ripple/ledger/Args.py

@@ -13,6 +13,7 @@ from ripple.util.Function import Function
 
 NAME = 'LedgerTool'
 VERSION = '0.1'
+NONE = '(none)'
 
 _parser = argparse.ArgumentParser(
     prog=NAME,
@@ -57,7 +58,14 @@ _parser.add_argument(
     )
 
 _parser.add_argument(
-    '--display', '-d',
+    '--database', '-d',
+    nargs='*',
+    default=NONE,
+    help='Specify a database.',
+    )
+
+_parser.add_argument(
+    '--display',
     help='Specify a function to display ledgers.',
     )
 
@@ -103,6 +111,12 @@ _parser.add_argument(
     help='If true, display times in UTC rather than local time.',
     )
 
+_parser.add_argument(
+    '--validations',
+    default=3,
+    help='The number of validations needed before considering a ledger valid.',
+    )
+
 _parser.add_argument(
     '--version',
     action='version',
@@ -131,6 +145,7 @@ _parser.add_argument(
 
 # Read the arguments from the command line.
 ARGS = _parser.parse_args()
+ARGS.NONE = NONE
 
 Log.VERBOSE = ARGS.verbose
 
@@ -162,10 +177,11 @@ if ARGS.window < 0:
 
 PrettyPrint.INDENT = (ARGS.indent * ' ')
 
-_loaders = bool(ARGS.server) + bool(ARGS.rippled)
+_loaders = (ARGS.database != NONE) + bool(ARGS.rippled) + bool(ARGS.server)
 
 if not _loaders:
     ARGS.rippled = 'rippled'
 
 elif _loaders > 1:
-    raise ValueError('At most one of --rippled and --server must be specified')
+    raise ValueError('At most one of --database,  --rippled and --server '
+                     'may be specified')

bin/ripple/ledger/DatabaseReader.py

@@ -0,0 +1,78 @@
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import json
+import os
+import subprocess
+
+from ripple.ledger.Args import ARGS
+from ripple.util import ConfigFile
+from ripple.util import Database
+from ripple.util import File
+from ripple.util import Log
+from ripple.util import Range
+
+LEDGER_QUERY = """
+SELECT
+     L.*, count(1) validations
+FROM
+    (select LedgerHash, LedgerSeq from Ledgers ORDER BY LedgerSeq DESC) L
+    JOIN Validations V
+    ON (V.LedgerHash = L.LedgerHash)
+    GROUP BY L.LedgerHash
+    HAVING validations >= {validation_quorum}
+    ORDER BY 2;
+"""
+
+COMPLETE_QUERY = """
+SELECT
+     L.LedgerSeq, count(*) validations
+FROM
+    (select LedgerHash, LedgerSeq from Ledgers ORDER BY LedgerSeq) L
+    JOIN Validations V
+    ON (V.LedgerHash = L.LedgerHash)
+    GROUP BY L.LedgerHash
+    HAVING validations >= :validation_quorum
+    ORDER BY 2;
+"""
+
+_DATABASE_NAME = 'ledger.db'
+
+USE_PLACEHOLDERS = False
+
+class DatabaseReader(object):
+    def __init__(self, config):
+        assert ARGS.database != ARGS.NONE
+        database = ARGS.database or config['database_path']
+        if not database.endswith(_DATABASE_NAME):
+            database = os.path.join(database, _DATABASE_NAME)
+        if USE_PLACEHOLDERS:
+            cursor = Database.fetchall(
+                database, COMPLETE_QUERY, config)
+        else:
+            cursor = Database.fetchall(
+                database, LEDGER_QUERY.format(**config), {})
+        self.complete = [c[1] for c in cursor]
+
+    def name_to_ledger_index(self, ledger_name, is_full=False):
+        if not self.complete:
+            return None
+        if ledger_name == 'closed':
+            return self.complete[-1]
+        if ledger_name == 'current':
+            return None
+        if ledger_name == 'validated':
+            return self.complete[-1]
+
+    def get_ledger(self, name, is_full=False):
+        cmd = ['ledger', str(name)]
+        if is_full:
+            cmd.append('full')
+        response = self._command(*cmd)
+        result = response.get('ledger')
+        if result:
+            return result
+        error = response['error']
+        etext = _ERROR_TEXT.get(error)
+        if etext:
+            error = '%s (%s)' % (etext, error)
+        Log.fatal(_ERROR_TEXT.get(error, error))

bin/ripple/ledger/RippledReader.py

@@ -22,13 +22,13 @@ _ERROR_TEXT = {
 _DEFAULT_ERROR_ = "Couldn't connect to server."
 
 class RippledReader(object):
-    def __init__(self):
+    def __init__(self, config):
         fname = File.normalize(ARGS.rippled)
         if not os.path.exists(fname):
             raise Exception('No rippled found at %s.' % fname)
         self.cmd = [fname]
         if ARGS.config:
-            self.cmd.extend(['--conf', _normalize(ARGS.config)])
+            self.cmd.extend(['--conf', File.normalize(ARGS.config)])
         self.info = self._command('server_info')['info']
         c = self.info.get('complete_ledgers')
         if c == 'empty':

bin/ripple/ledger/Server.py

@@ -3,17 +3,21 @@ from __future__ import absolute_import, division, print_function, unicode_litera
 import json
 import os
 
-from ripple.ledger import RippledReader, ServerReader
+from ripple.ledger import DatabaseReader, RippledReader
 from ripple.ledger.Args import ARGS
 from ripple.util.FileCache import FileCache
+from ripple.util import ConfigFile
+from ripple.util import File
 from ripple.util import Range
 
 class Server(object):
     def __init__(self):
-        if ARGS.rippled:
+        cfg_file = File.normalize(ARGS.config or 'rippled.cfg')
-            reader = RippledReader.RippledReader()
+        self.config = ConfigFile.read(open(cfg_file))
+        if ARGS.database != ARGS.NONE:
+            reader = DatabaseReader.DatabaseReader(self.config)
         else:
-            reader = ServerReader.ServerReader()
+            reader = RippledReader.RippledReader(self.config)
 
         self.reader = reader
         self.complete = reader.complete
@@ -23,8 +27,7 @@ class Server(object):
             'current': reader.name_to_ledger_index('current'),
             'validated': reader.name_to_ledger_index('validated'),
             'first': self.complete[0] if self.complete else None,
-            'last': self.complete[-1] if self.complete else None
+            'last': self.complete[-1] if self.complete else None,
-,
         }
         self.__dict__.update(names)
         self.ledgers = sorted(Range.join_ranges(*ARGS.ledgers, **names))

bin/ripple/ledger/ServerReader.py

@@ -1,5 +1,5 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 class ServerReader(object):
-    def __init__(self, server):
+    def __init__(self, config):
         raise ValueError('Direct server connections are not yet implemented.')

bin/ripple/ledger/commands/Info.py

@@ -10,8 +10,8 @@ SAFE = True
 HELP = 'info - return server_info'
 
 def info(server):
-    Log.out('first = ', server.first)
+    Log.out('first =', server.first)
-    Log.out('last = ', server.last)
+    Log.out('last =', server.last)
     Log.out('closed =', server.closed)
     Log.out('current =', server.current)
     Log.out('validated =', server.validated)

bin/ripple/util/ConfigFile.py

@@ -0,0 +1,54 @@
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import json
+
+"""Ripple has a proprietary format for their .cfg files, so we need a reader for
+them."""
+
+def read(lines):
+    sections = []
+    section = []
+    for line in lines:
+        line = line.strip()
+        if (not line) or line[0] == '#':
+            continue
+        if line.startswith('['):
+            if section:
+                sections.append(section)
+                section = []
+        section.append(line)
+    if section:
+        sections.append(section)
+
+    result = {}
+    for section in sections:
+        option = section.pop(0)
+        assert section, ('No value for option "%s".' % option)
+        assert option.startswith('[') and option.endswith(']'), (
+            'No option name in block "%s"' % p[0])
+        option = option[1:-1]
+        assert option not in result, 'Duplicate option "%s".' % option
+
+        subdict = {}
+        items = []
+        for part in section:
+            if '=' in part:
+                assert not items, 'Dictionary mixed with list.'
+                k, v = part.split('=', 1)
+                assert k not in subdict, 'Repeated dictionary entry ' + k
+                subdict[k] = v
+            else:
+                assert not subdict, 'List mixed with dictionary.'
+                if part.startswith('{'):
+                    items.append(json.loads(part))
+                else:
+                    words = part.split()
+                    if len(words) > 1:
+                        items.append(words)
+                    else:
+                        items.append(part)
+        if len(items) == 1:
+            result[option] = items[0]
+        else:
+            result[option] = items or subdict
+    return result

bin/ripple/util/Database.py

@@ -0,0 +1,12 @@
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import sqlite3
+
+def fetchall(database, query, kwds):
+    conn = sqlite3.connect(database)
+    try:
+        cursor = conn.execute(query, kwds)
+        return cursor.fetchall()
+
+    finally:
+        conn.close()

bin/ripple/util/Function.py

@@ -20,7 +20,10 @@ REMAPPINGS = {
 }
 
 def eval_arguments(args):
-    tokens = tokenize.generate_tokens(StringIO(args or '()').readline)
+    args = args.strip()
+    if not args or (args == '()'):
+        return ()
+    tokens = list(tokenize.generate_tokens(StringIO(args).readline))
     def remap():
         for type, name, _, _, _ in tokens:
             if type == tokenize.NAME and name not in REMAPPINGS:
@@ -30,7 +33,11 @@ def eval_arguments(args):
     untok = tokenize.untokenize(remap())
     if untok[1:-1].strip():
         untok = untok[:-1] + ',)'  # Force a tuple.
-    return eval(untok, REMAPPINGS)
+    try:
+        return eval(untok, REMAPPINGS)
+    except Exception as e:
+        raise ValueError('Couldn\'t evaluate expression "%s" (became "%s"), '
+                         'error "%s"' % (args, untok, str(e)))
 
 class Function(object):
     def __init__(self, desc='', default_path=''):

bin/ripple/util/test_ConfigFile.py

@@ -0,0 +1,163 @@
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from ripple.util import ConfigFile
+
+from unittest import TestCase
+
+class test_ConfigFile(TestCase):
+    def test_trivial(self):
+        self.assertEquals(ConfigFile.read(''), {})
+
+    def test_full(self):
+        self.assertEquals(ConfigFile.read(FULL.splitlines()), RESULT)
+
+RESULT = {
+    'websocket_port': '6206',
+    'database_path': '/development/alpha/db',
+    'sntp_servers':
+        ['time.windows.com', 'time.apple.com', 'time.nist.gov', 'pool.ntp.org'],
+    'validation_seed': 'sh1T8T9yGuV7Jb6DPhqSzdU2s5LcV',
+    'node_size': 'medium',
+    'rpc_startup': {
+        'command': 'log_level',
+        'severity': 'debug'},
+    'ips': ['r.ripple.com', '51235'],
+    'node_db': {
+        'file_size_mult': '2',
+        'file_size_mb': '8',
+        'cache_mb': '256',
+        'path': '/development/alpha/db/rocksdb',
+        'open_files': '2000',
+        'type': 'RocksDB',
+        'filter_bits': '12'},
+    'peer_port': '53235',
+    'ledger_history': 'full',
+    'rpc_ip': '127.0.0.1',
+    'websocket_public_ip': '0.0.0.0',
+    'rpc_allow_remote': '0',
+    'validators':
+         [['n949f75evCHwgyP4fPVgaHqNHxUVN15PsJEZ3B3HnXPcPjcZAoy7', 'RL1'],
+          ['n9MD5h24qrQqiyBC8aeqqCWvpiBiYQ3jxSr91uiDvmrkyHRdYLUj', 'RL2'],
+          ['n9L81uNCaPgtUJfaHh89gmdvXKAmSt5Gdsw2g1iPWaPkAHW5Nm4C', 'RL3'],
+          ['n9KiYM9CgngLvtRCQHZwgC2gjpdaZcCcbt3VboxiNFcKuwFVujzS', 'RL4'],
+          ['n9LdgEtkmGB9E2h3K4Vp7iGUaKuq23Zr32ehxiU8FWY7xoxbWTSA', 'RL5']],
+    'debug_logfile': '/development/alpha/debug.log',
+    'websocket_public_port': '5206',
+    'peer_ip': '0.0.0.0',
+    'rpc_port': '5205',
+    'validation_quorum': '3',
+    'websocket_ip': '127.0.0.1'}
+
+FULL = """
+[ledger_history]
+full
+
+# Allow other peers to connect to this server.
+#
+[peer_ip]
+0.0.0.0
+
+[peer_port]
+53235
+
+# Allow untrusted clients to connect to this server.
+#
+[websocket_public_ip]
+0.0.0.0
+
+[websocket_public_port]
+5206
+
+# Provide trusted websocket ADMIN access to the localhost.
+#
+[websocket_ip]
+127.0.0.1
+
+[websocket_port]
+6206
+
+# Provide trusted json-rpc ADMIN access to the localhost.
+#
+[rpc_ip]
+127.0.0.1
+
+[rpc_port]
+5205
+
+[rpc_allow_remote]
+0
+
+[node_size]
+medium
+
+# This is primary persistent datastore for rippled.  This includes transaction
+# metadata, account states, and ledger headers.  Helpful information can be
+# found here: https://ripple.com/wiki/NodeBackEnd
+[node_db]
+type=RocksDB
+path=/development/alpha/db/rocksdb
+open_files=2000
+filter_bits=12
+cache_mb=256
+file_size_mb=8
+file_size_mult=2
+
+[database_path]
+/development/alpha/db
+
+# This needs to be an absolute directory reference, not a relative one.
+# Modify this value as required.
+[debug_logfile]
+/development/alpha/debug.log
+
+[sntp_servers]
+time.windows.com
+time.apple.com
+time.nist.gov
+pool.ntp.org
+
+# Where to find some other servers speaking the Ripple protocol.
+#
+[ips]
+r.ripple.com 51235
+
+# The latest validators can be obtained from
+# https://ripple.com/ripple.txt
+#
+[validators]
+n949f75evCHwgyP4fPVgaHqNHxUVN15PsJEZ3B3HnXPcPjcZAoy7	RL1
+n9MD5h24qrQqiyBC8aeqqCWvpiBiYQ3jxSr91uiDvmrkyHRdYLUj	RL2
+n9L81uNCaPgtUJfaHh89gmdvXKAmSt5Gdsw2g1iPWaPkAHW5Nm4C	RL3
+n9KiYM9CgngLvtRCQHZwgC2gjpdaZcCcbt3VboxiNFcKuwFVujzS	RL4
+n9LdgEtkmGB9E2h3K4Vp7iGUaKuq23Zr32ehxiU8FWY7xoxbWTSA	RL5
+
+# Ditto.
+[validation_quorum]
+3
+
+[validation_seed]
+sh1T8T9yGuV7Jb6DPhqSzdU2s5LcV
+
+# Turn down default logging to save disk space in the long run.
+# Valid values here are trace, debug, info, warning, error, and fatal
+[rpc_startup]
+{ "command": "log_level", "severity": "debug" }
+
+# Configure SSL for WebSockets.  Not enabled by default because not everybody
+# has an SSL cert on their server, but if you uncomment the following lines and
+# set the path to the SSL certificate and private key the WebSockets protocol
+# will be protected by SSL/TLS.
+#[websocket_secure]
+#1
+
+#[websocket_ssl_cert]
+#/etc/ssl/certs/server.crt
+
+#[websocket_ssl_key]
+#/etc/ssl/private/server.key
+
+# Defaults to 0 ("no") so that you can use self-signed SSL certificates for
+# development, or internally.
+#[ssl_verify]
+#0
+""".strip()