Tidying & Selectively forward manifests to peers:

* Do not forward manifests to peers that already know that manifest
* Do not forward historical manifests to peers
* Save/Load ValidatorManifests from a database
* Python test for setting ephmeral keys
* Cleanup manifest interface

bin/python/ripple/util/ValidatorManifestTest.py

@@ -0,0 +1,682 @@
+#!/usr/bin/env python
+"""
+Test for setting ephemeral keys for the validator manifest.
+"""
+
+from __future__ import (
+    absolute_import, division, print_function, unicode_literals
+)
+
+import argparse
+import contextlib
+from contextlib import contextmanager
+import json
+import os
+import platform
+import shutil
+import subprocess
+import time
+
+DELAY_WHILE_PROCESS_STARTS_UP = 1.5
+ARGS = None
+
+NOT_FOUND = -1  # not in log
+ACCEPTED_NEW = 0  # added new manifest
+ACCEPTED_UPDATE = 1  # replaced old manifest with new
+UNTRUSTED = 2  # don't trust master key
+STALE = 3  # seq is too old
+REVOKED = 4  # revoked validator key
+INVALID = 5  # invalid signature
+
+MANIFEST_ACTION_STR_TO_ID = {
+    'NotFound': NOT_FOUND,  # not found in log
+    'AcceptedNew': ACCEPTED_NEW,
+    'AcceptedUpdate': ACCEPTED_UPDATE,
+    'Untrusted': UNTRUSTED,
+    'Stale': STALE,
+    'Revoked': REVOKED,
+    'Invalid': INVALID,
+}
+
+MANIFEST_ACTION_ID_TO_STR = {
+    v: k for k, v in MANIFEST_ACTION_STR_TO_ID.items()
+}
+
+CONF_TEMPLATE = """
+[server]
+port_rpc
+port_peer
+port_wss_admin
+
+[port_rpc]
+port = {rpc_port}
+ip = 127.0.0.1
+admin = 127.0.0.1
+protocol = https
+
+[port_peer]
+port = {peer_port}
+ip = 0.0.0.0
+protocol = peer
+
+[port_wss_admin]
+port = {wss_port}
+ip = 127.0.0.1
+admin = 127.0.0.1
+protocol = wss
+
+[node_size]
+medium
+
+[node_db]
+type={node_db_type}
+path={node_db_path}
+open_files=2000
+filter_bits=12
+cache_mb=256
+file_size_mb=8
+file_size_mult=2
+online_delete=256
+advisory_delete=0
+
+[database_path]
+{db_path}
+
+[debug_logfile]
+{debug_logfile}
+
+[sntp_servers]
+time.windows.com
+time.apple.com
+time.nist.gov
+pool.ntp.org
+
+[ips]
+r.ripple.com 51235
+
+[ips_fixed]
+{sibling_ip} {sibling_port}
+
+[validators]
+n949f75evCHwgyP4fPVgaHqNHxUVN15PsJEZ3B3HnXPcPjcZAoy7    RL1
+n9MD5h24qrQqiyBC8aeqqCWvpiBiYQ3jxSr91uiDvmrkyHRdYLUj    RL2
+n9L81uNCaPgtUJfaHh89gmdvXKAmSt5Gdsw2g1iPWaPkAHW5Nm4C    RL3
+n9KiYM9CgngLvtRCQHZwgC2gjpdaZcCcbt3VboxiNFcKuwFVujzS    RL4
+n9LdgEtkmGB9E2h3K4Vp7iGUaKuq23Zr32ehxiU8FWY7xoxbWTSA    RL5
+
+[validation_quorum]
+3
+
+[validation_seed]
+{validation_seed}
+#vaidation_public_key: {validation_public_key}
+
+# Other rippled's trusting this validator need this key
+[validator_keys]
+{all_validator_keys}
+
+[peer_private]
+1
+
+[overlay]
+expire = 1
+auto_connect = 1
+
+[validation_manifest]
+{validation_manifest}
+
+[rpc_startup]
+{{ "command": "log_level", "severity": "debug" }}
+
+[ssl_verify]
+0
+"""
+# End config template
+
+
+def static_vars(**kwargs):
+    def decorate(func):
+        for k in kwargs:
+            setattr(func, k, kwargs[k])
+        return func
+    return decorate
+
+
+@static_vars(rpc=5005, peer=51235, wss=6006)
+def checkout_port_nums():
+    """Returns a tuple of port nums for rpc, peer, and wss_admin"""
+    checkout_port_nums.rpc += 1
+    checkout_port_nums.peer += 1
+    checkout_port_nums.wss += 1
+    return (
+        checkout_port_nums.rpc,
+        checkout_port_nums.peer,
+        checkout_port_nums.wss
+    )
+
+
+def is_windows():
+    return platform.system() == 'Windows'
+
+
+def manifest_create():
+    """returns dict with keys: 'validator_keys', 'master_secret'"""
+    to_run = ['python', ARGS.ripple_home + '/bin/python/Manifest.py', 'create']
+    r = subprocess.check_output(to_run)
+    result = {}
+    k = None
+    for l in r.splitlines():
+        l = l.strip()
+        if not l:
+            continue
+        elif l == '[validator_keys]':
+            k = l[1:-1]
+        elif l == '[master_secret]':
+            k = l[1:-1]
+        elif l.startswith('['):
+            raise ValueError(
+                'Unexpected key: {} from `manifest create`'.format(l))
+        else:
+            if not k:
+                raise ValueError('Value with no key')
+            result[k] = l
+            k = None
+
+        if k in result:
+            raise ValueError('Repeat key from `manifest create`: ' + k)
+    if len(result) != 2:
+        raise ValueError(
+            'Expected 2 keys from `manifest create` but got {} keys instead ({})'.
+            format(len(result), result))
+
+    return result
+
+
+def sign_manifest(seq, validation_pk, master_secret):
+    """returns the signed manifest as a string"""
+    to_run = ['python', ARGS.ripple_home + '/bin/python/Manifest.py', 'sign',
+              str(seq), validation_pk, master_secret]
+    try:
+        r = subprocess.check_output(to_run)
+    except subprocess.CalledProcessError as e:
+        print('Error in sign_manifest: ', e.output)
+        raise e
+    result = []
+    for l in r.splitlines():
+        l.strip()
+        if not l or l == '[validation_manifest]':
+            continue
+        result.append(l)
+    return '\n'.join(result)
+
+
+def get_ripple_exe():
+    """Find the rippled executable"""
+    prefix = ARGS.ripple_home + '/build/'
+    exe = ['rippled', 'RippleD.exe']
+    to_test = [prefix + t + '.debug/' + e
+               for t in ['clang', 'gcc', 'msvc'] for e in exe]
+    for e in exe:
+        to_test.append(prefix + '/' + e)
+    for t in to_test:
+        if os.path.isfile(t):
+            return t
+
+
+class RippledServer(object):
+    def __init__(self, exe, config_file, server_out):
+        self.config_file = config_file
+        self.exe = exe
+        self.process = None
+        self.server_out = server_out
+        self.reinit(config_file)
+
+    def reinit(self, config_file):
+        self.config_file = config_file
+        self.to_run = [self.exe, '--verbose', '--conf', self.config_file]
+
+    @property
+    def config_root(self):
+        return os.path.dirname(self.config_file)
+
+    @property
+    def master_secret_file(self):
+        return self.config_root + '/master_secret.txt'
+
+    def startup(self):
+        if ARGS.verbose:
+            print('starting rippled:' + self.config_file)
+        fout = open(self.server_out, 'w')
+        self.process = subprocess.Popen(
+            self.to_run, stdout=fout, stderr=subprocess.STDOUT)
+
+    def shutdown(self):
+        if not self.process:
+            return
+        fout = open(os.devnull, 'w')
+        subprocess.Popen(
+            self.to_run + ['stop'], stdout=fout, stderr=subprocess.STDOUT)
+        self.process.wait()
+        self.process = None
+
+    def rotate_logfile(self):
+        if self.server_out == os.devnull:
+            return
+        for i in range(100):
+            backup_name = '{}.{}'.format(self.server_out, i)
+            if not os.path.exists(backup_name):
+                os.rename(self.server_out, backup_name)
+                return
+        raise ValueError('Could not rotate logfile: {}'.
+                         format(self.server_out))
+
+    def validation_create(self):
+        """returns dict with keys:
+        'validation_key', 'validation_public_key', 'validation_seed'
+        """
+        to_run = [self.exe, '-q', '--conf', self.config_file,
+                  '--', 'validation_create']
+        try:
+            return json.loads(subprocess.check_output(to_run))['result']
+        except subprocess.CalledProcessError as e:
+            print('Error in validation_create: ', e.output)
+            raise e
+
+
+@contextmanager
+def rippled_server(config_file, server_out=os.devnull):
+    """Start a ripple server"""
+    try:
+        server = None
+        server = RippledServer(ARGS.ripple_exe, config_file, server_out)
+        server.startup()
+        yield server
+    finally:
+        if server:
+            server.shutdown()
+
+
+@contextmanager
+def pause_server(server, config_file):
+    """Shutdown and then restart a ripple server"""
+    try:
+        server.shutdown()
+        server.rotate_logfile()
+        yield server
+    finally:
+        server.reinit(config_file)
+        server.startup()
+
+
+def parse_date(d, t):
+    """Return the timestamp of a line, or none if the line has no timestamp"""
+    try:
+        return time.strptime(d+' '+t, '%Y-%B-%d %H:%M:%S')
+    except:
+        return None
+
+
+def to_dict(l):
+    """Given a line of the form Key0: Value0;Key2: Valuue2; Return a dict"""
+    fields = l.split(';')
+    result = {}
+    for f in fields:
+        if f:
+            v = f.split(':')
+            assert len(v) == 2
+            result[v[0].strip()] = v[1].strip()
+    return result
+
+
+def check_ephemeral_key(validator_key,
+                        log_file,
+                        seq,
+                        change_time):
+    """
+    Detect when a server is informed of a validator's ephemeral key change.
+    `change_time` and `seq` may be None, in which case they are ignored.
+    """
+    manifest_prefix = 'Manifest:'
+    # a manifest line has the form Manifest: action; Key: value;
+    # Key can be Pk (public key), Seq, OldSeq,
+    for l in open(log_file):
+        sa = l.split()
+        if len(sa) < 5 or sa[4] != manifest_prefix:
+            continue
+
+        d = to_dict(' '.join(sa[4:]))
+        # check the seq number and validator_key
+        if d['Pk'] != validator_key:
+            continue
+        if seq is not None and int(d['Seq']) != seq:
+            continue
+
+        if change_time:
+            t = parse_date(sa[0], sa[1])
+            if not t or t < change_time:
+                continue
+        action = d['Manifest']
+        return MANIFEST_ACTION_STR_TO_ID[action]
+    return NOT_FOUND
+
+
+def check_ephemeral_keys(validator_key,
+                         log_files,
+                         seq,
+                         change_time=None,
+                         timeout_s=60):
+    result = [NOT_FOUND for i in range(len(log_files))]
+    if timeout_s < 10:
+        sleep_time = 1
+    elif timeout_s < 60:
+        sleep_time = 5
+    else:
+        sleep_time = 10
+    n = timeout_s//sleep_time
+    if n == 0:
+        n = 1
+    start_time = time.time()
+    for _ in range(n):
+        for i, lf in enumerate(log_files):
+            if result[i] != NOT_FOUND:
+                continue
+            result[i] = check_ephemeral_key(validator_key,
+                                            lf,
+                                            seq,
+                                            change_time)
+            if result[i] != NOT_FOUND:
+                if all(r != NOT_FOUND for r in result):
+                    return result
+                else:
+                    server_dir = os.path.basename(os.path.dirname(log_files[i]))
+                    if ARGS.verbose:
+                        print('Check for {}: {}'.format(
+                            server_dir, MANIFEST_ACTION_ID_TO_STR[result[i]]))
+        tsf = time.time() - start_time
+        if tsf > 20:
+            if ARGS.verbose:
+                print('Waiting for key to propigate: ', tsf)
+        time.sleep(sleep_time)
+    return result
+
+
+def get_validator_key(config_file):
+    in_validator_keys = False
+    for l in open(config_file):
+        sl = l.strip()
+        if not in_validator_keys and sl == '[validator_keys]':
+            in_validator_keys = True
+            continue
+        if in_validator_keys:
+            if sl.startswith('['):
+                raise ValueError('ThisServer validator key not found')
+            if sl.startswith('#'):
+                continue
+            s = sl.split()
+            if len(s) == 2 and s[1] == 'ThisServer':
+                return s[0]
+
+
+def new_config_ephemeral_key(
+        server, seq, rm_dbs=False, master_secret_file=None):
+    """Generate a new ephemeral key, add to config, restart server"""
+    config_root = server.config_root
+    config_file = config_root + '/rippled.cfg'
+    db_dir = config_root + '/db'
+    if not master_secret_file:
+        master_secret_file = server.master_secret_file
+    with open(master_secret_file) as f:
+        master_secret = f.read()
+    v = server.validation_create()
+    signed = sign_manifest(seq, v['validation_public_key'], master_secret)
+    with pause_server(server, config_file):
+        if rm_dbs and os.path.exists(db_dir):
+            shutil.rmtree(db_dir)
+            os.makedirs(db_dir)
+        # replace the validation_manifest section with `signed`
+        bak = config_file + '.bak'
+        if is_windows() and os.path.isfile(bak):
+            os.remove(bak)
+        os.rename(config_file, bak)
+        in_manifest = False
+        with open(bak, 'r') as src:
+            with open(config_file, 'w') as out:
+                for l in src:
+                    sl = l.strip()
+                    if not in_manifest and sl == '[validation_manifest]':
+                        in_manifest = True
+                    elif in_manifest:
+                        if sl.startswith('[') or sl.startswith('#'):
+                            in_manifest = False
+                            out.write(signed)
+                            out.write('\n\n')
+                        else:
+                            continue
+                    out.write(l)
+    return (bak, config_file)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description=('Create config files for n validators')
+    )
+
+    parser.add_argument(
+        '--ripple_home', '-r',
+        default=os.sep.join(os.path.realpath(__file__).split(os.sep)[:-5]),
+        help=('Root directory of the ripple repo'), )
+    parser.add_argument('--num_validators', '-n',
+                        default=2,
+                        help=('Number of validators'), )
+    parser.add_argument('--conf', '-c', help=('rippled config file'), )
+    parser.add_argument('--out', '-o',
+                        default='test_output',
+                        help=('config root directory'), )
+    parser.add_argument(
+        '--existing', '-e',
+        action='store_true',
+        help=('use existing config files'), )
+    parser.add_argument(
+        '--generate', '-g',
+        action='store_true',
+        help=('generate conf files only'), )
+    parser.add_argument(
+        '--verbose', '-v',
+        action='store_true',
+        help=('verbose status reporting'), )
+    parser.add_argument(
+        '--quiet', '-q',
+        action='store_true',
+        help=('quiet status reporting'), )
+
+    return parser.parse_args()
+
+
+def get_configs(manifest_seq):
+    global ARGS
+    ARGS.ripple_home = os.path.expanduser(ARGS.ripple_home)
+
+    n = int(ARGS.num_validators)
+    if n<2:
+        raise ValueError(
+            'Need at least 2 rippled servers. Specified: {}'.format(n))
+    config_root = ARGS.out
+    ARGS.ripple_exe = get_ripple_exe()
+    if not ARGS.ripple_exe:
+        raise ValueError('No Exe Found')
+
+    if ARGS.existing:
+        return [
+            os.path.abspath('{}/validator_{}/rippled.cfg'.format(config_root, i))
+            for i in range(n)
+        ]
+
+    initial_config = ARGS.conf
+
+    manifests = [manifest_create() for i in range(n)]
+    port_nums = [checkout_port_nums() for i in range(n)]
+    with rippled_server(initial_config) as server:
+        time.sleep(DELAY_WHILE_PROCESS_STARTS_UP)
+        validations = [server.validation_create() for i in range(n)]
+
+    signed_manifests = [sign_manifest(manifest_seq,
+                                      v['validation_public_key'],
+                                      m['master_secret'])
+                        for m, v in zip(manifests, validations)]
+    node_db_type = 'RocksDB' if not is_windows() else 'NuDB'
+    node_db_filename = node_db_type.lower()
+
+    config_files = []
+    for i, (m, v, s) in enumerate(zip(manifests, validations, signed_manifests)):
+        sibling_index = (i - 1) % len(manifests)
+        all_validator_keys = '\n'.join([
+            m['validator_keys'] + ' ThisServer',
+            manifests[sibling_index]['validator_keys'] + ' NextInRing'])
+        this_validator_dir = os.path.abspath(
+            '{}/validator_{}'.format(config_root, i))
+        db_path = this_validator_dir + '/db'
+        node_db_path = db_path + '/' + node_db_filename
+        log_path = this_validator_dir + '/log'
+        debug_logfile = log_path + '/debug.log'
+        rpc_port, peer_port, wss_port = port_nums[i]
+        sibling_ip = '127.0.0.1'
+        sibling_port = port_nums[sibling_index][1]
+        d = {
+            'validation_manifest': s,
+            'all_validator_keys': all_validator_keys,
+            'node_db_type': node_db_type,
+            'node_db_path': node_db_path,
+            'db_path': db_path,
+            'debug_logfile': debug_logfile,
+            'rpc_port': rpc_port,
+            'peer_port': peer_port,
+            'wss_port': wss_port,
+            'sibling_ip': sibling_ip,
+            'sibling_port': sibling_port,
+        }
+        d.update(m)
+        d.update(v)
+
+        for p in [this_validator_dir, db_path, log_path]:
+            if not os.path.exists(p):
+                os.makedirs(p)
+
+        config_files.append('{}/rippled.cfg'.format(this_validator_dir))
+        with open(config_files[-1], 'w') as f:
+            f.write(CONF_TEMPLATE.format(**d))
+
+        with open('{}/master_secret.txt'.format(this_validator_dir), 'w') as f:
+            f.write(m['master_secret'])
+
+    return config_files
+
+
+def update_ephemeral_key(
+        server, new_seq, log_files,
+        expected=None, rm_dbs=False, master_secret_file=None,
+        restore_origional_conf=False, timeout_s=300):
+    if not expected:
+        expected = {}
+
+    change_time = time.gmtime()
+    back_conf, new_conf = new_config_ephemeral_key(
+        server,
+        new_seq,
+        rm_dbs,
+        master_secret_file
+    )
+    validator_key = get_validator_key(server.config_file)
+    start_time = time.time()
+    ck = check_ephemeral_keys(validator_key,
+                              log_files,
+                              seq=new_seq,
+                              change_time=change_time,
+                              timeout_s=timeout_s)
+    if ARGS.verbose:
+        print('Check finished: {} secs.'.format(int(time.time() - start_time)))
+    all_success = True
+    for i, r in enumerate(ck):
+        e = expected.get(i, UNTRUSTED)
+        server_dir = os.path.basename(os.path.dirname(log_files[i]))
+        status = 'OK' if e == r else 'FAIL'
+        print('{}: Server: {} Expected: {} Got: {}'.
+              format(status, server_dir,
+                     MANIFEST_ACTION_ID_TO_STR[e], MANIFEST_ACTION_ID_TO_STR[r]))
+        all_success = all_success and (e == r)
+    if restore_origional_conf:
+        if is_windows() and os.path.isfile(new_conf):
+            os.remove(new_conf)
+        os.rename(back_conf, new_conf)
+    return all_success
+
+
+def run_main():
+    global ARGS
+    ARGS = parse_args()
+    manifest_seq = 1
+    config_files = get_configs(manifest_seq)
+    if ARGS.generate:
+        return
+    if len(config_files) <= 1:
+        print('Script requires at least 2 servers. Actual #: {}'.
+              format(len(config_files)))
+        return
+    with contextlib.nested(*(rippled_server(c, os.path.dirname(c)+'/log.txt')
+                             for c in config_files)) as servers:
+        log_files = [os.path.dirname(cf)+'/log.txt' for cf in config_files[1:]]
+        validator_key = get_validator_key(config_files[0])
+        start_time = time.time()
+        ck = check_ephemeral_keys(validator_key,
+                                  [log_files[0]],
+                                  seq=None,
+                                  timeout_s=60)
+        if ARGS.verbose:
+            print('Check finished: {} secs.'.format(
+                int(time.time() - start_time)))
+        if any(r == NOT_FOUND for r in ck):
+            print('FAIL: Initial key did not propigate to all servers')
+            return
+
+        manifest_seq += 2
+        expected = {i: UNTRUSTED for i in range(len(log_files))}
+        expected[0] = ACCEPTED_UPDATE
+        if not ARGS.quiet:
+            print('Testing key update')
+        kr = update_ephemeral_key(servers[0], manifest_seq, log_files, expected)
+        if not kr:
+            print('\nFail: Key Update Test. Exiting')
+            return
+
+        expected = {i: UNTRUSTED for i in range(len(log_files))}
+        expected[0] = STALE
+        if not ARGS.quiet:
+            print('Testing stale key')
+        kr = update_ephemeral_key(
+            servers[0], manifest_seq-1, log_files, expected, rm_dbs=True)
+        if not kr:
+            print('\nFail: Stale Key Test. Exiting')
+            return
+
+        expected = {i: UNTRUSTED for i in range(len(log_files))}
+        expected[0] = STALE
+        if not ARGS.quiet:
+            print('Testing stale key 2')
+        kr = update_ephemeral_key(
+            servers[0], manifest_seq, log_files, expected, rm_dbs=True)
+        if not kr:
+            print('\nFail: Stale Key Test. Exiting')
+            return
+
+        expected = {i: UNTRUSTED for i in range(len(log_files))}
+        expected[0] = REVOKED
+        if not ARGS.quiet:
+            print('Testing revoked key')
+        kr = update_ephemeral_key(
+            servers[0], 0xffffffff, log_files, expected, rm_dbs=True)
+        if not kr:
+            print('\nFail: Revoked Key Text. Exiting')
+            return
+        print('\nOK: All tests passed')
+
+if __name__ == '__main__':
+    run_main()

doc/manifest-tool-guide.md

@@ -0,0 +1,110 @@
+# Manifest Tool Guide
+
+This guide explains how to setup a validator so the key pairs used to sign and
+verify validations may safely change. This procedure does not require manual
+reconfiguration of servers that trust this validator.
+
+Validators use two types of key pairs: *master keys* and *ephemeral
+keys*. Ephemeral keys are used to sign and verify validations. Master keys are
+used to sign and verify manifests that change ephemeral keys. The master secret
+key should be tightly controlled. The ephemeral secret key needs to be present
+in the config file.
+
+## Validator Keys
+
+When first setting up a validator, use the `manifest` script to generate a
+master key pair:
+
+```
+  $ bin/manifest create
+```
+
+Sample output:
+```
+  [validator_keys]
+  nHUSSzGw4A9zEmFtK2Q2NcWDH9xmGdXMHc1MsVej3QkLTgvDNeBr
+
+  [master_secret]
+  pnxayCakmZRQE2qhEVRbFjiWCunReSbN1z64vPL36qwyLgogyYc
+```
+
+The first value is the master public key. Add the public key to the config
+for this validator. A one-word comment must be added after the key (for example
+*ThisServersName*). Any other rippled trusting the validator needs to add the
+master public key to its config. Only add keys received from trusted sources.
+
+The second value is the corresponding master secret key. **DO NOT INSTALL THIS
+IN THE CONFIG**. The master secret key will be used to sign manifests that
+change validation keys. Put the master secret key in a secure but recoverable
+location.
+
+## Validation Keys
+
+When first setting up a validator, or when changing the ephemeral keys, use the
+`rippled` program to create a new ephemeral key pair:
+
+```
+  $ rippled validation_create
+```
+
+Sample output:
+
+```
+  Loading: "/Users/alice/.config/ripple/rippled.cfg"
+  Securely connecting to 127.0.0.1:5005
+  {
+     "result" : {
+      "status" : "success",
+      "validation_key" : "TOO EDNA SHUN FEUD STAB JOAN BIAS FLEA WISE BOHR LOSS WEEK",
+      "validation_public_key" : "n9JzKV3ZrcZ3DW5pwjakj4hpijJ9oMiyrPDGJc3mpsndL6Gf3zwd",
+      "validation_seed" : "sahzkAajS2dyhjXg2yovjdZhXmjsx"
+     }
+  }
+```
+
+Add the `validation_seed` value (the ephemeral secret key) to this validator's
+config. It is recommended to add the ephemeral public key and the sequence
+number as a comment as well (sequence numbers are be explained below):
+
+```
+  [validation_seed]
+  sahzkAajS2dyhjXg2yovjdZhXmjsx
+  # validation_public_key: n9JzKV3ZrcZ3DW5pwjakj4hpijJ9oMiyrPDGJc3mpsndL6Gf3zwd
+  # sequence number: 1
+```
+
+A manifest is a signed message used to inform other servers of this validator's
+ephemeral public key. A manifest contains a sequence number, the new ephemeral
+public key, and it is signed with the master secret key. The sequence number
+should be higher than the previous sequence number (if it is not, the manifest
+will be ignored). Usually the previous sequence number will be incremented by
+one. Use the `manifest` script to create a manifest. It has the form:
+
+```
+ $ bin/manifest sign sequence_number validation_public_key master_secret
+```
+
+For example:
+
+```
+  $ bin/manifest sign 1 n9JzKV3Z...L6Gf3zwd pnxayCak...yLgogyYc
+```
+
+Sample output:
+
+```
+  [validation_manifest]
+  JAAAAAFxIe2PEzNhe996gykB1PJQNoDxvr/Y0XhDELw8d/i
+  Fcgz3A3MhAjqhKsgZTmK/3BPEI+kzjV1p9ip7pl/AtF7CKd
+  NSfAH9dkCxezV6apS4FLYzAcQilONx315HvebwAB/pLPaM4
+  2sWCEppSuLNKN/JJjTABOo9tmAiNnnstF83yvecKMJzniwN
+```  
+
+Copy this to the config for this validator. Don't forget to update the comment
+noting the sequence number.
+
+## Revoking a key
+
+If a master key is compromised, the key may be revoked permanently. To revoke a
+master key, sign a manifest with the highest possible sequence number:
+`4,294,967,295`

src/ripple/app/main/Application.cpp

@@ -806,6 +806,8 @@ public:
             getConfig());
         add (*m_overlay); // add to PropertyStream
 
+        m_overlay->setupValidatorKeyManifests (getConfig (), getWalletDB ());
+
         {
             auto setup = setup_ServerHandler(getConfig(), std::cerr);
             setup.makeContexts();
@@ -900,6 +902,8 @@ public:
 
         mValidations->flush ();
 
+        m_overlay->saveValidatorKeyManifests (getWalletDB ());
+
         RippleAddress::clearCache ();
         stopped ();
     }

src/ripple/app/main/DBInit.cpp

@@ -231,6 +231,13 @@ const char* WalletDBInit[] =
         PRIMARY KEY (Validator,Entry)               \
     );",
 
+    // Validator Manifests
+    R"(
+        CREATE TABLE IF NOT EXISTS ValidatorManifests (
+        RawData          BLOB NOT NULL
+        );
+    )",
+
     // List of referrals from ripple.txt files.
     // Validator:
     //  Public key of referree.

src/ripple/app/misc/UniqueNodeList.h

@@ -20,7 +20,7 @@
 #ifndef RIPPLE_APP_PEERS_UNIQUENODELIST_H_INCLUDED
 #define RIPPLE_APP_PEERS_UNIQUENODELIST_H_INCLUDED
 
-#include <ripple/app/peers/ClusterNodeStatus.h>
+#include <ripple/overlay/ClusterNodeStatus.h>
 #include <ripple/protocol/AnyPublicKey.h>
 #include <ripple/protocol/RippleAddress.h>
 #include <beast/cxx14/memory.h> // <memory>

src/ripple/core/SociDB.h

@@ -114,6 +114,7 @@ size_t getKBUsedDB (soci::session& s);
 void convert (soci::blob& from, std::vector<std::uint8_t>& to);
 void convert (soci::blob& from, std::string& to);
 void convert (std::vector<std::uint8_t> const& from, soci::blob& to);
+void convert (std::string const& from, soci::blob& to);
 
 class Checkpointer
 {

src/ripple/core/impl/SociDB.cpp

@@ -160,6 +160,16 @@ void convert (std::vector<std::uint8_t> const& from, soci::blob& to)
 {
     if (!from.empty ())
         to.write (0, reinterpret_cast<char const*>(&from[0]), from.size ());
+    else
+        to.trim (0);
+}
+
+void convert (std::string const& from, soci::blob& to)
+{
+    if (!from.empty ())
+        to.write (0, from.data (), from.size ());
+    else
+        to.trim (0);
 }
 
 namespace {

src/ripple/overlay/Overlay.h

@@ -38,6 +38,9 @@ namespace boost { namespace asio { namespace ssl { class context; } } }
 
 namespace ripple {
 
+class DatabaseCon;
+class BasicConfig;
+
 /** Manages the set of connected peers. */
 class Overlay
     : public beast::Stoppable
@@ -156,6 +159,15 @@ public:
     relay (protocol::TMValidation& m,
         uint256 const& uid) = 0;
 
+    virtual
+    void
+    setupValidatorKeyManifests (BasicConfig const& config,
+                                DatabaseCon& db) = 0;
+
+    virtual
+    void
+    saveValidatorKeyManifests (DatabaseCon& db) const = 0;
+
     /** Visit every active peer and return a value
         The functor must:
         - Be callable as:
@@ -170,12 +182,11 @@ public:
 
         @note The functor is passed by value!
     */
-    template<typename Function>
+    template <typename UnaryFunc>
-    std::enable_if_t <
+    std::enable_if_t<! std::is_void<
-        ! std::is_void <typename Function::return_type>::value,
+            typename UnaryFunc::return_type>::value,
-        typename Function::return_type
+                typename UnaryFunc::return_type>
-    >
+    foreach (UnaryFunc f)
-    foreach(Function f)
     {
         PeerSequence peers (getActivePeers());
         for(PeerSequence::const_iterator i = peers.begin(); i != peers.end(); ++i)

src/ripple/overlay/impl/Manifest.cpp

@@ -18,7 +18,8 @@
 //==============================================================================
 
 #include <ripple/app/main/Application.h>
-#include <ripple/app/peers/UniqueNodeList.h>
+#include <ripple/app/misc/UniqueNodeList.h>
+#include <ripple/core/DatabaseCon.h>
 #include <ripple/overlay/impl/Manifest.h>
 #include <ripple/protocol/RippleAddress.h>
 #include <ripple/protocol/Sign.h>
@@ -27,12 +28,102 @@
 
 namespace ripple {
 
+boost::optional<Manifest>
+make_Manifest (std::string s)
+{
+    try
+    {
+        STObject st (sfGeneric);
+        SerialIter sit (s.data (), s.size ());
+        st.set (sit);
+        auto const opt_pk = get<AnyPublicKey>(st, sfPublicKey);
+        auto const opt_spk = get<AnyPublicKey>(st, sfSigningPubKey);
+        auto const opt_seq = get (st, sfSequence);
+        auto const opt_sig = get (st, sfSignature);
+        if (!opt_pk || !opt_spk || !opt_seq || !opt_sig)
+        {
+            return boost::none;
+        }
+        return Manifest (std::move (s), *opt_pk, *opt_spk, *opt_seq);
+    }
+    catch (...)
+    {
+        return boost::none;
+    }
+}
+
+template<class Stream>
+Stream&
+logMftAct (
+    Stream& s,
+    std::string const& action,
+    AnyPublicKey const& pk,
+    std::uint32_t seq)
+{
+    s << "Manifest: " << action <<
+         ";Pk: " << toString (pk) <<
+         ";Seq: " << seq << ";";
+    return s;
+}
+
+template<class Stream>
+Stream& logMftAct (
+    Stream& s,
+    std::string const& action,
+    AnyPublicKey const& pk,
+    std::uint32_t seq,
+    std::uint32_t oldSeq)
+{
+    s << "Manifest: " << action <<
+         ";Pk: " << toString (pk) <<
+         ";Seq: " << seq <<
+         ";OldSeq: " << oldSeq << ";";
+    return s;
+}
+
+Manifest::Manifest (std::string s,
+                    AnyPublicKey pk,
+                    AnyPublicKey spk,
+                    std::uint32_t seq)
+    : serialized (std::move (s))
+    , masterKey (std::move (pk))
+    , signingKey (std::move (spk))
+    , sequence (seq)
+{
+}
+
+bool Manifest::verify () const
+{
+    STObject st (sfGeneric);
+    SerialIter sit (serialized.data (), serialized.size ());
+    st.set (sit);
+    return ripple::verify (st, HashPrefix::manifest, masterKey);
+}
+
+uint256 Manifest::hash () const
+{
+    STObject st (sfGeneric);
+    SerialIter sit (serialized.data (), serialized.size ());
+    st.set (sit);
+    return st.getHash (HashPrefix::manifest);
+}
+
+bool Manifest::revoked () const
+{
+    /*
+        The maximum possible sequence number means that the master key
+        has been revoked.
+    */
+    return sequence == std::numeric_limits<std::uint32_t>::max ();
+}
+
 void
-ManifestCache::configValidatorKey(std::string const& line, beast::Journal const& journal)
+ManifestCache::configValidatorKey(
+    std::string const& line, beast::Journal const& journal)
 {
     auto const words = beast::rfc2616::split(line.begin(), line.end(), ' ');
 
-    if (words.size() != 2)
+    if (words.size () != 2)
     {
         throw std::runtime_error ("[validator_keys] format is `<key> <comment>");
     }
@@ -56,44 +147,23 @@ ManifestCache::configValidatorKey(std::string const& line, beast::Journal const&
     }
 
     auto const masterKey = AnyPublicKey (key.data() + 1, key.size() - 1);
-    auto const& comment = words[1];
+    std::string comment = std::move(words[1]);
 
     if (journal.debug) journal.debug
         << masterKey << " " << comment;
 
-    addTrustedKey (masterKey, comment);
+    addTrustedKey (masterKey, std::move(comment));
 }
 
 void
-ManifestCache::configManifest (std::string s, beast::Journal const& journal)
+ManifestCache::configManifest(Manifest m, beast::Journal const& journal)
 {
-    STObject st(sfGeneric);
+    if (!m.verify())
-    try
-    {
-        SerialIter sit(s.data(), s.size());
-        st.set(sit);
-    }
-    catch(...)
-    {
-        throw std::runtime_error("Malformed manifest in config");
-    }
-
-    auto const mseq = get(st, sfSequence);
-    auto const msig = get(st, sfSignature);
-    auto mpk  = get<AnyPublicKey>(st, sfPublicKey);
-    auto mspk = get<AnyPublicKey>(st, sfSigningPubKey);
-    if (! mseq || ! msig || ! mpk || ! mspk)
-    {
-        throw std::runtime_error("Missing fields in manifest in config");
-    }
-    auto const& pk = *mpk;
-
-    if (! verify(st, HashPrefix::manifest, pk))
     {
         throw std::runtime_error("Unverifiable manifest in config");
     }
 
-    auto const result = applyManifest (std::move(s), journal);
+    auto const result = applyManifest (std::move(m), journal);
 
     if (result != ManifestDisposition::accepted)
     {
@@ -102,7 +172,7 @@ ManifestCache::configManifest (std::string s, beast::Journal const& journal)
 }
 
 void
-ManifestCache::addTrustedKey (AnyPublicKey const& pk, std::string const& comment)
+ManifestCache::addTrustedKey (AnyPublicKey const& pk, std::string comment)
 {
     std::lock_guard<std::mutex> lock (mutex_);
 
@@ -110,14 +180,15 @@ ManifestCache::addTrustedKey (AnyPublicKey const& pk, std::string const& comment
 
     if (value.m)
     {
-        throw std::runtime_error ("New trusted validator key already has a manifest");
+        throw std::runtime_error (
+            "New trusted validator key already has a manifest");
     }
 
-    value.comment = comment;
+    value.comment = std::move(comment);
 }
 
 ManifestDisposition
-ManifestCache::preflightManifest_locked (AnyPublicKey const& pk, std::uint32_t seq,
+ManifestCache::canApply (AnyPublicKey const& pk, std::uint32_t seq,
     beast::Journal const& journal) const
 {
     auto const iter = map_.find(pk);
@@ -129,14 +200,14 @@ ManifestCache::preflightManifest_locked (AnyPublicKey const& pk, std::uint32_t s
             Since rippled always sends all of its current manifests,
             this will happen normally any time a peer connects.
         */
-        if (journal.debug) journal.debug
+        if (journal.debug)
-            << "Ignoring manifest #" << seq << " from untrusted key " << pk;
+            logMftAct(journal.debug, "Untrusted", pk, seq);
         return ManifestDisposition::untrusted;
     }
 
     auto& old = iter->second.m;
 
-    if (old  &&  seq <= old->seq)
+    if (old  &&  seq <= old->sequence)
     {
         /*
             A manifest was received for a validator we're tracking, but
@@ -144,67 +215,41 @@ ManifestCache::preflightManifest_locked (AnyPublicKey const& pk, std::uint32_t s
             This will happen normally when a peer without the latest gossip
             connects.
         */
-        if (journal.debug) journal.debug
+        if (journal.debug)
-            << "Ignoring manifest #"      << seq
+            logMftAct(journal.debug, "Stale", pk, seq, old->sequence);
-            << "which isn't newer than #" << old->seq;
         return ManifestDisposition::stale;  // not a newer manifest, ignore
     }
 
     return ManifestDisposition::accepted;
 }
 
+
 ManifestDisposition
-ManifestCache::applyManifest (std::string s, beast::Journal const& journal)
+ManifestCache::applyManifest (Manifest m, beast::Journal const& journal)
 {
-    STObject st(sfGeneric);
-    try
-    {
-        SerialIter sit(s.data(), s.size());
-        st.set(sit);
-    }
-    catch(...)
-    {
-        return ManifestDisposition::malformed;
-    }
-
-    auto const opt_pk = get<AnyPublicKey>(st, sfPublicKey);
-    auto const opt_spk = get<AnyPublicKey>(st, sfSigningPubKey);
-    auto const opt_seq = get(st, sfSequence);
-    auto const opt_sig = get(st, sfSignature);
-
-    if (! opt_pk  ||  ! opt_spk  ||  ! opt_seq  ||  ! opt_sig)
-    {
-        return ManifestDisposition::incomplete;
-    }
-
-    auto const pk = *opt_pk;
-    auto const spk = *opt_spk;
-    auto const seq = *opt_seq;
-
     {
         std::lock_guard<std::mutex> lock (mutex_);
 
         /*
-            "Preflight" the manifest -- before we spend time checking the
+            before we spend time checking the signature, make sure we trust the
-            signature, make sure we trust the master key and the sequence
+            master key and the sequence number is newer than any we have.
-            number is newer than any we have.
         */
-        auto const preflight = preflightManifest_locked(pk, seq, journal);
+        auto const chk = canApply(m.masterKey, m.sequence, journal);
 
-        if (preflight != ManifestDisposition::accepted)
+        if (chk != ManifestDisposition::accepted)
         {
-            return preflight;
+            return chk;
         }
     }
 
-    if (! verify(st, HashPrefix::manifest, pk))
+    if (! m.verify())
     {
         /*
-            A manifest's signature is invalid.
+          A manifest's signature is invalid.
-            This shouldn't happen normally.
+          This shouldn't happen normally.
         */
-        if (journal.warning) journal.warning
+        if (journal.warning)
-            << "Failed to verify manifest #" << seq;
+            logMftAct(journal.warning, "Invalid", m.masterKey, m.sequence);
         return ManifestDisposition::invalid;
     }
 
@@ -213,26 +258,20 @@ ManifestCache::applyManifest (std::string s, beast::Journal const& journal)
     std::lock_guard<std::mutex> lock (mutex_);
 
     /*
-        We released the lock above, so we have to preflight again, in case
+        We released the lock above, so we have to check again, in case
         another thread accepted a newer manifest.
     */
-    auto const preflight = preflightManifest_locked(pk, seq, journal);
+    auto const chk = canApply(m.masterKey, m.sequence, journal);
 
-    if (preflight != ManifestDisposition::accepted)
+    if (chk != ManifestDisposition::accepted)
     {
-        return preflight;
+        return chk;
     }
 
-    auto const iter = map_.find(pk);
+    auto const iter = map_.find(m.masterKey);
 
     auto& old = iter->second.m;
 
-    /*
-        The maximum possible sequence number means that the master key
-        has been revoked.
-    */
-    auto const revoked = std::uint32_t (-1);
-
     if (! old)
     {
         /*
@@ -241,20 +280,20 @@ ManifestCache::applyManifest (std::string s, beast::Journal const& journal)
             run (and possibly not at all, if there's an obsolete entry in
             [validator_keys] for a validator that no longer exists).
         */
-        if (journal.info) journal.info
+        if (journal.info)
-            << "Adding new manifest #" << seq;
+            logMftAct(journal.info, "AcceptedNew", m.masterKey, m.sequence);
     }
     else
     {
-        if (seq == revoked)
+        if (m.revoked ())
         {
             /*
                The MASTER key for this validator was revoked.  This is
                expected, but should happen at most *very* rarely.
             */
-            if (journal.warning) journal.warning
+            if (journal.info)
-                << "Dropping old manifest #" << old->seq
+                logMftAct(journal.info, "Revoked",
-                << " because the master key was revoked";
+                          m.masterKey, m.sequence, old->sequence);
         }
         else
         {
@@ -262,19 +301,19 @@ ManifestCache::applyManifest (std::string s, beast::Journal const& journal)
                 An ephemeral key was revoked and superseded by a new key.
                 This is expected, but should happen infrequently.
             */
-            if (journal.warning) journal.warning
+            if (journal.info)
-                << "Dropping old manifest #" << old->seq
+                logMftAct(journal.info, "AcceptedUpdate",
-                << " in favor of #"          << seq;
+                          m.masterKey, m.sequence, old->sequence);
         }
 
         unl.deleteEphemeralKey (old->signingKey);
     }
 
-    if (seq == revoked)
+    if (m.revoked ())
     {
         // The master key is revoked -- don't insert the signing key
-        if (auto const& j = journal.warning)
+        if (journal.warning)
-            j << "Revoking master key: " << pk;
+            logMftAct(journal.warning, "Revoked", m.masterKey, m.sequence);
 
         /*
             A validator master key has been compromised, so its manifests
@@ -286,12 +325,68 @@ ManifestCache::applyManifest (std::string s, beast::Journal const& journal)
     }
     else
     {
-        unl.insertEphemeralKey (spk, iter->second.comment);
+        unl.insertEphemeralKey (m.signingKey, iter->second.comment);
     }
 
-    old = Manifest(std::move (s), std::move (pk), std::move (spk), seq);
+    old = std::move(m);
 
     return ManifestDisposition::accepted;
 }
 
+void ManifestCache::load (
+    DatabaseCon& dbCon, beast::Journal const& journal)
+{
+    static const char* const sql =
+        "SELECT RawData FROM ValidatorManifests;";
+    auto db = dbCon.checkoutDb ();
+    soci::blob sociRawData (*db);
+    soci::statement st =
+        (db->prepare << sql,
+             soci::into (sociRawData));
+    st.execute ();
+    while (st.fetch ())
+    {
+        std::string serialized;
+        convert (sociRawData, serialized);
+        if (auto mo = make_Manifest (std::move (serialized)))
+        {
+            if (!mo->verify())
+            {
+                throw std::runtime_error("Unverifiable manifest in db");
+            }
+            // add trusted key
+            map_[mo->masterKey];
+
+            // OK if not accepted (may have been loaded from the config file)
+            applyManifest (std::move(*mo), journal);
+        }
+        else
+        {
+            if (journal.warning)
+                journal.warning << "Malformed manifest in database";
+        }
+    }
+}
+
+void ManifestCache::save (DatabaseCon& dbCon) const
+{
+    auto db = dbCon.checkoutDb ();
+
+    soci::transaction tr(*db);
+    *db << "DELETE FROM ValidatorManifests";
+    static const char* const sql =
+        "INSERT INTO ValidatorManifests (RawData) VALUES (:rawData);";
+    // soci does not support bulk insertion of blob data
+    soci::blob rawData(*db);
+    for (auto const& v : map_)
+    {
+        if (!v.second.m)
+            continue;
+
+        convert (v.second.m->serialized, rawData);
+        *db << sql,
+            soci::use (rawData);
+    }
+    tr.commit ();
+}
 }

src/ripple/overlay/impl/Manifest.h

@@ -34,17 +34,15 @@ namespace ripple {
     Validator key manifests
     -----------------------
 
-    First, a rationale:  Suppose a system adminstrator leaves the company.
+    Suppose the secret keys installed on a Ripple validator are compromised.  Not
-    You err on the side of caution (if not paranoia) and assume that the
+    only do you have to generate and install new key pairs on each validator,
-    secret keys installed on Ripple validators are compromised.  Not only
+    EVERY rippled needs to have its config updated with the new public keys, and
-    do you have to generate and install new key pairs on each validator,
+    is vulnerable to forged validation signatures until this is done.  The
-    EVERY rippled needs to have its config updated with the new public keys,
+    solution is a new layer of indirection: A master secret key under
-    and is vulnerable to forged validation signatures until this is done.
-    The solution is a new layer of indirection:  A master secret key under
     restrictive access control is used to sign a "manifest": essentially, a
     certificate including the master public key, an ephemeral public key for
-    verifying validations (which will be signed by its secret counterpart),
+    verifying validations (which will be signed by its secret counterpart), a
-    a sequence number, and a digital signature.
+    sequence number, and a digital signature.
 
     The manifest has two serialized forms: one which includes the digital
     signature and one which doesn't.  There is an obvious causal dependency
@@ -89,22 +87,16 @@ struct Manifest
     std::string serialized;
     AnyPublicKey masterKey;
     AnyPublicKey signingKey;
-    std::uint32_t seq;
+    std::uint32_t sequence;
 
-    Manifest(std::string s, AnyPublicKey pk, AnyPublicKey spk, std::uint32_t seq)
+    Manifest(std::string s, AnyPublicKey pk, AnyPublicKey spk, std::uint32_t seq);
-        : serialized(std::move(s))
-        , masterKey(std::move(pk))
-        , signingKey(std::move(spk))
-        , seq(seq)
-    {
-    }
 
 #ifdef _MSC_VER
     Manifest(Manifest&& other)
       : serialized(std::move(other.serialized))
       , masterKey(std::move(other.masterKey))
       , signingKey(std::move(other.signingKey))
-      , seq(other.seq)
+      , sequence(other.sequence)
     {
     }
 
@@ -113,26 +105,42 @@ struct Manifest
         serialized = std::move(other.serialized);
         masterKey = std::move(other.masterKey);
         signingKey = std::move(other.signingKey);
-        seq = other.seq;
+        sequence = other.sequence;
         return *this;
     }
 #else
     Manifest(Manifest&& other) = default;
     Manifest& operator=(Manifest&& other) = default;
 #endif
+
+    bool verify () const;
+    uint256 hash () const;
+    bool revoked () const;
 };
 
+boost::optional<Manifest> make_Manifest(std::string s);
+
+inline bool operator==(Manifest const& lhs, Manifest const& rhs)
+{
+    return lhs.serialized == rhs.serialized && lhs.masterKey == rhs.masterKey &&
+           lhs.signingKey == rhs.signingKey && lhs.sequence == rhs.sequence;
+}
+
+inline bool operator!=(Manifest const& lhs, Manifest const& rhs)
+{
+    return !(lhs == rhs);
+}
+
 enum class ManifestDisposition
 {
-    accepted = 0,  // everything checked out
+    accepted = 0, // everything checked out
 
-    malformed,   // deserialization fails
+    untrusted,    // manifest declares a master key we don't trust
-    incomplete,  // fields are missing
+    stale,        // trusted master key, but seq is too old
-    untrusted,   // manifest declares a master key we don't trust
+    invalid,      // trusted and timely, but invalid signature
-    stale,       // trusted master key, but seq is too old
-    invalid,     // trusted and timely, but invalid signature
 };
 
+class DatabaseCon;
 /** Remembers manifests with the highest sequence number. */
 class ManifestCache
 {
@@ -140,6 +148,30 @@ private:
     struct MappedType
     {
         MappedType() = default;
+#ifdef _MSC_VER
+        MappedType(MappedType&& rhs)
+            :comment (std::move (rhs.comment))
+            , m (std::move (rhs.m))
+        {
+        }
+        MappedType& operator=(MappedType&& rhs)
+        {
+            comment = std::move (rhs.comment);
+            m = std::move (rhs.m);
+            return *this;
+        }
+#else
+        MappedType(MappedType&&) = default;
+        MappedType& operator=(MappedType&&) = default;
+#endif
+        MappedType(std::string comment,
+                   std::string serialized,
+                   AnyPublicKey pk, AnyPublicKey spk, std::uint32_t seq)
+            :comment (std::move(comment))
+        {
+            m.emplace (std::move(serialized), std::move(pk), std::move(spk),
+                       seq);
+        }
 
         std::string comment;
         boost::optional<Manifest> m;
@@ -151,7 +183,7 @@ private:
     MapType map_;
 
     ManifestDisposition
-    preflightManifest_locked (AnyPublicKey const& pk, std::uint32_t seq,
+    canApply (AnyPublicKey const& pk, std::uint32_t seq,
         beast::Journal const& journal) const;
 
 public:
@@ -161,12 +193,15 @@ public:
     ~ManifestCache() = default;
 
     void configValidatorKey(std::string const& line, beast::Journal const& journal);
-    void configManifest(std::string s, beast::Journal const& journal);
+    void configManifest(Manifest m, beast::Journal const& journal);
 
-    void addTrustedKey (AnyPublicKey const& pk, std::string const& comment);
+    void addTrustedKey (AnyPublicKey const& pk, std::string comment);
 
     ManifestDisposition
-    applyManifest (std::string s, beast::Journal const& journal);
+    applyManifest (Manifest m, beast::Journal const& journal);
+
+    void load (DatabaseCon& dbCon, beast::Journal const& journal);
+    void save (DatabaseCon& dbCon) const;
 
     // A "for_each" for populated manifests only
     template <class Function>
@@ -180,6 +215,22 @@ public:
                 f(*m);
         }
     }
+
+    // A "for_each" for populated manifests only
+    // The PreFun is called with the maximum number of
+    // times EachFun will be called (useful for memory allocations)
+    template <class PreFun, class EachFun>
+    void
+    for_each_manifest(PreFun&& pf, EachFun&& f) const
+    {
+        std::lock_guard<std::mutex> lock (mutex_);
+        pf(map_.size ());
+        for (auto const& e : map_)
+        {
+            if (auto const& m = e.second.m)
+                f(*m);
+        }
+    }
 };
 
 } // ripple

src/ripple/overlay/impl/OverlayImpl.cpp

@@ -19,6 +19,7 @@
 
 #include <BeastConfig.h>
 #include <ripple/app/misc/IHashRouter.h>
+#include <ripple/core/DatabaseCon.h>
 #include <ripple/basics/Log.h>
 #include <ripple/basics/make_SSLContext.h>
 #include <ripple/protocol/JsonFields.h>
@@ -426,24 +427,24 @@ OverlayImpl::checkStopped ()
         stopped();
 }
 
-static
 void
-prepareValidatorKeyManifests (ManifestCache& mc, beast::Journal const& journal)
+OverlayImpl::setupValidatorKeyManifests (BasicConfig const& config,
+                                         DatabaseCon& db)
 {
-    auto const validator_keys = getConfig().section("validator_keys");
+    auto const validator_keys = config.section ("validator_keys");
-    auto const validation_manifest = getConfig().section("validation_manifest");
+    auto const validation_manifest = config.section ("validation_manifest");
 
     if (! validator_keys.lines().empty())
     {
         for (auto const& line : validator_keys.lines())
         {
-            mc.configValidatorKey (line, journal);
+            manifestCache_.configValidatorKey (line, journal_);
         }
     }
     else
     {
-        if (journal.warning)
+        if (journal_.warning)
-            journal.warning << "[validator_keys] is empty";
+            journal_.warning << "[validator_keys] is empty";
     }
 
     if (! validation_manifest.lines().empty())
@@ -452,21 +453,33 @@ prepareValidatorKeyManifests (ManifestCache& mc, beast::Journal const& journal)
         for (auto const& line : validation_manifest.lines())
             s += beast::rfc2616::trim(line);
         s = beast::base64_decode(s);
-        mc.configManifest(std::move(s), journal);
+        if (auto mo = make_Manifest (std::move (s)))
+        {
+            manifestCache_.configManifest (std::move (*mo), journal_);
+        }
+        else
+        {
+            throw std::runtime_error("Malformed manifest in config");
+        }
     }
     else
     {
-        if (journal.warning)
+        if (journal_.warning)
-            journal.warning << "No [validation_manifest] section in config";
+            journal_.warning << "No [validation_manifest] section in config";
     }
 
+    manifestCache_.load (db, journal_);
+}
+
+void
+OverlayImpl::saveValidatorKeyManifests (DatabaseCon& db) const
+{
+    manifestCache_.save (db);
 }
 
 void
 OverlayImpl::onPrepare()
 {
-    prepareValidatorKeyManifests (manifestCache_, journal_);
-
     PeerFinder::Config config;
 
     if (getConfig ().PEERS_MAX != 0)
@@ -617,53 +630,74 @@ OverlayImpl::onPeerDeactivate (Peer::id_t id,
 
 void
 OverlayImpl::onManifests (Job&,
-    std::shared_ptr<protocol::TMManifests> const& inbox,
+    std::shared_ptr<protocol::TMManifests> const& m,
         std::shared_ptr<PeerImp> const& from)
 {
     auto& hashRouter = getApp().getHashRouter();
-    auto const n = inbox->list_size();
+    auto const n = m->list_size();
     auto const& journal = from->pjournal();
 
     if (journal.debug) journal.debug
         << "TMManifest, " << n << (n == 1 ? " item" : " items");
 
-    protocol::TMManifests outbox;
+    bool const history = m->history ();
-
     for (std::size_t i = 0; i < n; ++i)
     {
-        auto& s = inbox->list().Get(i).stobject();
+        auto& s = m->list ().Get (i).stobject ();
 
-        uint256 const hash = getSHA512Half (s);
+        if (auto mo = make_Manifest (s))
-
-        auto const result = manifestCache_.applyManifest (s, journal);
-
-        if (result == ManifestDisposition::accepted)
         {
-            protocol::TMManifests outbox;
+            uint256 const hash = mo->hash ();
+            if (!hashRouter.addSuppressionPeer (hash, from->id ()))
+                continue;
 
-            outbox.add_list()->set_stobject(s);
+            auto const result =
+                manifestCache_.applyManifest (std::move (*mo), journal);
 
-            auto const msg = std::make_shared<Message>(outbox, protocol::mtMANIFESTS);
+            if (result == ManifestDisposition::accepted)
+            {
+                auto db = getApp ().getWalletDB ().checkoutDb ();
 
-			for_each( [&](std::shared_ptr<PeerImp> const& peer)
+                soci::transaction tr(*db);
-			{
+                static const char* const sql =
-				if (hashRouter.addSuppressionPeer (hash, peer->id())  &&  peer != from)
+                        "INSERT INTO ValidatorManifests (RawData) VALUES (:rawData);";
-				{
+                soci::blob rawData(*db);
-					if (auto& j = peer->pjournal().warning)
+                convert (mo->serialized, rawData);
-					    j << "Forwarding manifest with hash " << hash;
+                *db << sql, soci::use (rawData);
-					peer->send(msg);
+                tr.commit ();
-				}
+            }
-				else if (peer != from)
+
-				{
+            if (history)
-					if (auto& j = peer->pjournal().warning)
+            {
-					    j << "Suppressed manifest with hash " << hash;
+                // Historical manifests are sent on initial peer connections.
-				}
+                // They do not need to be forwarded to other peers.
-			});
+                std::set<Peer::id_t> peers;
+                hashRouter.swapSet (hash, peers, SF_RELAYED);
+                continue;
+            }
+
+            if (result == ManifestDisposition::accepted)
+            {
+                protocol::TMManifests o;
+                o.add_list ()->set_stobject (s);
+
+                std::set<Peer::id_t> peers;
+                hashRouter.swapSet (hash, peers, SF_RELAYED);
+                foreach (send_if_not (
+                    std::make_shared<Message>(o, protocol::mtMANIFESTS),
+                    peer_in_set (peers)));
+            }
+            else
+            {
+                if (journal.info)
+                    journal.info << "Bad manifest #" << i + 1;
+            }
         }
         else
         {
-            if (journal.info) journal.info
+            if (journal.warning)
-                << "Bad manifest #" << i + 1;
+                journal.warning << "Malformed manifest #" << i + 1;
+            continue;
         }
     }
 }

src/ripple/overlay/impl/OverlayImpl.h

@@ -192,6 +192,15 @@ public:
     relay (protocol::TMValidation& m,
         uint256 const& uid) override;
 
+    virtual
+    void
+    setupValidatorKeyManifests (BasicConfig const& config,
+                                DatabaseCon& db) override;
+
+    virtual
+    void
+    saveValidatorKeyManifests (DatabaseCon& db) const override;
+
     //--------------------------------------------------------------------------
     //
     // OverlayImpl

src/ripple/overlay/impl/PeerImp.cpp

@@ -655,9 +655,11 @@ PeerImp::doProtocolStart()
     onReadMessage(error_code(), 0);
 
     protocol::TMManifests tm;
+    tm.set_history (true);
 
-    overlay_.manifestCache().for_each_manifest(
+    overlay_.manifestCache ().for_each_manifest (
-        [&](Manifest const& manifest)
+        [&tm](size_t s){tm.mutable_list()->Reserve(s);},
+        [&tm](Manifest const& manifest)
         {
             auto const& s = manifest.serialized;
             auto& tm_e = *tm.add_list();

src/ripple/overlay/tests/manifest_test.cpp

@@ -20,21 +20,76 @@
 #include <BeastConfig.h>
 #include <ripple/basics/TestSuite.h>
 #include <ripple/overlay/impl/Manifest.h>
+#include <ripple/core/DatabaseCon.h>
+#include <ripple/app/main/DBInit.h>
 #include <ripple/protocol/Sign.h>
 #include <ripple/protocol/STExchange.h>
+#include <boost/filesystem.hpp>
+#include <boost/algorithm/string.hpp>
 
 namespace ripple {
 namespace tests {
 
 class manifest_test : public ripple::TestSuite
 {
+private:
+    static void cleanupDatabaseDir (boost::filesystem::path const& dbPath)
+    {
+        using namespace boost::filesystem;
+        if (!exists (dbPath) || !is_directory (dbPath) || !is_empty (dbPath))
+            return;
+        remove (dbPath);
+    }
+
+    static void setupDatabaseDir (boost::filesystem::path const& dbPath)
+    {
+        using namespace boost::filesystem;
+        if (!exists (dbPath))
+        {
+            create_directory (dbPath);
+            return;
+        }
+
+        if (!is_directory (dbPath))
+        {
+            // someone created a file where we want to put our directory
+            throw std::runtime_error ("Cannot create directory: " +
+                                      dbPath.string ());
+        }
+    }
+    static boost::filesystem::path getDatabasePath ()
+    {
+        return boost::filesystem::current_path () / "manifest_test_databases";
+    }
 public:
-    // Return a manifest in both serialized and STObject form
+    manifest_test ()
-    std::string
+    {
-    make_manifest(AnySecretKey const& sk, AnyPublicKey const& spk, int seq, bool broken = false)
+        try
+        {
+            setupDatabaseDir (getDatabasePath ());
+        }
+        catch (...)
+        {
+        }
+    }
+    ~manifest_test ()
+    {
+        try
+        {
+            cleanupDatabaseDir (getDatabasePath ());
+        }
+        catch (...)
+        {
+        }
+    }
+
+    Manifest
+    make_Manifest
+        (AnySecretKey const& sk, AnyPublicKey const& spk, int seq,
+         bool broken = false)
     {
         auto const pk = sk.publicKey();
-        
+
         STObject st(sfGeneric);
         set(st, sfSequence, seq);
         set(st, sfPublicKey, pk);
@@ -52,53 +107,125 @@ public:
         st.add(s);
 
         std::string const m (static_cast<char const*> (s.data()), s.size());
-        return m;
+        if (auto r = ripple::make_Manifest (std::move (m)))
+        {
+            return std::move (*r);
+        }
+        throw std::runtime_error("Could not create a manifest");
+    }
+
+    Manifest
+    clone (Manifest const& m)
+    {
+        return Manifest (m.serialized, m.masterKey, m.signingKey, m.sequence);
+    }
+
+    void testLoadStore (ManifestCache const& m)
+    {
+        testcase ("load/store");
+
+        std::string const dbName("ManifestCacheTestDB");
+        { 
+            // create a database, save the manifest to the db and reload and
+            // check that the manifest caches are the same
+            DatabaseCon::Setup setup;
+            setup.dataDir = getDatabasePath ();
+            DatabaseCon dbCon(setup, dbName, WalletDBInit, WalletDBCount);
+
+            m.save (dbCon);
+
+            ManifestCache loaded;
+            beast::Journal journal;
+            loaded.load (dbCon, journal);
+
+            auto getPopulatedManifests =
+                    [](ManifestCache const& cache) -> std::vector<Manifest const*>
+                    {
+                        std::vector<Manifest const*> result;
+                        result.reserve (32);
+                        cache.for_each_manifest (
+                            [&result](Manifest const& m)
+            {result.push_back (&m);});
+                        return result;
+                    };
+            auto sort =
+                    [](std::vector<Manifest const*> mv) -> std::vector<Manifest const*>
+                    {
+                        std::sort (mv.begin (),
+                                   mv.end (),
+                                   [](Manifest const* lhs, Manifest const* rhs)
+            {return lhs->serialized < rhs->serialized;});
+                        return mv;
+                    };
+            std::vector<Manifest const*> const inManifests (
+                sort (getPopulatedManifests (m)));
+            std::vector<Manifest const*> const loadedManifests (
+                sort (getPopulatedManifests (loaded)));
+            if (inManifests.size () == loadedManifests.size ())
+            {
+                expect (std::equal
+                        (inManifests.begin (), inManifests.end (),
+                         loadedManifests.begin (),
+                         [](Manifest const* lhs, Manifest const* rhs)
+                         {return *lhs == *rhs;}));
+            }
+            else
+            {
+                fail ();
+            }
+        }
+        boost::filesystem::remove (getDatabasePath () /
+                                   boost::filesystem::path (dbName));
     }
 
     void
     run() override
     {
-        auto const accepted  = ManifestDisposition::accepted;
-        auto const malformed = ManifestDisposition::malformed;
-        auto const untrusted = ManifestDisposition::untrusted;
-        auto const stale     = ManifestDisposition::stale;
-        auto const invalid   = ManifestDisposition::invalid;
-
-        beast::Journal journal;
-
-        auto const sk_a = AnySecretKey::make_ed25519();
-        auto const sk_b = AnySecretKey::make_ed25519();
-        auto const pk_a = sk_a.publicKey();
-        auto const pk_b = sk_b.publicKey();
-        auto const kp_a = AnySecretKey::make_secp256k1_pair();
-        auto const kp_b = AnySecretKey::make_secp256k1_pair();
-
-        auto const s_a0 = make_manifest(sk_a, kp_a.second, 0);
-        auto const s_a1 = make_manifest(sk_a, kp_a.second, 1);
-        auto const s_b0 = make_manifest(sk_b, kp_b.second, 0);
-        auto const s_b1 = make_manifest(sk_b, kp_b.second, 1);
-        auto const s_b2 = make_manifest(sk_b, kp_b.second, 2, true);  // broken
-        auto const fake = s_b1 + '\0';
-
         ManifestCache cache;
+        {
+            testcase ("apply");
+            auto const accepted = ManifestDisposition::accepted;
+            auto const untrusted = ManifestDisposition::untrusted;
+            auto const stale = ManifestDisposition::stale;
+            auto const invalid = ManifestDisposition::invalid;
 
-        expect(cache.applyManifest(s_a0, journal) == untrusted, "have to install a trusted key first");
+            beast::Journal journal;
 
-        cache.addTrustedKey(pk_a, "a");
+            auto const sk_a = AnySecretKey::make_ed25519 ();
-        cache.addTrustedKey(pk_b, "b");
+            auto const sk_b = AnySecretKey::make_ed25519 ();
+            auto const pk_a = sk_a.publicKey ();
+            auto const pk_b = sk_b.publicKey ();
+            auto const kp_a = AnySecretKey::make_secp256k1_pair ();
+            auto const kp_b = AnySecretKey::make_secp256k1_pair ();
 
-        expect(cache.applyManifest(s_a0, journal) == accepted);
+            auto const s_a0 = make_Manifest (sk_a, kp_a.second, 0);
-        expect(cache.applyManifest(s_a0, journal) == stale);
+            auto const s_a1 = make_Manifest (sk_a, kp_a.second, 1);
+            auto const s_b0 = make_Manifest (sk_b, kp_b.second, 0);
+            auto const s_b1 = make_Manifest (sk_b, kp_b.second, 1);
+            auto const s_b2 =
+                make_Manifest (sk_b, kp_b.second, 2, true);  // broken
+            auto const fake = s_b1.serialized + '\0';
 
-        expect(cache.applyManifest(s_a1, journal) == accepted);
+            expect (cache.applyManifest (clone (s_a0), journal) == untrusted,
-        expect(cache.applyManifest(s_a1, journal) == stale);
+                    "have to install a trusted key first");
-        expect(cache.applyManifest(s_a0, journal) == stale);
 
-        expect(cache.applyManifest(s_b0, journal) == accepted);
+            cache.addTrustedKey (pk_a, "a");
-        expect(cache.applyManifest(s_b0, journal) == stale);
+            cache.addTrustedKey (pk_b, "b");
 
-        expect(cache.applyManifest(fake, journal) == malformed);
+            expect (cache.applyManifest (clone (s_a0), journal) == accepted);
-        expect(cache.applyManifest(s_b2, journal) == invalid);
+            expect (cache.applyManifest (clone (s_a0), journal) == stale);
+
+            expect (cache.applyManifest (clone (s_a1), journal) == accepted);
+            expect (cache.applyManifest (clone (s_a1), journal) == stale);
+            expect (cache.applyManifest (clone (s_a0), journal) == stale);
+
+            expect (cache.applyManifest (clone (s_b0), journal) == accepted);
+            expect (cache.applyManifest (clone (s_b0), journal) == stale);
+
+            expect (!ripple::make_Manifest(fake));
+            expect (cache.applyManifest (clone (s_b2), journal) == invalid);
+        }
+        testLoadStore (cache);
     }
 };
 

src/ripple/proto/ripple.proto

@@ -44,6 +44,9 @@ message TMManifest
 message TMManifests
 {
     repeated TMManifest list = 1;
+    // The manifests sent when a peer first connects to another peer are `history`.
+    // The receiving peer does not forward them.
+    optional bool history = 2 [default = false];
 }
 
 //------------------------------------------------------------------------------

src/ripple/protocol/AnyPublicKey.h

@@ -123,7 +123,8 @@ public:
     AnyPublicKey& operator= (AnyPublicKey&& other)
     {
         buffer_type::member =
-            std::move(other.buffer_type::member);
+            std::move (other.buffer_type::member);
+        AnyPublicKeySlice::operator= (other);
         return *this;
     }
 #else
@@ -179,6 +180,9 @@ struct STExchange<STBlob, AnyPublicKey>
     }
 };
 
+std::string
+toString (AnyPublicKey const& pk);
+
 } // ripple
 
 #endif

src/ripple/protocol/impl/AnyPublicKey.cpp

@@ -90,4 +90,15 @@ AnyPublicKeySlice::verify (
     return false;
 }
 
+std::string
+toString (AnyPublicKey const& pk)
+{
+    Blob buffer;
+    buffer.reserve (1 + pk.size ());
+    buffer.push_back (VER_NODE_PUBLIC);
+    auto const data = pk.data ();
+    buffer.insert (buffer.end (), data, data + pk.size ());
+    return Base58::encodeWithCheck (buffer);
+}
+
 } // ripple

src/ripple/protocol/impl/HashPrefix.cpp

@@ -34,6 +34,6 @@ HashPrefix const HashPrefix::txSign                      ('S', 'T', 'X');
 HashPrefix const HashPrefix::txMultiSign                 ('S', 'M', 'T');
 HashPrefix const HashPrefix::validation                  ('V', 'A', 'L');
 HashPrefix const HashPrefix::proposal                    ('P', 'R', 'P');
-HashPrefix const HashPrefix::manifest                   ('M', 'A', 'N');
+HashPrefix const HashPrefix::manifest                    ('M', 'A', 'N');
 
 } // ripple