ARCHIVE/rippled
1
0
Fork 0
mirror of https://github.com/XRPLF/rippled.git synced 2025-11-18 10:05:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Tidying & Selectively forward manifests to peers:

Browse Source 
* Do not forward manifests to peers that already know that manifest
* Do not forward historical manifests to peers
* Save/Load ValidatorManifests from a database
* Python test for setting ephmeral keys
* Cleanup manifest interface
This commit is contained in:
seelabs
2015-04-30 13:27:35 -07:00
parent 31d352b3aa
commit 1b4e0f5f48
18 changed files with 1382 additions and 221 deletions
Download Patch File Download Diff File Expand all files Collapse all files

682
bin/python/ripple/util/ValidatorManifestTest.py Executable file
View File

@@ -0,0 +1,682 @@
#!/usr/bin/env python
"""
Test for setting ephemeral keys for the validator manifest.
"""
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
import argparse
import contextlib
from contextlib import contextmanager
import json
import os
import platform
import shutil
import subprocess
import time
DELAY_WHILE_PROCESS_STARTS_UP = 1.5
ARGS = None
NOT_FOUND = -1 # not in log
ACCEPTED_NEW = 0 # added new manifest
ACCEPTED_UPDATE = 1 # replaced old manifest with new
UNTRUSTED = 2 # don't trust master key
STALE = 3 # seq is too old
REVOKED = 4 # revoked validator key
INVALID = 5 # invalid signature
MANIFEST_ACTION_STR_TO_ID = {
'NotFound': NOT_FOUND, # not found in log
'AcceptedNew': ACCEPTED_NEW,
'AcceptedUpdate': ACCEPTED_UPDATE,
'Untrusted': UNTRUSTED,
'Stale': STALE,
'Revoked': REVOKED,
'Invalid': INVALID,
}
MANIFEST_ACTION_ID_TO_STR = {
v: k for k, v in MANIFEST_ACTION_STR_TO_ID.items()
}
CONF_TEMPLATE = """
[server]
port_rpc
port_peer
port_wss_admin
[port_rpc]
port = {rpc_port}
ip = 127.0.0.1
admin = 127.0.0.1
protocol = https
[port_peer]
port = {peer_port}
ip = 0.0.0.0
protocol = peer
[port_wss_admin]
port = {wss_port}
ip = 127.0.0.1
admin = 127.0.0.1
protocol = wss
[node_size]
medium
[node_db]
type={node_db_type}
path={node_db_path}
open_files=2000
filter_bits=12
cache_mb=256
file_size_mb=8
file_size_mult=2
online_delete=256
advisory_delete=0
[database_path]
{db_path}
[debug_logfile]
{debug_logfile}
[sntp_servers]
time.windows.com
time.apple.com
time.nist.gov
pool.ntp.org
[ips]
r.ripple.com 51235
[ips_fixed]
{sibling_ip} {sibling_port}
[validators]
n949f75evCHwgyP4fPVgaHqNHxUVN15PsJEZ3B3HnXPcPjcZAoy7 RL1
n9MD5h24qrQqiyBC8aeqqCWvpiBiYQ3jxSr91uiDvmrkyHRdYLUj RL2
n9L81uNCaPgtUJfaHh89gmdvXKAmSt5Gdsw2g1iPWaPkAHW5Nm4C RL3
n9KiYM9CgngLvtRCQHZwgC2gjpdaZcCcbt3VboxiNFcKuwFVujzS RL4
n9LdgEtkmGB9E2h3K4Vp7iGUaKuq23Zr32ehxiU8FWY7xoxbWTSA RL5
[validation_quorum]
3
[validation_seed]
{validation_seed}
#vaidation_public_key: {validation_public_key}
# Other rippled's trusting this validator need this key
[validator_keys]
{all_validator_keys}
[peer_private]
1
[overlay]
expire = 1
auto_connect = 1
[validation_manifest]
{validation_manifest}
[rpc_startup]
{{ "command": "log_level", "severity": "debug" }}
[ssl_verify]
0
"""
# End config template
def static_vars(**kwargs):
def decorate(func):
for k in kwargs:
setattr(func, k, kwargs[k])
return func
return decorate
@static_vars(rpc=5005, peer=51235, wss=6006)
def checkout_port_nums():
"""Returns a tuple of port nums for rpc, peer, and wss_admin"""
checkout_port_nums.rpc += 1
checkout_port_nums.peer += 1
checkout_port_nums.wss += 1
return (
checkout_port_nums.rpc,
checkout_port_nums.peer,
checkout_port_nums.wss
)
def is_windows():
return platform.system() == 'Windows'
def manifest_create():
"""returns dict with keys: 'validator_keys', 'master_secret'"""
to_run = ['python', ARGS.ripple_home + '/bin/python/Manifest.py', 'create']
r = subprocess.check_output(to_run)
result = {}
k = None
for l in r.splitlines():
l = l.strip()
if not l:
continue
elif l == '[validator_keys]':
k = l[1:-1]
elif l == '[master_secret]':
k = l[1:-1]
elif l.startswith('['):
raise ValueError(
'Unexpected key: {} from `manifest create`'.format(l))
else:
if not k:
raise ValueError('Value with no key')
result[k] = l
k = None
if k in result:
raise ValueError('Repeat key from `manifest create`: ' + k)
if len(result) != 2:
raise ValueError(
'Expected 2 keys from `manifest create` but got {} keys instead ({})'.
format(len(result), result))
return result
def sign_manifest(seq, validation_pk, master_secret):
"""returns the signed manifest as a string"""
to_run = ['python', ARGS.ripple_home + '/bin/python/Manifest.py', 'sign',
str(seq), validation_pk, master_secret]
try:
r = subprocess.check_output(to_run)
except subprocess.CalledProcessError as e:
print('Error in sign_manifest: ', e.output)
raise e
result = []
for l in r.splitlines():
l.strip()
if not l or l == '[validation_manifest]':
continue
result.append(l)
return '\n'.join(result)
def get_ripple_exe():
"""Find the rippled executable"""
prefix = ARGS.ripple_home + '/build/'
exe = ['rippled', 'RippleD.exe']
to_test = [prefix + t + '.debug/' + e
for t in ['clang', 'gcc', 'msvc'] for e in exe]
for e in exe:
to_test.append(prefix + '/' + e)
for t in to_test:
if os.path.isfile(t):
return t
class RippledServer(object):
def __init__(self, exe, config_file, server_out):
self.config_file = config_file
self.exe = exe
self.process = None
self.server_out = server_out
self.reinit(config_file)
def reinit(self, config_file):
self.config_file = config_file
self.to_run = [self.exe, '--verbose', '--conf', self.config_file]
@property
def config_root(self):
return os.path.dirname(self.config_file)
@property
def master_secret_file(self):
return self.config_root + '/master_secret.txt'
def startup(self):
if ARGS.verbose:
print('starting rippled:' + self.config_file)
fout = open(self.server_out, 'w')
self.process = subprocess.Popen(
self.to_run, stdout=fout, stderr=subprocess.STDOUT)
def shutdown(self):
if not self.process:
return
fout = open(os.devnull, 'w')
subprocess.Popen(
self.to_run + ['stop'], stdout=fout, stderr=subprocess.STDOUT)
self.process.wait()
self.process = None
def rotate_logfile(self):
if self.server_out == os.devnull:
return
for i in range(100):
backup_name = '{}.{}'.format(self.server_out, i)
if not os.path.exists(backup_name):
os.rename(self.server_out, backup_name)
return
raise ValueError('Could not rotate logfile: {}'.
format(self.server_out))
def validation_create(self):
"""returns dict with keys:
'validation_key', 'validation_public_key', 'validation_seed'
"""
to_run = [self.exe, '-q', '--conf', self.config_file,
'--', 'validation_create']
try:
return json.loads(subprocess.check_output(to_run))['result']
except subprocess.CalledProcessError as e:
print('Error in validation_create: ', e.output)
raise e
@contextmanager
def rippled_server(config_file, server_out=os.devnull):
"""Start a ripple server"""
try:
server = None
server = RippledServer(ARGS.ripple_exe, config_file, server_out)
server.startup()
yield server
finally:
if server:
server.shutdown()
@contextmanager
def pause_server(server, config_file):
"""Shutdown and then restart a ripple server"""
try:
server.shutdown()
server.rotate_logfile()
yield server
finally:
server.reinit(config_file)
server.startup()
def parse_date(d, t):
"""Return the timestamp of a line, or none if the line has no timestamp"""
try:
return time.strptime(d+' '+t, '%Y-%B-%d %H:%M:%S')
except:
return None
def to_dict(l):
"""Given a line of the form Key0: Value0;Key2: Valuue2; Return a dict"""
fields = l.split(';')
result = {}
for f in fields:
if f:
v = f.split(':')
assert len(v) == 2
result[v[0].strip()] = v[1].strip()
return result
def check_ephemeral_key(validator_key,
log_file,
seq,
change_time):
"""
Detect when a server is informed of a validator's ephemeral key change.
`change_time` and `seq` may be None, in which case they are ignored.
"""
manifest_prefix = 'Manifest:'
# a manifest line has the form Manifest: action; Key: value;
# Key can be Pk (public key), Seq, OldSeq,
for l in open(log_file):
sa = l.split()
if len(sa) < 5 or sa[4] != manifest_prefix:
continue
d = to_dict(' '.join(sa[4:]))
# check the seq number and validator_key
if d['Pk'] != validator_key:
continue
if seq is not None and int(d['Seq']) != seq:
continue
if change_time:
t = parse_date(sa[0], sa[1])
if not t or t < change_time:
continue
action = d['Manifest']
return MANIFEST_ACTION_STR_TO_ID[action]
return NOT_FOUND
def check_ephemeral_keys(validator_key,
log_files,
seq,
change_time=None,
timeout_s=60):
result = [NOT_FOUND for i in range(len(log_files))]
if timeout_s < 10:
sleep_time = 1
elif timeout_s < 60:
sleep_time = 5
else:
sleep_time = 10
n = timeout_s//sleep_time
if n == 0:
n = 1
start_time = time.time()
for _ in range(n):
for i, lf in enumerate(log_files):
if result[i] != NOT_FOUND:
continue
result[i] = check_ephemeral_key(validator_key,
lf,
seq,
change_time)
if result[i] != NOT_FOUND:
if all(r != NOT_FOUND for r in result):
return result
else:
server_dir = os.path.basename(os.path.dirname(log_files[i]))
if ARGS.verbose:
print('Check for {}: {}'.format(
server_dir, MANIFEST_ACTION_ID_TO_STR[result[i]]))
tsf = time.time() - start_time
if tsf > 20:
if ARGS.verbose:
print('Waiting for key to propigate: ', tsf)
time.sleep(sleep_time)
return result
def get_validator_key(config_file):
in_validator_keys = False
for l in open(config_file):
sl = l.strip()
if not in_validator_keys and sl == '[validator_keys]':
in_validator_keys = True
continue
if in_validator_keys:
if sl.startswith('['):
raise ValueError('ThisServer validator key not found')
if sl.startswith('#'):
continue
s = sl.split()
if len(s) == 2 and s[1] == 'ThisServer':
return s[0]
def new_config_ephemeral_key(
server, seq, rm_dbs=False, master_secret_file=None):
"""Generate a new ephemeral key, add to config, restart server"""
config_root = server.config_root
config_file = config_root + '/rippled.cfg'
db_dir = config_root + '/db'
if not master_secret_file:
master_secret_file = server.master_secret_file
with open(master_secret_file) as f:
master_secret = f.read()
v = server.validation_create()
signed = sign_manifest(seq, v['validation_public_key'], master_secret)
with pause_server(server, config_file):
if rm_dbs and os.path.exists(db_dir):
shutil.rmtree(db_dir)
os.makedirs(db_dir)
# replace the validation_manifest section with `signed`
bak = config_file + '.bak'
if is_windows() and os.path.isfile(bak):
os.remove(bak)
os.rename(config_file, bak)
in_manifest = False
with open(bak, 'r') as src:
with open(config_file, 'w') as out:
for l in src:
sl = l.strip()
if not in_manifest and sl == '[validation_manifest]':
in_manifest = True
elif in_manifest:
if sl.startswith('[') or sl.startswith('#'):
in_manifest = False
out.write(signed)
out.write('\n\n')
else:
continue
out.write(l)
return (bak, config_file)
def parse_args():
parser = argparse.ArgumentParser(
description=('Create config files for n validators')
)
parser.add_argument(
'--ripple_home', '-r',
default=os.sep.join(os.path.realpath(__file__).split(os.sep)[:-5]),
help=('Root directory of the ripple repo'), )
parser.add_argument('--num_validators', '-n',
default=2,
help=('Number of validators'), )
parser.add_argument('--conf', '-c', help=('rippled config file'), )
parser.add_argument('--out', '-o',
default='test_output',
help=('config root directory'), )
parser.add_argument(
'--existing', '-e',
action='store_true',
help=('use existing config files'), )
parser.add_argument(
'--generate', '-g',
action='store_true',
help=('generate conf files only'), )
parser.add_argument(
'--verbose', '-v',
action='store_true',
help=('verbose status reporting'), )
parser.add_argument(
'--quiet', '-q',
action='store_true',
help=('quiet status reporting'), )
return parser.parse_args()
def get_configs(manifest_seq):
global ARGS
ARGS.ripple_home = os.path.expanduser(ARGS.ripple_home)
n = int(ARGS.num_validators)
if n<2:
raise ValueError(
'Need at least 2 rippled servers. Specified: {}'.format(n))
config_root = ARGS.out
ARGS.ripple_exe = get_ripple_exe()
if not ARGS.ripple_exe:
raise ValueError('No Exe Found')
if ARGS.existing:
return [
os.path.abspath('{}/validator_{}/rippled.cfg'.format(config_root, i))
for i in range(n)
]
initial_config = ARGS.conf
manifests = [manifest_create() for i in range(n)]
port_nums = [checkout_port_nums() for i in range(n)]
with rippled_server(initial_config) as server:
time.sleep(DELAY_WHILE_PROCESS_STARTS_UP)
validations = [server.validation_create() for i in range(n)]
signed_manifests = [sign_manifest(manifest_seq,
v['validation_public_key'],
m['master_secret'])
for m, v in zip(manifests, validations)]
node_db_type = 'RocksDB' if not is_windows() else 'NuDB'
node_db_filename = node_db_type.lower()
config_files = []
for i, (m, v, s) in enumerate(zip(manifests, validations, signed_manifests)):
sibling_index = (i - 1) % len(manifests)
all_validator_keys = '\n'.join([
m['validator_keys'] + ' ThisServer',
manifests[sibling_index]['validator_keys'] + ' NextInRing'])
this_validator_dir = os.path.abspath(
'{}/validator_{}'.format(config_root, i))
db_path = this_validator_dir + '/db'
node_db_path = db_path + '/' + node_db_filename
log_path = this_validator_dir + '/log'
debug_logfile = log_path + '/debug.log'
rpc_port, peer_port, wss_port = port_nums[i]
sibling_ip = '127.0.0.1'
sibling_port = port_nums[sibling_index][1]
d = {
'validation_manifest': s,
'all_validator_keys': all_validator_keys,
'node_db_type': node_db_type,
'node_db_path': node_db_path,
'db_path': db_path,
'debug_logfile': debug_logfile,
'rpc_port': rpc_port,
'peer_port': peer_port,
'wss_port': wss_port,
'sibling_ip': sibling_ip,
'sibling_port': sibling_port,
}
d.update(m)
d.update(v)
for p in [this_validator_dir, db_path, log_path]:
if not os.path.exists(p):
os.makedirs(p)
config_files.append('{}/rippled.cfg'.format(this_validator_dir))
with open(config_files[-1], 'w') as f:
f.write(CONF_TEMPLATE.format(**d))
with open('{}/master_secret.txt'.format(this_validator_dir), 'w') as f:
f.write(m['master_secret'])
return config_files
def update_ephemeral_key(
server, new_seq, log_files,
expected=None, rm_dbs=False, master_secret_file=None,
restore_origional_conf=False, timeout_s=300):
if not expected:
expected = {}
change_time = time.gmtime()
back_conf, new_conf = new_config_ephemeral_key(
server,
new_seq,
rm_dbs,
master_secret_file
)
validator_key = get_validator_key(server.config_file)
start_time = time.time()
ck = check_ephemeral_keys(validator_key,
log_files,
seq=new_seq,
change_time=change_time,
timeout_s=timeout_s)
if ARGS.verbose:
print('Check finished: {} secs.'.format(int(time.time() - start_time)))
all_success = True
for i, r in enumerate(ck):
e = expected.get(i, UNTRUSTED)
server_dir = os.path.basename(os.path.dirname(log_files[i]))
status = 'OK' if e == r else 'FAIL'
print('{}: Server: {} Expected: {} Got: {}'.
format(status, server_dir,
MANIFEST_ACTION_ID_TO_STR[e], MANIFEST_ACTION_ID_TO_STR[r]))
all_success = all_success and (e == r)
if restore_origional_conf:
if is_windows() and os.path.isfile(new_conf):
os.remove(new_conf)
os.rename(back_conf, new_conf)
return all_success
def run_main():
global ARGS
ARGS = parse_args()
manifest_seq = 1
config_files = get_configs(manifest_seq)
if ARGS.generate:
return
if len(config_files) <= 1:
print('Script requires at least 2 servers. Actual #: {}'.
format(len(config_files)))
return
with contextlib.nested(*(rippled_server(c, os.path.dirname(c)+'/log.txt')
for c in config_files)) as servers:
log_files = [os.path.dirname(cf)+'/log.txt' for cf in config_files[1:]]
validator_key = get_validator_key(config_files[0])
start_time = time.time()
ck = check_ephemeral_keys(validator_key,
[log_files[0]],
seq=None,
timeout_s=60)
if ARGS.verbose:
print('Check finished: {} secs.'.format(
int(time.time() - start_time)))
if any(r == NOT_FOUND for r in ck):
print('FAIL: Initial key did not propigate to all servers')
return
manifest_seq += 2
expected = {i: UNTRUSTED for i in range(len(log_files))}
expected[0] = ACCEPTED_UPDATE
if not ARGS.quiet:
print('Testing key update')
kr = update_ephemeral_key(servers[0], manifest_seq, log_files, expected)
if not kr:
print('\nFail: Key Update Test. Exiting')
return
expected = {i: UNTRUSTED for i in range(len(log_files))}
expected[0] = STALE
if not ARGS.quiet:
print('Testing stale key')
kr = update_ephemeral_key(
servers[0], manifest_seq-1, log_files, expected, rm_dbs=True)
if not kr:
print('\nFail: Stale Key Test. Exiting')
return
expected = {i: UNTRUSTED for i in range(len(log_files))}
expected[0] = STALE
if not ARGS.quiet:
print('Testing stale key 2')
kr = update_ephemeral_key(
servers[0], manifest_seq, log_files, expected, rm_dbs=True)
if not kr:
print('\nFail: Stale Key Test. Exiting')
return
expected = {i: UNTRUSTED for i in range(len(log_files))}
expected[0] = REVOKED
if not ARGS.quiet:
print('Testing revoked key')
kr = update_ephemeral_key(
servers[0], 0xffffffff, log_files, expected, rm_dbs=True)
if not kr:
print('\nFail: Revoked Key Text. Exiting')
return
print('\nOK: All tests passed')
if __name__ == '__main__':
run_main()

110
doc/manifest-tool-guide.md Normal file
View File

@@ -0,0 +1,110 @@
# Manifest Tool Guide
This guide explains how to setup a validator so the key pairs used to sign and
verify validations may safely change. This procedure does not require manual
reconfiguration of servers that trust this validator.
Validators use two types of key pairs: *master keys* and *ephemeral
keys*. Ephemeral keys are used to sign and verify validations. Master keys are
used to sign and verify manifests that change ephemeral keys. The master secret
key should be tightly controlled. The ephemeral secret key needs to be present
in the config file.
## Validator Keys
When first setting up a validator, use the `manifest` script to generate a
master key pair:
```
$ bin/manifest create
```
Sample output:
```
[validator_keys]
nHUSSzGw4A9zEmFtK2Q2NcWDH9xmGdXMHc1MsVej3QkLTgvDNeBr
[master_secret]
pnxayCakmZRQE2qhEVRbFjiWCunReSbN1z64vPL36qwyLgogyYc
```
The first value is the master public key. Add the public key to the config
for this validator. A one-word comment must be added after the key (for example
*ThisServersName*). Any other rippled trusting the validator needs to add the
master public key to its config. Only add keys received from trusted sources.
The second value is the corresponding master secret key. **DO NOT INSTALL THIS
IN THE CONFIG**. The master secret key will be used to sign manifests that
change validation keys. Put the master secret key in a secure but recoverable
location.
## Validation Keys
When first setting up a validator, or when changing the ephemeral keys, use the
`rippled` program to create a new ephemeral key pair:
```
$ rippled validation_create
```
Sample output:
```
Loading: "/Users/alice/.config/ripple/rippled.cfg"
Securely connecting to 127.0.0.1:5005
{
"result" : {
"status" : "success",
"validation_key" : "TOO EDNA SHUN FEUD STAB JOAN BIAS FLEA WISE BOHR LOSS WEEK",
"validation_public_key" : "n9JzKV3ZrcZ3DW5pwjakj4hpijJ9oMiyrPDGJc3mpsndL6Gf3zwd",
"validation_seed" : "sahzkAajS2dyhjXg2yovjdZhXmjsx"
}
}
```
Add the `validation_seed` value (the ephemeral secret key) to this validator's
config. It is recommended to add the ephemeral public key and the sequence
number as a comment as well (sequence numbers are be explained below):
```
[validation_seed]
sahzkAajS2dyhjXg2yovjdZhXmjsx
# validation_public_key: n9JzKV3ZrcZ3DW5pwjakj4hpijJ9oMiyrPDGJc3mpsndL6Gf3zwd
# sequence number: 1
```
A manifest is a signed message used to inform other servers of this validator's
ephemeral public key. A manifest contains a sequence number, the new ephemeral
public key, and it is signed with the master secret key. The sequence number
should be higher than the previous sequence number (if it is not, the manifest
will be ignored). Usually the previous sequence number will be incremented by
one. Use the `manifest` script to create a manifest. It has the form:
```
$ bin/manifest sign sequence_number validation_public_key master_secret
```
For example:
```
$ bin/manifest sign 1 n9JzKV3Z...L6Gf3zwd pnxayCak...yLgogyYc
```
Sample output:
```
[validation_manifest]
JAAAAAFxIe2PEzNhe996gykB1PJQNoDxvr/Y0XhDELw8d/i
Fcgz3A3MhAjqhKsgZTmK/3BPEI+kzjV1p9ip7pl/AtF7CKd
NSfAH9dkCxezV6apS4FLYzAcQilONx315HvebwAB/pLPaM4
2sWCEppSuLNKN/JJjTABOo9tmAiNnnstF83yvecKMJzniwN
```
Copy this to the config for this validator. Don't forget to update the comment
noting the sequence number.
## Revoking a key
If a master key is compromised, the key may be revoked permanently. To revoke a
master key, sign a manifest with the highest possible sequence number:
`4,294,967,295`

4
src/ripple/app/main/Application.cpp
View File

@@ -806,6 +806,8 @@ public:
getConfig());
add (*m_overlay); // add to PropertyStream
m_overlay->setupValidatorKeyManifests (getConfig (), getWalletDB ());
{
auto setup = setup_ServerHandler(getConfig(), std::cerr);
setup.makeContexts();
@@ -900,6 +902,8 @@ public:
mValidations->flush ();
m_overlay->saveValidatorKeyManifests (getWalletDB ());
RippleAddress::clearCache ();
stopped ();
}

7
src/ripple/app/main/DBInit.cpp
View File

@@ -231,6 +231,13 @@ const char* WalletDBInit[] =
PRIMARY KEY (Validator,Entry) \
);",
// Validator Manifests
R"(
CREATE TABLE IF NOT EXISTS ValidatorManifests (
RawData BLOB NOT NULL
);
)",
// List of referrals from ripple.txt files.
// Validator:
// Public key of referree.

2
src/ripple/app/misc/UniqueNodeList.h
View File

@@ -20,7 +20,7 @@
#ifndef RIPPLE_APP_PEERS_UNIQUENODELIST_H_INCLUDED
#define RIPPLE_APP_PEERS_UNIQUENODELIST_H_INCLUDED
#include <ripple/app/peers/ClusterNodeStatus.h>
#include <ripple/overlay/ClusterNodeStatus.h>
#include <ripple/protocol/AnyPublicKey.h>
#include <ripple/protocol/RippleAddress.h>
#include <beast/cxx14/memory.h> // <memory>

1
src/ripple/core/SociDB.h
View File

@@ -114,6 +114,7 @@ size_t getKBUsedDB (soci::session& s);
void convert (soci::blob& from, std::vector<std::uint8_t>& to);
void convert (soci::blob& from, std::string& to);
void convert (std::vector<std::uint8_t> const& from, soci::blob& to);
void convert (std::string const& from, soci::blob& to);
class Checkpointer
{

10
src/ripple/core/impl/SociDB.cpp
View File

@@ -160,6 +160,16 @@ void convert (std::vector<std::uint8_t> const& from, soci::blob& to)
{
if (!from.empty ())
to.write (0, reinterpret_cast<char const*>(&from[0]), from.size ());
else
to.trim (0);
}
void convert (std::string const& from, soci::blob& to)
{
if (!from.empty ())
to.write (0, from.data (), from.size ());
else
to.trim (0);
}
namespace {

23
src/ripple/overlay/Overlay.h
View File

@@ -38,6 +38,9 @@ namespace boost { namespace asio { namespace ssl { class context; } } }
namespace ripple {
class DatabaseCon;
class BasicConfig;
/** Manages the set of connected peers. */
class Overlay
: public beast::Stoppable
@@ -156,6 +159,15 @@ public:
relay (protocol::TMValidation& m,
uint256 const& uid) = 0;
virtual
void
setupValidatorKeyManifests (BasicConfig const& config,
DatabaseCon& db) = 0;
virtual
void
saveValidatorKeyManifests (DatabaseCon& db) const = 0;
/** Visit every active peer and return a value
The functor must:
- Be callable as:
@@ -170,12 +182,11 @@ public:
@note The functor is passed by value!
*/
template<typename Function>
std::enable_if_t <
! std::is_void <typename Function::return_type>::value,
typename Function::return_type
>
foreach(Function f)
template <typename UnaryFunc>
std::enable_if_t<! std::is_void<
typename UnaryFunc::return_type>::value,
typename UnaryFunc::return_type>
foreach (UnaryFunc f)
{
PeerSequence peers (getActivePeers());
for(PeerSequence::const_iterator i = peers.begin(); i != peers.end(); ++i)

297
src/ripple/overlay/impl/Manifest.cpp
View File

@@ -18,7 +18,8 @@
//==============================================================================
#include <ripple/app/main/Application.h>
#include <ripple/app/peers/UniqueNodeList.h>
#include <ripple/app/misc/UniqueNodeList.h>
#include <ripple/core/DatabaseCon.h>
#include <ripple/overlay/impl/Manifest.h>
#include <ripple/protocol/RippleAddress.h>
#include <ripple/protocol/Sign.h>
@@ -27,12 +28,102 @@
namespace ripple {
boost::optional<Manifest>
make_Manifest (std::string s)
{
try
{
STObject st (sfGeneric);
SerialIter sit (s.data (), s.size ());
st.set (sit);
auto const opt_pk = get<AnyPublicKey>(st, sfPublicKey);
auto const opt_spk = get<AnyPublicKey>(st, sfSigningPubKey);
auto const opt_seq = get (st, sfSequence);
auto const opt_sig = get (st, sfSignature);
if (!opt_pk || !opt_spk || !opt_seq || !opt_sig)
{
return boost::none;
}
return Manifest (std::move (s), *opt_pk, *opt_spk, *opt_seq);
}
catch (...)
{
return boost::none;
}
}
template<class Stream>
Stream&
logMftAct (
Stream& s,
std::string const& action,
AnyPublicKey const& pk,
std::uint32_t seq)
{
s << "Manifest: " << action <<
";Pk: " << toString (pk) <<
";Seq: " << seq << ";";
return s;
}
template<class Stream>
Stream& logMftAct (
Stream& s,
std::string const& action,
AnyPublicKey const& pk,
std::uint32_t seq,
std::uint32_t oldSeq)
{
s << "Manifest: " << action <<
";Pk: " << toString (pk) <<
";Seq: " << seq <<
";OldSeq: " << oldSeq << ";";
return s;
}
Manifest::Manifest (std::string s,
AnyPublicKey pk,
AnyPublicKey spk,
std::uint32_t seq)
: serialized (std::move (s))
, masterKey (std::move (pk))
, signingKey (std::move (spk))
, sequence (seq)
{
}
bool Manifest::verify () const
{
STObject st (sfGeneric);
SerialIter sit (serialized.data (), serialized.size ());
st.set (sit);
return ripple::verify (st, HashPrefix::manifest, masterKey);
}
uint256 Manifest::hash () const
{
STObject st (sfGeneric);
SerialIter sit (serialized.data (), serialized.size ());
st.set (sit);
return st.getHash (HashPrefix::manifest);
}
bool Manifest::revoked () const
{
/*
The maximum possible sequence number means that the master key
has been revoked.
*/
return sequence == std::numeric_limits<std::uint32_t>::max ();
}
void
ManifestCache::configValidatorKey(std::string const& line, beast::Journal const& journal)
ManifestCache::configValidatorKey(
std::string const& line, beast::Journal const& journal)
{
auto const words = beast::rfc2616::split(line.begin(), line.end(), ' ');
if (words.size() != 2)
if (words.size () != 2)
{
throw std::runtime_error ("[validator_keys] format is `<key> <comment>");
}
@@ -56,44 +147,23 @@ ManifestCache::configValidatorKey(std::string const& line, beast::Journal const&
}
auto const masterKey = AnyPublicKey (key.data() + 1, key.size() - 1);
auto const& comment = words[1];
std::string comment = std::move(words[1]);
if (journal.debug) journal.debug
<< masterKey << " " << comment;
addTrustedKey (masterKey, comment);
addTrustedKey (masterKey, std::move(comment));
}
void
ManifestCache::configManifest (std::string s, beast::Journal const& journal)
ManifestCache::configManifest(Manifest m, beast::Journal const& journal)
{
STObject st(sfGeneric);
try
{
SerialIter sit(s.data(), s.size());
st.set(sit);
}
catch(...)
{
throw std::runtime_error("Malformed manifest in config");
}
auto const mseq = get(st, sfSequence);
auto const msig = get(st, sfSignature);
auto mpk = get<AnyPublicKey>(st, sfPublicKey);
auto mspk = get<AnyPublicKey>(st, sfSigningPubKey);
if (! mseq || ! msig || ! mpk || ! mspk)
{
throw std::runtime_error("Missing fields in manifest in config");
}
auto const& pk = *mpk;
if (! verify(st, HashPrefix::manifest, pk))
if (!m.verify())
{
throw std::runtime_error("Unverifiable manifest in config");
}
auto const result = applyManifest (std::move(s), journal);
auto const result = applyManifest (std::move(m), journal);
if (result != ManifestDisposition::accepted)
{
@@ -102,7 +172,7 @@ ManifestCache::configManifest (std::string s, beast::Journal const& journal)
}
void
ManifestCache::addTrustedKey (AnyPublicKey const& pk, std::string const& comment)
ManifestCache::addTrustedKey (AnyPublicKey const& pk, std::string comment)
{
std::lock_guard<std::mutex> lock (mutex_);
@@ -110,14 +180,15 @@ ManifestCache::addTrustedKey (AnyPublicKey const& pk, std::string const& comment
if (value.m)
{
throw std::runtime_error ("New trusted validator key already has a manifest");
throw std::runtime_error (
"New trusted validator key already has a manifest");
}
value.comment = comment;
value.comment = std::move(comment);
}
ManifestDisposition
ManifestCache::preflightManifest_locked (AnyPublicKey const& pk, std::uint32_t seq,
ManifestCache::canApply (AnyPublicKey const& pk, std::uint32_t seq,
beast::Journal const& journal) const
{
auto const iter = map_.find(pk);
@@ -129,14 +200,14 @@ ManifestCache::preflightManifest_locked (AnyPublicKey const& pk, std::uint32_t s
Since rippled always sends all of its current manifests,
this will happen normally any time a peer connects.
*/
if (journal.debug) journal.debug
<< "Ignoring manifest #" << seq << " from untrusted key " << pk;
if (journal.debug)
logMftAct(journal.debug, "Untrusted", pk, seq);
return ManifestDisposition::untrusted;
}
auto& old = iter->second.m;
if (old && seq <= old->seq)
if (old && seq <= old->sequence)
{
/*
A manifest was received for a validator we're tracking, but
@@ -144,67 +215,41 @@ ManifestCache::preflightManifest_locked (AnyPublicKey const& pk, std::uint32_t s
This will happen normally when a peer without the latest gossip
connects.
*/
if (journal.debug) journal.debug
<< "Ignoring manifest #" << seq
<< "which isn't newer than #" << old->seq;
if (journal.debug)
logMftAct(journal.debug, "Stale", pk, seq, old->sequence);
return ManifestDisposition::stale; // not a newer manifest, ignore
}
return ManifestDisposition::accepted;
}
ManifestDisposition
ManifestCache::applyManifest (std::string s, beast::Journal const& journal)
ManifestCache::applyManifest (Manifest m, beast::Journal const& journal)
{
STObject st(sfGeneric);
try
{
SerialIter sit(s.data(), s.size());
st.set(sit);
}
catch(...)
{
return ManifestDisposition::malformed;
}
auto const opt_pk = get<AnyPublicKey>(st, sfPublicKey);
auto const opt_spk = get<AnyPublicKey>(st, sfSigningPubKey);
auto const opt_seq = get(st, sfSequence);
auto const opt_sig = get(st, sfSignature);
if (! opt_pk || ! opt_spk || ! opt_seq || ! opt_sig)
{
return ManifestDisposition::incomplete;
}
auto const pk = *opt_pk;
auto const spk = *opt_spk;
auto const seq = *opt_seq;
{
std::lock_guard<std::mutex> lock (mutex_);
/*
"Preflight" the manifest -- before we spend time checking the
signature, make sure we trust the master key and the sequence
number is newer than any we have.
before we spend time checking the signature, make sure we trust the
master key and the sequence number is newer than any we have.
*/
auto const preflight = preflightManifest_locked(pk, seq, journal);
auto const chk = canApply(m.masterKey, m.sequence, journal);
if (preflight != ManifestDisposition::accepted)
if (chk != ManifestDisposition::accepted)
{
return preflight;
return chk;
}
}
if (! verify(st, HashPrefix::manifest, pk))
if (! m.verify())
{
/*
A manifest's signature is invalid.
This shouldn't happen normally.
A manifest's signature is invalid.
This shouldn't happen normally.
*/
if (journal.warning) journal.warning
<< "Failed to verify manifest #" << seq;
if (journal.warning)
logMftAct(journal.warning, "Invalid", m.masterKey, m.sequence);
return ManifestDisposition::invalid;
}
@@ -213,26 +258,20 @@ ManifestCache::applyManifest (std::string s, beast::Journal const& journal)
std::lock_guard<std::mutex> lock (mutex_);
/*
We released the lock above, so we have to preflight again, in case
We released the lock above, so we have to check again, in case
another thread accepted a newer manifest.
*/
auto const preflight = preflightManifest_locked(pk, seq, journal);
auto const chk = canApply(m.masterKey, m.sequence, journal);
if (preflight != ManifestDisposition::accepted)
if (chk != ManifestDisposition::accepted)
{
return preflight;
return chk;
}
auto const iter = map_.find(pk);
auto const iter = map_.find(m.masterKey);
auto& old = iter->second.m;
/*
The maximum possible sequence number means that the master key
has been revoked.
*/
auto const revoked = std::uint32_t (-1);
if (! old)
{
/*
@@ -241,20 +280,20 @@ ManifestCache::applyManifest (std::string s, beast::Journal const& journal)
run (and possibly not at all, if there's an obsolete entry in
[validator_keys] for a validator that no longer exists).
*/
if (journal.info) journal.info
<< "Adding new manifest #" << seq;
if (journal.info)
logMftAct(journal.info, "AcceptedNew", m.masterKey, m.sequence);
}
else
{
if (seq == revoked)
if (m.revoked ())
{
/*
The MASTER key for this validator was revoked. This is
expected, but should happen at most *very* rarely.
*/
if (journal.warning) journal.warning
<< "Dropping old manifest #" << old->seq
<< " because the master key was revoked";
if (journal.info)
logMftAct(journal.info, "Revoked",
m.masterKey, m.sequence, old->sequence);
}
else
{
@@ -262,19 +301,19 @@ ManifestCache::applyManifest (std::string s, beast::Journal const& journal)
An ephemeral key was revoked and superseded by a new key.
This is expected, but should happen infrequently.
*/
if (journal.warning) journal.warning
<< "Dropping old manifest #" << old->seq
<< " in favor of #" << seq;
if (journal.info)
logMftAct(journal.info, "AcceptedUpdate",
m.masterKey, m.sequence, old->sequence);
}
unl.deleteEphemeralKey (old->signingKey);
}
if (seq == revoked)
if (m.revoked ())
{
// The master key is revoked -- don't insert the signing key
if (auto const& j = journal.warning)
j << "Revoking master key: " << pk;
if (journal.warning)
logMftAct(journal.warning, "Revoked", m.masterKey, m.sequence);
/*
A validator master key has been compromised, so its manifests
@@ -286,12 +325,68 @@ ManifestCache::applyManifest (std::string s, beast::Journal const& journal)
}
else
{
unl.insertEphemeralKey (spk, iter->second.comment);
unl.insertEphemeralKey (m.signingKey, iter->second.comment);
}
old = Manifest(std::move (s), std::move (pk), std::move (spk), seq);
old = std::move(m);
return ManifestDisposition::accepted;
}
void ManifestCache::load (
DatabaseCon& dbCon, beast::Journal const& journal)
{
static const char* const sql =
"SELECT RawData FROM ValidatorManifests;";
auto db = dbCon.checkoutDb ();
soci::blob sociRawData (*db);
soci::statement st =
(db->prepare << sql,
soci::into (sociRawData));
st.execute ();
while (st.fetch ())
{
std::string serialized;
convert (sociRawData, serialized);
if (auto mo = make_Manifest (std::move (serialized)))
{
if (!mo->verify())
{
throw std::runtime_error("Unverifiable manifest in db");
}
// add trusted key
map_[mo->masterKey];
// OK if not accepted (may have been loaded from the config file)
applyManifest (std::move(*mo), journal);
}
else
{
if (journal.warning)
journal.warning << "Malformed manifest in database";
}
}
}
void ManifestCache::save (DatabaseCon& dbCon) const
{
auto db = dbCon.checkoutDb ();
soci::transaction tr(*db);
*db << "DELETE FROM ValidatorManifests";
static const char* const sql =
"INSERT INTO ValidatorManifests (RawData) VALUES (:rawData);";
// soci does not support bulk insertion of blob data
soci::blob rawData(*db);
for (auto const& v : map_)
{
if (!v.second.m)
continue;
convert (v.second.m->serialized, rawData);
*db << sql,
soci::use (rawData);
}
tr.commit ();
}
}

109
src/ripple/overlay/impl/Manifest.h
View File

@@ -34,17 +34,15 @@ namespace ripple {
Validator key manifests
-----------------------
First, a rationale: Suppose a system adminstrator leaves the company.
You err on the side of caution (if not paranoia) and assume that the
secret keys installed on Ripple validators are compromised. Not only
do you have to generate and install new key pairs on each validator,
EVERY rippled needs to have its config updated with the new public keys,
and is vulnerable to forged validation signatures until this is done.
The solution is a new layer of indirection: A master secret key under
Suppose the secret keys installed on a Ripple validator are compromised. Not
only do you have to generate and install new key pairs on each validator,
EVERY rippled needs to have its config updated with the new public keys, and
is vulnerable to forged validation signatures until this is done. The
solution is a new layer of indirection: A master secret key under
restrictive access control is used to sign a "manifest": essentially, a
certificate including the master public key, an ephemeral public key for
verifying validations (which will be signed by its secret counterpart),
a sequence number, and a digital signature.
verifying validations (which will be signed by its secret counterpart), a
sequence number, and a digital signature.
The manifest has two serialized forms: one which includes the digital
signature and one which doesn't. There is an obvious causal dependency
@@ -89,22 +87,16 @@ struct Manifest
std::string serialized;
AnyPublicKey masterKey;
AnyPublicKey signingKey;
std::uint32_t seq;
std::uint32_t sequence;
Manifest(std::string s, AnyPublicKey pk, AnyPublicKey spk, std::uint32_t seq)
: serialized(std::move(s))
, masterKey(std::move(pk))
, signingKey(std::move(spk))
, seq(seq)
{
}
Manifest(std::string s, AnyPublicKey pk, AnyPublicKey spk, std::uint32_t seq);
#ifdef _MSC_VER
Manifest(Manifest&& other)
: serialized(std::move(other.serialized))
, masterKey(std::move(other.masterKey))
, signingKey(std::move(other.signingKey))
, seq(other.seq)
, sequence(other.sequence)
{
}
@@ -113,26 +105,42 @@ struct Manifest
serialized = std::move(other.serialized);
masterKey = std::move(other.masterKey);
signingKey = std::move(other.signingKey);
seq = other.seq;
sequence = other.sequence;
return *this;
}
#else
Manifest(Manifest&& other) = default;
Manifest& operator=(Manifest&& other) = default;
#endif
bool verify () const;
uint256 hash () const;
bool revoked () const;
};
boost::optional<Manifest> make_Manifest(std::string s);
inline bool operator==(Manifest const& lhs, Manifest const& rhs)
{
return lhs.serialized == rhs.serialized && lhs.masterKey == rhs.masterKey &&
lhs.signingKey == rhs.signingKey && lhs.sequence == rhs.sequence;
}
inline bool operator!=(Manifest const& lhs, Manifest const& rhs)
{
return !(lhs == rhs);
}
enum class ManifestDisposition
{
accepted = 0, // everything checked out
accepted = 0, // everything checked out
malformed, // deserialization fails
incomplete, // fields are missing
untrusted, // manifest declares a master key we don't trust
stale, // trusted master key, but seq is too old
invalid, // trusted and timely, but invalid signature
untrusted, // manifest declares a master key we don't trust
stale, // trusted master key, but seq is too old
invalid, // trusted and timely, but invalid signature
};
class DatabaseCon;
/** Remembers manifests with the highest sequence number. */
class ManifestCache
{
@@ -140,6 +148,30 @@ private:
struct MappedType
{
MappedType() = default;
#ifdef _MSC_VER
MappedType(MappedType&& rhs)
:comment (std::move (rhs.comment))
, m (std::move (rhs.m))
{
}
MappedType& operator=(MappedType&& rhs)
{
comment = std::move (rhs.comment);
m = std::move (rhs.m);
return *this;
}
#else
MappedType(MappedType&&) = default;
MappedType& operator=(MappedType&&) = default;
#endif
MappedType(std::string comment,
std::string serialized,
AnyPublicKey pk, AnyPublicKey spk, std::uint32_t seq)
:comment (std::move(comment))
{
m.emplace (std::move(serialized), std::move(pk), std::move(spk),
seq);
}
std::string comment;
boost::optional<Manifest> m;
@@ -151,7 +183,7 @@ private:
MapType map_;
ManifestDisposition
preflightManifest_locked (AnyPublicKey const& pk, std::uint32_t seq,
canApply (AnyPublicKey const& pk, std::uint32_t seq,
beast::Journal const& journal) const;
public:
@@ -161,12 +193,15 @@ public:
~ManifestCache() = default;
void configValidatorKey(std::string const& line, beast::Journal const& journal);
void configManifest(std::string s, beast::Journal const& journal);
void configManifest(Manifest m, beast::Journal const& journal);
void addTrustedKey (AnyPublicKey const& pk, std::string const& comment);
void addTrustedKey (AnyPublicKey const& pk, std::string comment);
ManifestDisposition
applyManifest (std::string s, beast::Journal const& journal);
applyManifest (Manifest m, beast::Journal const& journal);
void load (DatabaseCon& dbCon, beast::Journal const& journal);
void save (DatabaseCon& dbCon) const;
// A "for_each" for populated manifests only
template <class Function>
@@ -180,6 +215,22 @@ public:
f(*m);
}
}
// A "for_each" for populated manifests only
// The PreFun is called with the maximum number of
// times EachFun will be called (useful for memory allocations)
template <class PreFun, class EachFun>
void
for_each_manifest(PreFun&& pf, EachFun&& f) const
{
std::lock_guard<std::mutex> lock (mutex_);
pf(map_.size ());
for (auto const& e : map_)
{
if (auto const& m = e.second.m)
f(*m);
}
}
};
} // ripple

116
src/ripple/overlay/impl/OverlayImpl.cpp
View File

@@ -19,6 +19,7 @@
#include <BeastConfig.h>
#include <ripple/app/misc/IHashRouter.h>
#include <ripple/core/DatabaseCon.h>
#include <ripple/basics/Log.h>
#include <ripple/basics/make_SSLContext.h>
#include <ripple/protocol/JsonFields.h>
@@ -426,24 +427,24 @@ OverlayImpl::checkStopped ()
stopped();
}
static
void
prepareValidatorKeyManifests (ManifestCache& mc, beast::Journal const& journal)
OverlayImpl::setupValidatorKeyManifests (BasicConfig const& config,
DatabaseCon& db)
{
auto const validator_keys = getConfig().section("validator_keys");
auto const validation_manifest = getConfig().section("validation_manifest");
auto const validator_keys = config.section ("validator_keys");
auto const validation_manifest = config.section ("validation_manifest");
if (! validator_keys.lines().empty())
{
for (auto const& line : validator_keys.lines())
{
mc.configValidatorKey (line, journal);
manifestCache_.configValidatorKey (line, journal_);
}
}
else
{
if (journal.warning)
journal.warning << "[validator_keys] is empty";
if (journal_.warning)
journal_.warning << "[validator_keys] is empty";
}
if (! validation_manifest.lines().empty())
@@ -452,21 +453,33 @@ prepareValidatorKeyManifests (ManifestCache& mc, beast::Journal const& journal)
for (auto const& line : validation_manifest.lines())
s += beast::rfc2616::trim(line);
s = beast::base64_decode(s);
mc.configManifest(std::move(s), journal);
if (auto mo = make_Manifest (std::move (s)))
{
manifestCache_.configManifest (std::move (*mo), journal_);
}
else
{
throw std::runtime_error("Malformed manifest in config");
}
}
else
{
if (journal.warning)
journal.warning << "No [validation_manifest] section in config";
if (journal_.warning)
journal_.warning << "No [validation_manifest] section in config";
}
manifestCache_.load (db, journal_);
}
void
OverlayImpl::saveValidatorKeyManifests (DatabaseCon& db) const
{
manifestCache_.save (db);
}
void
OverlayImpl::onPrepare()
{
prepareValidatorKeyManifests (manifestCache_, journal_);
PeerFinder::Config config;
if (getConfig ().PEERS_MAX != 0)
@@ -617,53 +630,74 @@ OverlayImpl::onPeerDeactivate (Peer::id_t id,
void
OverlayImpl::onManifests (Job&,
std::shared_ptr<protocol::TMManifests> const& inbox,
std::shared_ptr<protocol::TMManifests> const& m,
std::shared_ptr<PeerImp> const& from)
{
auto& hashRouter = getApp().getHashRouter();
auto const n = inbox->list_size();
auto const n = m->list_size();
auto const& journal = from->pjournal();
if (journal.debug) journal.debug
<< "TMManifest, " << n << (n == 1 ? " item" : " items");
protocol::TMManifests outbox;
bool const history = m->history ();
for (std::size_t i = 0; i < n; ++i)
{
auto& s = inbox->list().Get(i).stobject();
auto& s = m->list ().Get (i).stobject ();
uint256 const hash = getSHA512Half (s);
auto const result = manifestCache_.applyManifest (s, journal);
if (result == ManifestDisposition::accepted)
if (auto mo = make_Manifest (s))
{
protocol::TMManifests outbox;
uint256 const hash = mo->hash ();
if (!hashRouter.addSuppressionPeer (hash, from->id ()))
continue;
outbox.add_list()->set_stobject(s);
auto const result =
manifestCache_.applyManifest (std::move (*mo), journal);
auto const msg = std::make_shared<Message>(outbox, protocol::mtMANIFESTS);
if (result == ManifestDisposition::accepted)
{
auto db = getApp ().getWalletDB ().checkoutDb ();
for_each( [&](std::shared_ptr<PeerImp> const& peer)
{
if (hashRouter.addSuppressionPeer (hash, peer->id()) && peer != from)
{
if (auto& j = peer->pjournal().warning)
j << "Forwarding manifest with hash " << hash;
peer->send(msg);
}
else if (peer != from)
{
if (auto& j = peer->pjournal().warning)
j << "Suppressed manifest with hash " << hash;
}
});
soci::transaction tr(*db);
static const char* const sql =
"INSERT INTO ValidatorManifests (RawData) VALUES (:rawData);";
soci::blob rawData(*db);
convert (mo->serialized, rawData);
*db << sql, soci::use (rawData);
tr.commit ();
}
if (history)
{
// Historical manifests are sent on initial peer connections.
// They do not need to be forwarded to other peers.
std::set<Peer::id_t> peers;
hashRouter.swapSet (hash, peers, SF_RELAYED);
continue;
}
if (result == ManifestDisposition::accepted)
{
protocol::TMManifests o;
o.add_list ()->set_stobject (s);
std::set<Peer::id_t> peers;
hashRouter.swapSet (hash, peers, SF_RELAYED);
foreach (send_if_not (
std::make_shared<Message>(o, protocol::mtMANIFESTS),
peer_in_set (peers)));
}
else
{
if (journal.info)
journal.info << "Bad manifest #" << i + 1;
}
}
else
{
if (journal.info) journal.info
<< "Bad manifest #" << i + 1;
if (journal.warning)
journal.warning << "Malformed manifest #" << i + 1;
continue;
}
}
}

9
src/ripple/overlay/impl/OverlayImpl.h
View File

@@ -192,6 +192,15 @@ public:
relay (protocol::TMValidation& m,
uint256 const& uid) override;
virtual
void
setupValidatorKeyManifests (BasicConfig const& config,
DatabaseCon& db) override;
virtual
void
saveValidatorKeyManifests (DatabaseCon& db) const override;
//--------------------------------------------------------------------------
//
// OverlayImpl

6
src/ripple/overlay/impl/PeerImp.cpp
View File

@@ -655,9 +655,11 @@ PeerImp::doProtocolStart()
onReadMessage(error_code(), 0);
protocol::TMManifests tm;
tm.set_history (true);
overlay_.manifestCache().for_each_manifest(
[&](Manifest const& manifest)
overlay_.manifestCache ().for_each_manifest (
[&tm](size_t s){tm.mutable_list()->Reserve(s);},
[&tm](Manifest const& manifest)
{
auto const& s = manifest.serialized;
auto& tm_e = *tm.add_list();

205
src/ripple/overlay/tests/manifest_test.cpp
View File

@@ -20,21 +20,76 @@
#include <BeastConfig.h>
#include <ripple/basics/TestSuite.h>
#include <ripple/overlay/impl/Manifest.h>
#include <ripple/core/DatabaseCon.h>
#include <ripple/app/main/DBInit.h>
#include <ripple/protocol/Sign.h>
#include <ripple/protocol/STExchange.h>
#include <boost/filesystem.hpp>
#include <boost/algorithm/string.hpp>
namespace ripple {
namespace tests {
class manifest_test : public ripple::TestSuite
{
private:
static void cleanupDatabaseDir (boost::filesystem::path const& dbPath)
{
using namespace boost::filesystem;
if (!exists (dbPath) || !is_directory (dbPath) || !is_empty (dbPath))
return;
remove (dbPath);
}
static void setupDatabaseDir (boost::filesystem::path const& dbPath)
{
using namespace boost::filesystem;
if (!exists (dbPath))
{
create_directory (dbPath);
return;
}
if (!is_directory (dbPath))
{
// someone created a file where we want to put our directory
throw std::runtime_error ("Cannot create directory: " +
dbPath.string ());
}
}
static boost::filesystem::path getDatabasePath ()
{
return boost::filesystem::current_path () / "manifest_test_databases";
}
public:
// Return a manifest in both serialized and STObject form
std::string
make_manifest(AnySecretKey const& sk, AnyPublicKey const& spk, int seq, bool broken = false)
manifest_test ()
{
try
{
setupDatabaseDir (getDatabasePath ());
}
catch (...)
{
}
}
~manifest_test ()
{
try
{
cleanupDatabaseDir (getDatabasePath ());
}
catch (...)
{
}
}
Manifest
make_Manifest
(AnySecretKey const& sk, AnyPublicKey const& spk, int seq,
bool broken = false)
{
auto const pk = sk.publicKey();
STObject st(sfGeneric);
set(st, sfSequence, seq);
set(st, sfPublicKey, pk);
@@ -52,53 +107,125 @@ public:
st.add(s);
std::string const m (static_cast<char const*> (s.data()), s.size());
return m;
if (auto r = ripple::make_Manifest (std::move (m)))
{
return std::move (*r);
}
throw std::runtime_error("Could not create a manifest");
}
Manifest
clone (Manifest const& m)
{
return Manifest (m.serialized, m.masterKey, m.signingKey, m.sequence);
}
void testLoadStore (ManifestCache const& m)
{
testcase ("load/store");
std::string const dbName("ManifestCacheTestDB");
{
// create a database, save the manifest to the db and reload and
// check that the manifest caches are the same
DatabaseCon::Setup setup;
setup.dataDir = getDatabasePath ();
DatabaseCon dbCon(setup, dbName, WalletDBInit, WalletDBCount);
m.save (dbCon);
ManifestCache loaded;
beast::Journal journal;
loaded.load (dbCon, journal);
auto getPopulatedManifests =
[](ManifestCache const& cache) -> std::vector<Manifest const*>
{
std::vector<Manifest const*> result;
result.reserve (32);
cache.for_each_manifest (
[&result](Manifest const& m)
{result.push_back (&m);});
return result;
};
auto sort =
[](std::vector<Manifest const*> mv) -> std::vector<Manifest const*>
{
std::sort (mv.begin (),
mv.end (),
[](Manifest const* lhs, Manifest const* rhs)
{return lhs->serialized < rhs->serialized;});
return mv;
};
std::vector<Manifest const*> const inManifests (
sort (getPopulatedManifests (m)));
std::vector<Manifest const*> const loadedManifests (
sort (getPopulatedManifests (loaded)));
if (inManifests.size () == loadedManifests.size ())
{
expect (std::equal
(inManifests.begin (), inManifests.end (),
loadedManifests.begin (),
[](Manifest const* lhs, Manifest const* rhs)
{return *lhs == *rhs;}));
}
else
{
fail ();
}
}
boost::filesystem::remove (getDatabasePath () /
boost::filesystem::path (dbName));
}
void
run() override
{
auto const accepted = ManifestDisposition::accepted;
auto const malformed = ManifestDisposition::malformed;
auto const untrusted = ManifestDisposition::untrusted;
auto const stale = ManifestDisposition::stale;
auto const invalid = ManifestDisposition::invalid;
beast::Journal journal;
auto const sk_a = AnySecretKey::make_ed25519();
auto const sk_b = AnySecretKey::make_ed25519();
auto const pk_a = sk_a.publicKey();
auto const pk_b = sk_b.publicKey();
auto const kp_a = AnySecretKey::make_secp256k1_pair();
auto const kp_b = AnySecretKey::make_secp256k1_pair();
auto const s_a0 = make_manifest(sk_a, kp_a.second, 0);
auto const s_a1 = make_manifest(sk_a, kp_a.second, 1);
auto const s_b0 = make_manifest(sk_b, kp_b.second, 0);
auto const s_b1 = make_manifest(sk_b, kp_b.second, 1);
auto const s_b2 = make_manifest(sk_b, kp_b.second, 2, true); // broken
auto const fake = s_b1 + '\0';
ManifestCache cache;
{
testcase ("apply");
auto const accepted = ManifestDisposition::accepted;
auto const untrusted = ManifestDisposition::untrusted;
auto const stale = ManifestDisposition::stale;
auto const invalid = ManifestDisposition::invalid;
expect(cache.applyManifest(s_a0, journal) == untrusted, "have to install a trusted key first");
beast::Journal journal;
cache.addTrustedKey(pk_a, "a");
cache.addTrustedKey(pk_b, "b");
auto const sk_a = AnySecretKey::make_ed25519 ();
auto const sk_b = AnySecretKey::make_ed25519 ();
auto const pk_a = sk_a.publicKey ();
auto const pk_b = sk_b.publicKey ();
auto const kp_a = AnySecretKey::make_secp256k1_pair ();
auto const kp_b = AnySecretKey::make_secp256k1_pair ();
expect(cache.applyManifest(s_a0, journal) == accepted);
expect(cache.applyManifest(s_a0, journal) == stale);
auto const s_a0 = make_Manifest (sk_a, kp_a.second, 0);
auto const s_a1 = make_Manifest (sk_a, kp_a.second, 1);
auto const s_b0 = make_Manifest (sk_b, kp_b.second, 0);
auto const s_b1 = make_Manifest (sk_b, kp_b.second, 1);
auto const s_b2 =
make_Manifest (sk_b, kp_b.second, 2, true); // broken
auto const fake = s_b1.serialized + '\0';
expect(cache.applyManifest(s_a1, journal) == accepted);
expect(cache.applyManifest(s_a1, journal) == stale);
expect(cache.applyManifest(s_a0, journal) == stale);
expect (cache.applyManifest (clone (s_a0), journal) == untrusted,
"have to install a trusted key first");
expect(cache.applyManifest(s_b0, journal) == accepted);
expect(cache.applyManifest(s_b0, journal) == stale);
cache.addTrustedKey (pk_a, "a");
cache.addTrustedKey (pk_b, "b");
expect(cache.applyManifest(fake, journal) == malformed);
expect(cache.applyManifest(s_b2, journal) == invalid);
expect (cache.applyManifest (clone (s_a0), journal) == accepted);
expect (cache.applyManifest (clone (s_a0), journal) == stale);
expect (cache.applyManifest (clone (s_a1), journal) == accepted);
expect (cache.applyManifest (clone (s_a1), journal) == stale);
expect (cache.applyManifest (clone (s_a0), journal) == stale);
expect (cache.applyManifest (clone (s_b0), journal) == accepted);
expect (cache.applyManifest (clone (s_b0), journal) == stale);
expect (!ripple::make_Manifest(fake));
expect (cache.applyManifest (clone (s_b2), journal) == invalid);
}
testLoadStore (cache);
}
};

3
src/ripple/proto/ripple.proto
View File

@@ -44,6 +44,9 @@ message TMManifest
message TMManifests
{
repeated TMManifest list = 1;
// The manifests sent when a peer first connects to another peer are `history`.
// The receiving peer does not forward them.
optional bool history = 2 [default = false];
}
//------------------------------------------------------------------------------

6
src/ripple/protocol/AnyPublicKey.h
View File

@@ -123,7 +123,8 @@ public:
AnyPublicKey& operator= (AnyPublicKey&& other)
{
buffer_type::member =
std::move(other.buffer_type::member);
std::move (other.buffer_type::member);
AnyPublicKeySlice::operator= (other);
return *this;
}
#else
@@ -179,6 +180,9 @@ struct STExchange<STBlob, AnyPublicKey>
}
};
std::string
toString (AnyPublicKey const& pk);
} // ripple
#endif

11
src/ripple/protocol/impl/AnyPublicKey.cpp
View File

@@ -90,4 +90,15 @@ AnyPublicKeySlice::verify (
return false;
}
std::string
toString (AnyPublicKey const& pk)
{
Blob buffer;
buffer.reserve (1 + pk.size ());
buffer.push_back (VER_NODE_PUBLIC);
auto const data = pk.data ();
buffer.insert (buffer.end (), data, data + pk.size ());
return Base58::encodeWithCheck (buffer);
}
} // ripple

2
src/ripple/protocol/impl/HashPrefix.cpp
View File

@@ -34,6 +34,6 @@ HashPrefix const HashPrefix::txSign ('S', 'T', 'X');
HashPrefix const HashPrefix::txMultiSign ('S', 'M', 'T');
HashPrefix const HashPrefix::validation ('V', 'A', 'L');
HashPrefix const HashPrefix::proposal ('P', 'R', 'P');
HashPrefix const HashPrefix::manifest ('M', 'A', 'N');
HashPrefix const HashPrefix::manifest ('M', 'A', 'N');
} // ripple