rippled
SHAMapDelta.cpp
1 //------------------------------------------------------------------------------
2 /*
3  This file is part of rippled: https://github.com/ripple/rippled
4  Copyright (c) 2012, 2013 Ripple Labs Inc.
5 
6  Permission to use, copy, modify, and/or distribute this software for any
7  purpose with or without fee is hereby granted, provided that the above
8  copyright notice and this permission notice appear in all copies.
9 
10  THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11  WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12  MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13  ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14  WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15  ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16  OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18 //==============================================================================
19 
20 #include <ripple/basics/contract.h>
21 #include <ripple/shamap/SHAMap.h>
22 
23 #include <array>
24 #include <stack>
25 #include <vector>
26 
27 namespace ripple {
28 
29 // This code is used to compare another node's transaction tree
30 // to our own. It returns a map containing all items that are different
31 // between two SHA maps. It is optimized not to descend down tree
32 // branches with the same branch hash. A limit can be passed so
33 // that we will abort early if a node sends a map to us that
34 // makes no sense at all. (And our sync algorithm will avoid
35 // synchronizing matching branches too.)
36 
37 bool
38 SHAMap::walkBranch(
39  SHAMapTreeNode* node,
40  std::shared_ptr<SHAMapItem const> const& otherMapItem,
41  bool isFirstMap,
42  Delta& differences,
43  int& maxCount) const
44 {
45  // Walk a branch of a SHAMap that's matched by an empty branch or single
46  // item in the other map
47  std::stack<SHAMapTreeNode*, std::vector<SHAMapTreeNode*>> nodeStack;
48  nodeStack.push(node);
49 
50  bool emptyBranch = !otherMapItem;
51 
52  while (!nodeStack.empty())
53  {
54  node = nodeStack.top();
55  nodeStack.pop();
56 
57  if (node->isInner())
58  {
59  // This is an inner node, add all non-empty branches
60  auto inner = static_cast<SHAMapInnerNode*>(node);
61  for (int i = 0; i < 16; ++i)
62  if (!inner->isEmptyBranch(i))
63  nodeStack.push({descendThrow(inner, i)});
64  }
65  else
66  {
67  // This is a leaf node, process its item
68  auto item = static_cast<SHAMapLeafNode*>(node)->peekItem();
69 
70  if (emptyBranch || (item->key() != otherMapItem->key()))
71  {
72  // unmatched
73  if (isFirstMap)
74  differences.insert(std::make_pair(
75  item->key(),
76  DeltaRef(item, std::shared_ptr<SHAMapItem const>())));
77  else
78  differences.insert(std::make_pair(
79  item->key(),
80  DeltaRef(std::shared_ptr<SHAMapItem const>(), item)));
81 
82  if (--maxCount <= 0)
83  return false;
84  }
85  else if (item->slice() != otherMapItem->slice())
86  {
87  // non-matching items with same tag
88  if (isFirstMap)
89  differences.insert(std::make_pair(
90  item->key(), DeltaRef(item, otherMapItem)));
91  else
92  differences.insert(std::make_pair(
93  item->key(), DeltaRef(otherMapItem, item)));
94 
95  if (--maxCount <= 0)
96  return false;
97 
98  emptyBranch = true;
99  }
100  else
101  {
102  // exact match
103  emptyBranch = true;
104  }
105  }
106  }
107 
108  if (!emptyBranch)
109  {
110  // otherMapItem was unmatched, must add
111  if (isFirstMap) // this is first map, so other item is from second
112  differences.insert(std::make_pair(
113  otherMapItem->key(),
114  DeltaRef(std::shared_ptr<SHAMapItem const>(), otherMapItem)));
115  else
116  differences.insert(std::make_pair(
117  otherMapItem->key(),
118  DeltaRef(otherMapItem, std::shared_ptr<SHAMapItem const>())));
119 
120  if (--maxCount <= 0)
121  return false;
122  }
123 
124  return true;
125 }
126 
127 bool
128 SHAMap::compare(SHAMap const& otherMap, Delta& differences, int maxCount) const
129 {
130  // compare two hash trees, add up to maxCount differences to the difference
131  // table return value: true=complete table of differences given, false=too
132  // many differences throws on corrupt tables or missing nodes CAUTION:
133  // otherMap is not locked and must be immutable
134 
135  assert(isValid() && otherMap.isValid());
136 
137  if (getHash() == otherMap.getHash())
138  return true;
139 
140  using StackEntry = std::pair<SHAMapTreeNode*, SHAMapTreeNode*>;
141  std::stack<StackEntry, std::vector<StackEntry>>
142  nodeStack; // track nodes we've pushed
143 
144  nodeStack.push({root_.get(), otherMap.root_.get()});
145  while (!nodeStack.empty())
146  {
147  auto [ourNode, otherNode] = nodeStack.top();
148  nodeStack.pop();
149 
150  if (!ourNode || !otherNode)
151  {
152  assert(false);
153  Throw<SHAMapMissingNode>(type_, uint256());
154  }
155 
156  if (ourNode->isLeaf() && otherNode->isLeaf())
157  {
158  // two leaves
159  auto ours = static_cast<SHAMapLeafNode*>(ourNode);
160  auto other = static_cast<SHAMapLeafNode*>(otherNode);
161  if (ours->peekItem()->key() == other->peekItem()->key())
162  {
163  if (ours->peekItem()->slice() != other->peekItem()->slice())
164  {
165  differences.insert(std::make_pair(
166  ours->peekItem()->key(),
167  DeltaRef(ours->peekItem(), other->peekItem())));
168  if (--maxCount <= 0)
169  return false;
170  }
171  }
172  else
173  {
174  differences.insert(std::make_pair(
175  ours->peekItem()->key(),
176  DeltaRef(
177  ours->peekItem(),
178  std::shared_ptr<SHAMapItem const>())));
179  if (--maxCount <= 0)
180  return false;
181 
182  differences.insert(std::make_pair(
183  other->peekItem()->key(),
184  DeltaRef(
185  std::shared_ptr<SHAMapItem const>(),
186  other->peekItem())));
187  if (--maxCount <= 0)
188  return false;
189  }
190  }
191  else if (ourNode->isInner() && otherNode->isLeaf())
192  {
193  auto ours = static_cast<SHAMapInnerNode*>(ourNode);
194  auto other = static_cast<SHAMapLeafNode*>(otherNode);
195  if (!walkBranch(
196  ours, other->peekItem(), true, differences, maxCount))
197  return false;
198  }
199  else if (ourNode->isLeaf() && otherNode->isInner())
200  {
201  auto ours = static_cast<SHAMapLeafNode*>(ourNode);
202  auto other = static_cast<SHAMapInnerNode*>(otherNode);
203  if (!otherMap.walkBranch(
204  other, ours->peekItem(), false, differences, maxCount))
205  return false;
206  }
207  else if (ourNode->isInner() && otherNode->isInner())
208  {
209  auto ours = static_cast<SHAMapInnerNode*>(ourNode);
210  auto other = static_cast<SHAMapInnerNode*>(otherNode);
211  for (int i = 0; i < 16; ++i)
212  if (ours->getChildHash(i) != other->getChildHash(i))
213  {
214  if (other->isEmptyBranch(i))
215  {
216  // We have a branch, the other tree does not
217  SHAMapTreeNode* iNode = descendThrow(ours, i);
218  if (!walkBranch(
219  iNode,
220  std::shared_ptr<SHAMapItem const>(),
221  true,
222  differences,
223  maxCount))
224  return false;
225  }
226  else if (ours->isEmptyBranch(i))
227  {
228  // The other tree has a branch, we do not
229  SHAMapTreeNode* iNode = otherMap.descendThrow(other, i);
230  if (!otherMap.walkBranch(
231  iNode,
232  std::shared_ptr<SHAMapItem const>(),
233  false,
234  differences,
235  maxCount))
236  return false;
237  }
238  else // The two trees have different non-empty branches
239  nodeStack.push(
240  {descendThrow(ours, i),
241  otherMap.descendThrow(other, i)});
242  }
243  }
244  else
245  assert(false);
246  }
247 
248  return true;
249 }
250 
251 void
252 SHAMap::walkMap(std::vector<SHAMapMissingNode>& missingNodes, int maxMissing)
253  const
254 {
255  if (!root_->isInner()) // root_ is only node, and we have it
256  return;
257 
258  using StackEntry = std::shared_ptr<SHAMapInnerNode>;
259  std::stack<StackEntry, std::vector<StackEntry>> nodeStack;
260 
261  nodeStack.push(std::static_pointer_cast<SHAMapInnerNode>(root_));
262 
263  while (!nodeStack.empty())
264  {
265  std::shared_ptr<SHAMapInnerNode> node = std::move(nodeStack.top());
266  nodeStack.pop();
267 
268  for (int i = 0; i < 16; ++i)
269  {
270  if (!node->isEmptyBranch(i))
271  {
272  std::shared_ptr<SHAMapTreeNode> nextNode =
273  descendNoStore(node, i);
274 
275  if (nextNode)
276  {
277  if (nextNode->isInner())
278  nodeStack.push(
279  std::static_pointer_cast<SHAMapInnerNode>(
280  nextNode));
281  }
282  else
283  {
284  missingNodes.emplace_back(type_, node->getChildHash(i));
285  if (--maxMissing <= 0)
286  return;
287  }
288  }
289  }
290  }
291 }
292 
293 bool
294 SHAMap::walkMapParallel(
295  std::vector<SHAMapMissingNode>& missingNodes,
296  int maxMissing) const
297 {
298  if (!root_->isInner()) // root_ is only node, and we have it
299  return false;
300 
301  using StackEntry = std::shared_ptr<SHAMapInnerNode>;
302  std::array<std::shared_ptr<SHAMapTreeNode>, 16> topChildren;
303  {
304  auto const& innerRoot =
305  std::static_pointer_cast<SHAMapInnerNode>(root_);
306  for (int i = 0; i < 16; ++i)
307  {
308  if (!innerRoot->isEmptyBranch(i))
309  topChildren[i] = descendNoStore(innerRoot, i);
310  }
311  }
312  std::vector<std::thread> workers;
313  workers.reserve(16);
314  std::vector<SHAMapMissingNode> exceptions;
315  exceptions.reserve(16);
316 
317  std::array<std::stack<StackEntry, std::vector<StackEntry>>, 16> nodeStacks;
318 
319  // This mutex is used inside the worker threads to protect `missingNodes`
320  // and `maxMissing` from race conditions
321  std::mutex m;
322 
323  for (int rootChildIndex = 0; rootChildIndex < 16; ++rootChildIndex)
324  {
325  auto const& child = topChildren[rootChildIndex];
326  if (!child || !child->isInner())
327  continue;
328 
329  nodeStacks[rootChildIndex].push(
330  std::static_pointer_cast<SHAMapInnerNode>(child));
331 
332  JLOG(journal_.debug()) << "starting worker " << rootChildIndex;
333  workers.push_back(std::thread(
334  [&m, &missingNodes, &maxMissing, &exceptions, this](
335  std::stack<StackEntry, std::vector<StackEntry>> nodeStack) {
336  try
337  {
338  while (!nodeStack.empty())
339  {
340  std::shared_ptr<SHAMapInnerNode> node =
341  std::move(nodeStack.top());
342  assert(node);
343  nodeStack.pop();
344 
345  for (int i = 0; i < 16; ++i)
346  {
347  if (node->isEmptyBranch(i))
348  continue;
349  std::shared_ptr<SHAMapTreeNode> nextNode =
350  descendNoStore(node, i);
351 
352  if (nextNode)
353  {
354  if (nextNode->isInner())
355  nodeStack.push(std::static_pointer_cast<
356  SHAMapInnerNode>(nextNode));
357  }
358  else
359  {
360  std::lock_guard l{m};
361  missingNodes.emplace_back(
362  type_, node->getChildHash(i));
363  if (--maxMissing <= 0)
364  return;
365  }
366  }
367  }
368  }
369  catch (SHAMapMissingNode const& e)
370  {
371  std::lock_guard l(m);
372  exceptions.push_back(e);
373  }
374  },
375  std::move(nodeStacks[rootChildIndex])));
376  }
377 
378  for (std::thread& worker : workers)
379  worker.join();
380 
381  std::lock_guard l(m);
382  if (exceptions.empty())
383  return true;
384  std::stringstream ss;
385  ss << "Exception(s) in ledger load: ";
386  for (auto const& e : exceptions)
387  ss << e.what() << ", ";
388  JLOG(journal_.error()) << ss.str();
389  return false;
390 }
391 
392 } // namespace ripple
ripple::SHAMap::isValid
bool isValid() const
Definition: SHAMap.h:617
std::shared_ptr
STL class.
ripple::SHAMap::getHash
SHAMapHash getHash() const
Definition: SHAMap.cpp:842
std::pair
std::vector::reserve
T reserve(T... args)
vector
stack
ripple::SHAMap::peekItem
std::shared_ptr< SHAMapItem const > const & peekItem(uint256 const &id) const
Definition: SHAMap.cpp:586
std::stringstream
STL class.
std::lock_guard
STL class.
ripple::SHAMapTreeNode::isInner
virtual bool isInner() const =0
Determines if this is an inner node.
ripple::SHAMapLeafNode
Definition: SHAMapLeafNode.h:32
ripple::SHAMapMissingNode
Definition: SHAMapMissingNode.h:55
std::vector::push_back
T push_back(T... args)
ripple::base_uint< 256 >
ripple::SHAMap::DeltaRef
std::pair< std::shared_ptr< SHAMapItem const > const &, std::shared_ptr< SHAMapItem const > const & > DeltaRef
Definition: SHAMap.h:365
std::thread
STL class.
ripple::SHAMapInnerNode
Definition: SHAMapInnerNode.h:41
ripple::SHAMap
A SHAMap is both a radix tree with a fan-out of 16 and a Merkle tree.
Definition: SHAMap.h:95
std::stack::pop
T pop(T... args)
std::stack::top
T top(T... args)
ripple::SHAMapTreeNode
Definition: SHAMapTreeNode.h:53
array
ripple::SHAMap::descendThrow
SHAMapTreeNode * descendThrow(SHAMapInnerNode *, int branch) const
Definition: SHAMap.cpp:283
std::map
STL class.
std::vector::emplace_back
T emplace_back(T... args)
ripple
Use hash_* containers for keys that do not need a cryptographically secure hashing algorithm.
Definition: RCLCensorshipDetector.h:29
std::map::insert
T insert(T... args)
std::stack::empty
T empty(T... args)
std::stack::push
T push(T... args)
std::mutex
STL class.
std::stringstream::str
T str(T... args)
std::make_pair
T make_pair(T... args)
ripple::SHAMap::walkBranch
bool walkBranch(SHAMapTreeNode *node, std::shared_ptr< SHAMapItem const > const &otherMapItem, bool isFirstMap, Delta &differences, int &maxCount) const
Definition: SHAMapDelta.cpp:38
std::thread::join
T join(T... args)
ripple::SHAMap::root_
std::shared_ptr< SHAMapTreeNode > root_
Definition: SHAMap.h:107
Generated by   doxygen 1.8.17