rippled
SHAMapDelta.cpp
1 //------------------------------------------------------------------------------
2 /*
3  This file is part of rippled: https://github.com/ripple/rippled
4  Copyright (c) 2012, 2013 Ripple Labs Inc.
5 
6  Permission to use, copy, modify, and/or distribute this software for any
7  purpose with or without fee is hereby granted, provided that the above
8  copyright notice and this permission notice appear in all copies.
9 
10  THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11  WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12  MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13  ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14  WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15  ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16  OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18 //==============================================================================
19 
20 #include <ripple/basics/contract.h>
21 #include <ripple/shamap/SHAMap.h>
22 
23 #include <array>
24 #include <stack>
25 #include <vector>
26 
27 namespace ripple {
28 
29 // This code is used to compare another node's transaction tree
30 // to our own. It returns a map containing all items that are different
31 // between two SHA maps. It is optimized not to descend down tree
32 // branches with the same branch hash. A limit can be passed so
33 // that we will abort early if a node sends a map to us that
34 // makes no sense at all. (And our sync algorithm will avoid
35 // synchronizing matching branches too.)
36 
37 bool
38 SHAMap::walkBranch(
39  SHAMapTreeNode* node,
40  std::shared_ptr<SHAMapItem const> const& otherMapItem,
41  bool isFirstMap,
42  Delta& differences,
43  int& maxCount) const
44 {
45  // Walk a branch of a SHAMap that's matched by an empty branch or single
46  // item in the other map
47  std::stack<SHAMapTreeNode*, std::vector<SHAMapTreeNode*>> nodeStack;
48  nodeStack.push(node);
49 
50  bool emptyBranch = !otherMapItem;
51 
52  while (!nodeStack.empty())
53  {
54  node = nodeStack.top();
55  nodeStack.pop();
56 
57  if (node->isInner())
58  {
59  // This is an inner node, add all non-empty branches
60  auto inner = static_cast<SHAMapInnerNode*>(node);
61  for (int i = 0; i < 16; ++i)
62  if (!inner->isEmptyBranch(i))
63  nodeStack.push({descendThrow(inner, i)});
64  }
65  else
66  {
67  // This is a leaf node, process its item
68  auto item = static_cast<SHAMapLeafNode*>(node)->peekItem();
69 
70  if (emptyBranch || (item->key() != otherMapItem->key()))
71  {
72  // unmatched
73  if (isFirstMap)
74  differences.insert(std::make_pair(
75  item->key(),
76  DeltaRef(item, std::shared_ptr<SHAMapItem const>())));
77  else
78  differences.insert(std::make_pair(
79  item->key(),
80  DeltaRef(std::shared_ptr<SHAMapItem const>(), item)));
81 
82  if (--maxCount <= 0)
83  return false;
84  }
85  else if (item->slice() != otherMapItem->slice())
86  {
87  // non-matching items with same tag
88  if (isFirstMap)
89  differences.insert(std::make_pair(
90  item->key(), DeltaRef(item, otherMapItem)));
91  else
92  differences.insert(std::make_pair(
93  item->key(), DeltaRef(otherMapItem, item)));
94 
95  if (--maxCount <= 0)
96  return false;
97 
98  emptyBranch = true;
99  }
100  else
101  {
102  // exact match
103  emptyBranch = true;
104  }
105  }
106  }
107 
108  if (!emptyBranch)
109  {
110  // otherMapItem was unmatched, must add
111  if (isFirstMap) // this is first map, so other item is from second
112  differences.insert(std::make_pair(
113  otherMapItem->key(),
114  DeltaRef(std::shared_ptr<SHAMapItem const>(), otherMapItem)));
115  else
116  differences.insert(std::make_pair(
117  otherMapItem->key(),
118  DeltaRef(otherMapItem, std::shared_ptr<SHAMapItem const>())));
119 
120  if (--maxCount <= 0)
121  return false;
122  }
123 
124  return true;
125 }
126 
127 bool
128 SHAMap::compare(SHAMap const& otherMap, Delta& differences, int maxCount) const
129 {
130  // compare two hash trees, add up to maxCount differences to the difference
131  // table return value: true=complete table of differences given, false=too
132  // many differences throws on corrupt tables or missing nodes CAUTION:
133  // otherMap is not locked and must be immutable
134 
135  assert(isValid() && otherMap.isValid());
136 
137  if (getHash() == otherMap.getHash())
138  return true;
139 
140  using StackEntry = std::pair<SHAMapTreeNode*, SHAMapTreeNode*>;
141  std::stack<StackEntry, std::vector<StackEntry>>
142  nodeStack; // track nodes we've pushed
143 
144  nodeStack.push({root_.get(), otherMap.root_.get()});
145  while (!nodeStack.empty())
146  {
147  auto [ourNode, otherNode] = nodeStack.top();
148  nodeStack.pop();
149 
150  if (!ourNode || !otherNode)
151  {
152  assert(false);
153  Throw<SHAMapMissingNode>(type_, uint256());
154  }
155 
156  if (ourNode->isLeaf() && otherNode->isLeaf())
157  {
158  // two leaves
159  auto ours = static_cast<SHAMapLeafNode*>(ourNode);
160  auto other = static_cast<SHAMapLeafNode*>(otherNode);
161  if (ours->peekItem()->key() == other->peekItem()->key())
162  {
163  if (ours->peekItem()->slice() != other->peekItem()->slice())
164  {
165  differences.insert(std::make_pair(
166  ours->peekItem()->key(),
167  DeltaRef(ours->peekItem(), other->peekItem())));
168  if (--maxCount <= 0)
169  return false;
170  }
171  }
172  else
173  {
174  differences.insert(std::make_pair(
175  ours->peekItem()->key(),
176  DeltaRef(
177  ours->peekItem(),
178  std::shared_ptr<SHAMapItem const>())));
179  if (--maxCount <= 0)
180  return false;
181 
182  differences.insert(std::make_pair(
183  other->peekItem()->key(),
184  DeltaRef(
185  std::shared_ptr<SHAMapItem const>(),
186  other->peekItem())));
187  if (--maxCount <= 0)
188  return false;
189  }
190  }
191  else if (ourNode->isInner() && otherNode->isLeaf())
192  {
193  auto ours = static_cast<SHAMapInnerNode*>(ourNode);
194  auto other = static_cast<SHAMapLeafNode*>(otherNode);
195  if (!walkBranch(
196  ours, other->peekItem(), true, differences, maxCount))
197  return false;
198  }
199  else if (ourNode->isLeaf() && otherNode->isInner())
200  {
201  auto ours = static_cast<SHAMapLeafNode*>(ourNode);
202  auto other = static_cast<SHAMapInnerNode*>(otherNode);
203  if (!otherMap.walkBranch(
204  other, ours->peekItem(), false, differences, maxCount))
205  return false;
206  }
207  else if (ourNode->isInner() && otherNode->isInner())
208  {
209  auto ours = static_cast<SHAMapInnerNode*>(ourNode);
210  auto other = static_cast<SHAMapInnerNode*>(otherNode);
211  for (int i = 0; i < 16; ++i)
212  if (ours->getChildHash(i) != other->getChildHash(i))
213  {
214  if (other->isEmptyBranch(i))
215  {
216  // We have a branch, the other tree does not
217  SHAMapTreeNode* iNode = descendThrow(ours, i);
218  if (!walkBranch(
219  iNode,
220  std::shared_ptr<SHAMapItem const>(),
221  true,
222  differences,
223  maxCount))
224  return false;
225  }
226  else if (ours->isEmptyBranch(i))
227  {
228  // The other tree has a branch, we do not
229  SHAMapTreeNode* iNode = otherMap.descendThrow(other, i);
230  if (!otherMap.walkBranch(
231  iNode,
232  std::shared_ptr<SHAMapItem const>(),
233  false,
234  differences,
235  maxCount))
236  return false;
237  }
238  else // The two trees have different non-empty branches
239  nodeStack.push(
240  {descendThrow(ours, i),
241  otherMap.descendThrow(other, i)});
242  }
243  }
244  else
245  assert(false);
246  }
247 
248  return true;
249 }
250 
251 void
252 SHAMap::walkMap(std::vector<SHAMapMissingNode>& missingNodes, int maxMissing)
253  const
254 {
255  if (!root_->isInner()) // root_ is only node, and we have it
256  return;
257 
258  using StackEntry = std::shared_ptr<SHAMapInnerNode>;
259  std::stack<StackEntry, std::vector<StackEntry>> nodeStack;
260 
261  nodeStack.push(std::static_pointer_cast<SHAMapInnerNode>(root_));
262 
263  while (!nodeStack.empty())
264  {
265  std::shared_ptr<SHAMapInnerNode> node = std::move(nodeStack.top());
266  nodeStack.pop();
267 
268  for (int i = 0; i < 16; ++i)
269  {
270  if (!node->isEmptyBranch(i))
271  {
272  std::shared_ptr<SHAMapTreeNode> nextNode =
273  descendNoStore(node, i);
274 
275  if (nextNode)
276  {
277  if (nextNode->isInner())
278  nodeStack.push(
279  std::static_pointer_cast<SHAMapInnerNode>(
280  nextNode));
281  }
282  else
283  {
284  missingNodes.emplace_back(type_, node->getChildHash(i));
285  if (--maxMissing <= 0)
286  return;
287  }
288  }
289  }
290  }
291 }
292 
293 void
294 SHAMap::walkMapParallel(
295  std::vector<SHAMapMissingNode>& missingNodes,
296  int maxMissing) const
297 {
298  if (!root_->isInner()) // root_ is only node, and we have it
299  return;
300 
301  using StackEntry = std::shared_ptr<SHAMapInnerNode>;
302  std::array<std::shared_ptr<SHAMapTreeNode>, 16> topChildren;
303  {
304  auto const& innerRoot =
305  std::static_pointer_cast<SHAMapInnerNode>(root_);
306  for (int i = 0; i < 16; ++i)
307  {
308  if (!innerRoot->isEmptyBranch(i))
309  topChildren[i] = descendNoStore(innerRoot, i);
310  }
311  }
312  std::vector<std::thread> workers;
313  workers.reserve(16);
314 
315  std::array<std::stack<StackEntry, std::vector<StackEntry>>, 16> nodeStacks;
316 
317  // This mutex is used inside the worker threads to protect `missingNodes`
318  // and `maxMissing` from race conditions
319  std::mutex m;
320 
321  for (int rootChildIndex = 0; rootChildIndex < 16; ++rootChildIndex)
322  {
323  auto const& child = topChildren[rootChildIndex];
324  if (!child || !child->isInner())
325  continue;
326 
327  nodeStacks[rootChildIndex].push(
328  std::static_pointer_cast<SHAMapInnerNode>(child));
329 
330  JLOG(journal_.debug()) << "starting worker " << rootChildIndex;
331  workers.push_back(std::thread(
332  [&m, &missingNodes, &maxMissing, this](
333  std::stack<StackEntry, std::vector<StackEntry>> nodeStack) {
334  while (!nodeStack.empty())
335  {
336  std::shared_ptr<SHAMapInnerNode> node =
337  std::move(nodeStack.top());
338  assert(node);
339  nodeStack.pop();
340 
341  for (int i = 0; i < 16; ++i)
342  {
343  if (node->isEmptyBranch(i))
344  continue;
345  std::shared_ptr<SHAMapTreeNode> nextNode =
346  descendNoStore(node, i);
347 
348  if (nextNode)
349  {
350  if (nextNode->isInner())
351  nodeStack.push(
352  std::static_pointer_cast<SHAMapInnerNode>(
353  nextNode));
354  }
355  else
356  {
357  std::lock_guard l{m};
358  missingNodes.emplace_back(
359  type_, node->getChildHash(i));
360  if (--maxMissing <= 0)
361  return;
362  }
363  }
364  }
365  },
366  std::move(nodeStacks[rootChildIndex])));
367  }
368 
369  for (std::thread& worker : workers)
370  worker.join();
371 }
372 
373 } // namespace ripple
ripple::SHAMap::isValid
bool isValid() const
Definition: SHAMap.h:584
std::shared_ptr
STL class.
ripple::SHAMap::getHash
SHAMapHash getHash() const
Definition: SHAMap.cpp:783
std::pair
std::vector::reserve
T reserve(T... args)
vector
stack
ripple::SHAMap::peekItem
std::shared_ptr< SHAMapItem const > const & peekItem(uint256 const &id) const
Definition: SHAMap.cpp:561
ripple::SHAMapTreeNode::isInner
virtual bool isInner() const =0
Determines if this is an inner node.
ripple::SHAMapLeafNode
Definition: SHAMapLeafNode.h:32
std::vector::push_back
T push_back(T... args)
ripple::base_uint< 256 >
ripple::SHAMap::DeltaRef
std::pair< std::shared_ptr< SHAMapItem const > const &, std::shared_ptr< SHAMapItem const > const & > DeltaRef
Definition: SHAMap.h:351
std::thread
STL class.
ripple::SHAMapInnerNode
Definition: SHAMapInnerNode.h:39
ripple::SHAMap
A SHAMap is both a radix tree with a fan-out of 16 and a Merkle tree.
Definition: SHAMap.h:95
std::stack::pop
T pop(T... args)
std::stack::top
T top(T... args)
ripple::SHAMapTreeNode
Definition: SHAMapTreeNode.h:53
array
ripple::SHAMap::descendThrow
SHAMapTreeNode * descendThrow(SHAMapInnerNode *, int branch) const
Definition: SHAMap.cpp:283
std::map
STL class.
std::vector::emplace_back
T emplace_back(T... args)
ripple
Use hash_* containers for keys that do not need a cryptographically secure hashing algorithm.
Definition: RCLCensorshipDetector.h:29
std::map::insert
T insert(T... args)
std::stack::empty
T empty(T... args)
std::stack::push
T push(T... args)
std::mutex
STL class.
std::make_pair
T make_pair(T... args)
ripple::SHAMap::walkBranch
bool walkBranch(SHAMapTreeNode *node, std::shared_ptr< SHAMapItem const > const &otherMapItem, bool isFirstMap, Delta &differences, int &maxCount) const
Definition: SHAMapDelta.cpp:38
std::thread::join
T join(T... args)
ripple::SHAMap::root_
std::shared_ptr< SHAMapTreeNode > root_
Definition: SHAMap.h:107
Generated by   doxygen 1.8.17