rippled
DatabaseShard_test.cpp
1 //------------------------------------------------------------------------------
2 /*
3  This file is part of rippled: https://github.com/ripple/rippled
4  Copyright (c) 2020 Ripple Labs Inc.
5 
6  Permission to use, copy, modify, and/or distribute this software for any
7  purpose with or without fee is hereby granted, provided that the above
8  copyright notice and this permission notice appear in all copies.
9 
10  THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11  WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12  MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13  ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14  WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15  ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16  OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18 //==============================================================================
19 
20 #include <ripple/app/ledger/LedgerMaster.h>
21 #include <ripple/app/ledger/LedgerToJson.h>
22 #include <ripple/beast/utility/temp_dir.h>
23 #include <ripple/core/ConfigSections.h>
24 #include <ripple/nodestore/DatabaseShard.h>
25 #include <ripple/nodestore/DummyScheduler.h>
26 #include <ripple/nodestore/impl/DecodedBlob.h>
27 #include <ripple/nodestore/impl/Shard.h>
28 #include <chrono>
29 #include <numeric>
30 #include <test/jtx.h>
31 #include <test/nodestore/TestBase.h>
32 
33 namespace ripple {
34 namespace NodeStore {
35 
36 // Tests DatabaseShard class
37 //
38 class DatabaseShard_test : public TestBase
39 {
40  static constexpr std::uint32_t maxSizeGb = 10;
41  static constexpr std::uint32_t maxHistoricalShards = 100;
42  static constexpr std::uint32_t ledgersPerShard = 256;
43  static constexpr std::uint32_t earliestSeq = ledgersPerShard + 1;
44  static constexpr std::uint32_t dataSizeMax = 4;
45  static constexpr std::uint32_t iniAmount = 1000000;
46  static constexpr std::uint32_t nTestShards = 4;
47  static constexpr std::chrono::seconds shardStoreTimeout =
48  std::chrono::seconds(60);
49  test::SuiteJournal journal_;
50  beast::temp_dir defNodeDir;
51 
52  struct TestData
53  {
54  /* ring used to generate pseudo-random sequence */
55  beast::xor_shift_engine rng_;
56  /* number of shards to generate */
57  int nShards_;
58  /* vector of accounts used to send test transactions */
59  std::vector<test::jtx::Account> accounts_;
60  /* nAccounts_[i] is the number of these accounts existed before i-th
61  * ledger */
62  std::vector<int> nAccounts_;
63  /* payAccounts_[i][j] = {from, to} is the pair which consists of two
64  * number of accounts: source and destinations, which participate in
65  * j-th payment on i-th ledger */
66  std::vector<std::vector<std::pair<int, int>>> payAccounts_;
67  /* xrpAmount_[i] is the amount for all payments on i-th ledger */
68  std::vector<int> xrpAmount_;
69  /* ledgers_[i] is the i-th ledger which contains the above described
70  * accounts and payments */
71  std::vector<std::shared_ptr<const Ledger>> ledgers_;
72 
73  TestData(
74  std::uint64_t const seedValue,
75  int dataSize = dataSizeMax,
76  int nShards = 1)
77  : rng_(seedValue), nShards_(nShards)
78  {
79  std::uint32_t n = 0;
80  std::uint32_t nLedgers = ledgersPerShard * nShards;
81 
82  nAccounts_.reserve(nLedgers);
83  payAccounts_.reserve(nLedgers);
84  xrpAmount_.reserve(nLedgers);
85 
86  for (std::uint32_t i = 0; i < nLedgers; ++i)
87  {
88  int p;
89  if (n >= 2)
90  p = rand_int(rng_, 2 * dataSize);
91  else
92  p = 0;
93 
94  std::vector<std::pair<int, int>> pay;
95  pay.reserve(p);
96 
97  for (int j = 0; j < p; ++j)
98  {
99  int from, to;
100  do
101  {
102  from = rand_int(rng_, n - 1);
103  to = rand_int(rng_, n - 1);
104  } while (from == to);
105 
106  pay.push_back(std::make_pair(from, to));
107  }
108 
109  n += !rand_int(rng_, nLedgers / dataSize);
110 
111  if (n > accounts_.size())
112  {
113  char str[9];
114  for (int j = 0; j < 8; ++j)
115  str[j] = 'a' + rand_int(rng_, 'z' - 'a');
116  str[8] = 0;
117  accounts_.emplace_back(str);
118  }
119 
120  nAccounts_.push_back(n);
121  payAccounts_.push_back(std::move(pay));
122  xrpAmount_.push_back(rand_int(rng_, 90) + 10);
123  }
124  }
125 
126  bool
127  isNewAccounts(int seq)
128  {
129  return nAccounts_[seq] > (seq ? nAccounts_[seq - 1] : 0);
130  }
131 
132  void
133  makeLedgerData(test::jtx::Env& env_, std::uint32_t seq)
134  {
135  using namespace test::jtx;
136 
137  if (isNewAccounts(seq))
138  env_.fund(XRP(iniAmount), accounts_[nAccounts_[seq] - 1]);
139 
140  for (std::uint32_t i = 0; i < payAccounts_[seq].size(); ++i)
141  {
142  env_(
143  pay(accounts_[payAccounts_[seq][i].first],
144  accounts_[payAccounts_[seq][i].second],
145  XRP(xrpAmount_[seq])));
146  }
147  }
148 
149  bool
150  makeLedgers(test::jtx::Env& env_, std::uint32_t startIndex = 0)
151  {
152  if (startIndex == 0)
153  {
154  for (std::uint32_t i = 3; i <= ledgersPerShard; ++i)
155  {
156  if (!env_.close())
157  return false;
158  std::shared_ptr<const Ledger> ledger =
159  env_.app().getLedgerMaster().getClosedLedger();
160  if (ledger->info().seq != i)
161  return false;
162  }
163  }
164 
165  for (std::uint32_t i = 0; i < ledgersPerShard * nShards_; ++i)
166  {
167  auto const index = i + (startIndex * ledgersPerShard);
168 
169  makeLedgerData(env_, i);
170  if (!env_.close())
171  return false;
172  std::shared_ptr<const Ledger> ledger =
173  env_.app().getLedgerMaster().getClosedLedger();
174  if (ledger->info().seq != index + ledgersPerShard + 1)
175  return false;
176  ledgers_.push_back(ledger);
177  }
178 
179  return true;
180  }
181  };
182 
183  void
184  testLedgerData(
185  TestData& data,
186  std::shared_ptr<Ledger> ledger,
187  std::uint32_t seq)
188  {
189  using namespace test::jtx;
190 
191  auto rootCount{0};
192  auto accCount{0};
193  auto sothCount{0};
194  for (auto const& sles : ledger->sles)
195  {
196  if (sles->getType() == ltACCOUNT_ROOT)
197  {
198  int sq = sles->getFieldU32(sfSequence);
199  int reqsq = -1;
200  const auto id = sles->getAccountID(sfAccount);
201 
202  for (int i = 0; i < data.accounts_.size(); ++i)
203  {
204  if (id == data.accounts_[i].id())
205  {
206  reqsq = ledgersPerShard + 1;
207  for (int j = 0; j <= seq; ++j)
208  if (data.nAccounts_[j] > i + 1 ||
209  (data.nAccounts_[j] == i + 1 &&
210  !data.isNewAccounts(j)))
211  {
212  for (int k = 0; k < data.payAccounts_[j].size();
213  ++k)
214  if (data.payAccounts_[j][k].first == i)
215  reqsq++;
216  }
217  else
218  reqsq++;
219  ++accCount;
220  break;
221  }
222  }
223  if (reqsq == -1)
224  {
225  reqsq = data.nAccounts_[seq] + 1;
226  ++rootCount;
227  }
228  BEAST_EXPECT(sq == reqsq);
229  }
230  else
231  ++sothCount;
232  }
233  BEAST_EXPECT(rootCount == 1);
234  BEAST_EXPECT(accCount == data.nAccounts_[seq]);
235  BEAST_EXPECT(sothCount == 3);
236 
237  auto iniCount{0};
238  auto setCount{0};
239  auto payCount{0};
240  auto tothCount{0};
241  for (auto const& tx : ledger->txs)
242  {
243  if (tx.first->getTxnType() == ttPAYMENT)
244  {
245  std::int64_t xrpAmount =
246  tx.first->getFieldAmount(sfAmount).xrp().decimalXRP();
247  if (xrpAmount == iniAmount)
248  ++iniCount;
249  else
250  {
251  ++payCount;
252  BEAST_EXPECT(xrpAmount == data.xrpAmount_[seq]);
253  }
254  }
255  else if (tx.first->getTxnType() == ttACCOUNT_SET)
256  ++setCount;
257  else
258  ++tothCount;
259  }
260  int newacc = data.isNewAccounts(seq) ? 1 : 0;
261  BEAST_EXPECT(iniCount == newacc);
262  BEAST_EXPECT(setCount == newacc);
263  BEAST_EXPECT(payCount == data.payAccounts_[seq].size());
264  BEAST_EXPECT(tothCount == !seq);
265  }
266 
267  bool
268  saveLedger(
269  Database& db,
270  Ledger const& ledger,
271  std::shared_ptr<Ledger const> const& next = {})
272  {
273  // Store header
274  {
275  Serializer s(128);
276  s.add32(HashPrefix::ledgerMaster);
277  addRaw(ledger.info(), s);
278  db.store(
279  hotLEDGER,
280  std::move(s.modData()),
281  ledger.info().hash,
282  ledger.info().seq);
283  }
284 
285  // Store the state map
286  auto visitAcc = [&](SHAMapAbstractNode& node) {
287  Serializer s;
288  node.addRaw(s, snfPREFIX);
289  db.store(
290  node.getType() == SHAMapAbstractNode::TNType::tnINNER
291  ? hotUNKNOWN
292  : hotACCOUNT_NODE,
293  std::move(s.modData()),
294  node.getNodeHash().as_uint256(),
295  ledger.info().seq);
296  return true;
297  };
298 
299  if (ledger.stateMap().getHash().isNonZero())
300  {
301  if (!ledger.stateMap().isValid())
302  return false;
303  if (next && next->info().parentHash == ledger.info().hash)
304  {
305  auto have = next->stateMap().snapShot(false);
306  ledger.stateMap().snapShot(false)->visitDifferences(
307  &(*have), visitAcc);
308  }
309  else
310  ledger.stateMap().snapShot(false)->visitNodes(visitAcc);
311  }
312 
313  // Store the transaction map
314  auto visitTx = [&](SHAMapAbstractNode& node) {
315  Serializer s;
316  node.addRaw(s, snfPREFIX);
317  db.store(
318  node.getType() == SHAMapAbstractNode::TNType::tnINNER
319  ? hotUNKNOWN
320  : hotTRANSACTION_NODE,
321  std::move(s.modData()),
322  node.getNodeHash().as_uint256(),
323  ledger.info().seq);
324  return true;
325  };
326 
327  if (ledger.info().txHash.isNonZero())
328  {
329  if (!ledger.txMap().isValid())
330  return false;
331  ledger.txMap().snapShot(false)->visitNodes(visitTx);
332  }
333 
334  return true;
335  }
336 
337  void
338  checkLedger(TestData& data, DatabaseShard& db, Ledger const& ledger)
339  {
340  auto fetched = db.fetchLedger(ledger.info().hash, ledger.info().seq);
341  if (!BEAST_EXPECT(fetched))
342  return;
343 
344  testLedgerData(data, fetched, ledger.info().seq - ledgersPerShard - 1);
345 
346  // verify the metadata/header info by serializing to json
347  BEAST_EXPECT(
348  getJson(
349  LedgerFill{ledger, LedgerFill::full | LedgerFill::expand}) ==
350  getJson(
351  LedgerFill{*fetched, LedgerFill::full | LedgerFill::expand}));
352 
353  BEAST_EXPECT(
354  getJson(
355  LedgerFill{ledger, LedgerFill::full | LedgerFill::binary}) ==
356  getJson(
357  LedgerFill{*fetched, LedgerFill::full | LedgerFill::binary}));
358 
359  // walk shamap and validate each node
360  auto fcompAcc = [&](SHAMapAbstractNode& node) -> bool {
361  Serializer s;
362  node.addRaw(s, snfPREFIX);
363  auto nSrc{NodeObject::createObject(
364  node.getType() == SHAMapAbstractNode::TNType::tnINNER
365  ? hotUNKNOWN
366  : hotACCOUNT_NODE,
367  std::move(s.modData()),
368  node.getNodeHash().as_uint256())};
369  if (!BEAST_EXPECT(nSrc))
370  return false;
371 
372  auto nDst =
373  db.fetch(node.getNodeHash().as_uint256(), ledger.info().seq);
374  if (!BEAST_EXPECT(nDst))
375  return false;
376 
377  BEAST_EXPECT(isSame(nSrc, nDst));
378 
379  return true;
380  };
381  if (ledger.stateMap().getHash().isNonZero())
382  ledger.stateMap().snapShot(false)->visitNodes(fcompAcc);
383 
384  auto fcompTx = [&](SHAMapAbstractNode& node) -> bool {
385  Serializer s;
386  node.addRaw(s, snfPREFIX);
387  auto nSrc{NodeObject::createObject(
388  node.getType() == SHAMapAbstractNode::TNType::tnINNER
389  ? hotUNKNOWN
390  : hotTRANSACTION_NODE,
391  std::move(s.modData()),
392  node.getNodeHash().as_uint256())};
393  if (!BEAST_EXPECT(nSrc))
394  return false;
395 
396  auto nDst =
397  db.fetch(node.getNodeHash().as_uint256(), ledger.info().seq);
398  if (!BEAST_EXPECT(nDst))
399  return false;
400 
401  BEAST_EXPECT(isSame(nSrc, nDst));
402 
403  return true;
404  };
405  if (ledger.info().txHash.isNonZero())
406  ledger.txMap().snapShot(false)->visitNodes(fcompTx);
407  }
408 
409  std::string
410  bitmask2Rangeset(std::uint64_t bitmask)
411  {
412  std::string set;
413  if (!bitmask)
414  return set;
415  bool empty = true;
416 
417  for (std::uint32_t i = 0; i < 64 && bitmask; i++)
418  {
419  if (bitmask & (1ll << i))
420  {
421  if (!empty)
422  set += ",";
423  set += std::to_string(i);
424  empty = false;
425  }
426  }
427 
428  RangeSet<std::uint32_t> rs;
429  from_string(rs, set);
430  return to_string(rs);
431  }
432 
433  std::unique_ptr<Config>
434  testConfig(
435  std::string const& testName,
436  std::string const& backendType,
437  std::string const& shardDir,
438  std::string const& nodeDir = std::string())
439  {
440  using namespace test::jtx;
441 
442  if (testName != "")
443  {
444  std::string caseName =
445  "DatabaseShard " + testName + " with backend " + backendType;
446  testcase(caseName);
447  }
448 
449  return envconfig([&](std::unique_ptr<Config> cfg) {
450  cfg->overwrite(ConfigSection::shardDatabase(), "type", backendType);
451  cfg->overwrite(ConfigSection::shardDatabase(), "path", shardDir);
452  cfg->overwrite(
453  ConfigSection::shardDatabase(),
454  "max_historical_shards",
455  std::to_string(maxHistoricalShards));
456  cfg->overwrite(
457  ConfigSection::shardDatabase(),
458  "ledgers_per_shard",
459  std::to_string(ledgersPerShard));
460  cfg->overwrite(
461  ConfigSection::shardDatabase(),
462  "earliest_seq",
463  std::to_string(earliestSeq));
464  cfg->overwrite(ConfigSection::nodeDatabase(), "type", backendType);
465  cfg->overwrite(
466  ConfigSection::nodeDatabase(),
467  "max_size_gb",
468  std::to_string(maxSizeGb));
469  cfg->overwrite(
470  ConfigSection::nodeDatabase(),
471  "earliest_seq",
472  std::to_string(earliestSeq));
473  if (nodeDir.empty())
474  cfg->overwrite(
475  ConfigSection::nodeDatabase(), "path", defNodeDir.path());
476  else
477  cfg->overwrite(ConfigSection::nodeDatabase(), "path", nodeDir);
478  return cfg;
479  });
480  }
481 
482  std::optional<int>
483  waitShard(
484  DatabaseShard& db,
485  int shardNumber,
486  std::chrono::seconds timeout = shardStoreTimeout)
487  {
488  RangeSet<std::uint32_t> rs;
489  auto start = std::chrono::system_clock::now();
490  auto end = start + timeout;
491  while (!from_string(rs, db.getCompleteShards()) ||
492  !boost::icl::contains(rs, shardNumber))
493  {
494  if (!BEAST_EXPECT(std::chrono::system_clock::now() < end))
495  return {};
496  std::this_thread::yield();
497  }
498 
499  return shardNumber;
500  }
501 
502  std::optional<int>
503  createShard(
504  TestData& data,
505  DatabaseShard& db,
506  int maxShardNumber = 1,
507  int ledgerOffset = 0)
508  {
509  int shardNumber = -1;
510 
511  for (std::uint32_t i = 0; i < ledgersPerShard; ++i)
512  {
513  auto ind = db.prepareLedger((maxShardNumber + 1) * ledgersPerShard);
514  if (!BEAST_EXPECT(ind != boost::none))
515  return {};
516  shardNumber = db.seqToShardIndex(*ind);
517  int arrInd =
518  *ind - (ledgersPerShard * ledgerOffset) - ledgersPerShard - 1;
519  BEAST_EXPECT(
520  arrInd >= 0 && arrInd < maxShardNumber * ledgersPerShard);
521  BEAST_EXPECT(saveLedger(db, *data.ledgers_[arrInd]));
522  if (arrInd % ledgersPerShard == (ledgersPerShard - 1))
523  {
524  uint256 const finalKey_{0};
525  Serializer s;
526  s.add32(Shard::version);
527  s.add32(db.firstLedgerSeq(shardNumber));
528  s.add32(db.lastLedgerSeq(shardNumber));
529  s.addRaw(data.ledgers_[arrInd]->info().hash.data(), 256 / 8);
530  db.store(hotUNKNOWN, std::move(s.modData()), finalKey_, *ind);
531  }
532  db.setStored(data.ledgers_[arrInd]);
533  }
534 
535  return waitShard(db, shardNumber);
536  }
537 
538  void
539  testStandalone(std::string const& backendType)
540  {
541  using namespace test::jtx;
542 
543  beast::temp_dir shardDir;
544  Env env{*this, testConfig("standalone", backendType, shardDir.path())};
545  DummyScheduler scheduler;
546  RootStoppable parent("TestRootStoppable");
547 
548  std::unique_ptr<DatabaseShard> db =
549  make_ShardStore(env.app(), parent, scheduler, 2, journal_);
550 
551  BEAST_EXPECT(db);
552  BEAST_EXPECT(db->ledgersPerShard() == db->ledgersPerShardDefault);
553  BEAST_EXPECT(db->init());
554  BEAST_EXPECT(db->ledgersPerShard() == ledgersPerShard);
555  BEAST_EXPECT(db->seqToShardIndex(ledgersPerShard + 1) == 1);
556  BEAST_EXPECT(db->seqToShardIndex(2 * ledgersPerShard) == 1);
557  BEAST_EXPECT(db->seqToShardIndex(2 * ledgersPerShard + 1) == 2);
558  BEAST_EXPECT(
559  db->earliestShardIndex() == (earliestSeq - 1) / ledgersPerShard);
560  BEAST_EXPECT(db->firstLedgerSeq(1) == ledgersPerShard + 1);
561  BEAST_EXPECT(db->lastLedgerSeq(1) == 2 * ledgersPerShard);
562  BEAST_EXPECT(db->getRootDir().string() == shardDir.path());
563  }
564 
565  void
566  testCreateShard(
567  std::string const& backendType,
568  std::uint64_t const seedValue)
569  {
570  using namespace test::jtx;
571 
572  beast::temp_dir shardDir;
573  Env env{*this, testConfig("createShard", backendType, shardDir.path())};
574  DatabaseShard* db = env.app().getShardStore();
575  BEAST_EXPECT(db);
576 
577  TestData data(seedValue);
578  if (!BEAST_EXPECT(data.makeLedgers(env)))
579  return;
580 
581  if (!createShard(data, *db, 1))
582  return;
583 
584  for (std::uint32_t i = 0; i < ledgersPerShard; ++i)
585  checkLedger(data, *db, *data.ledgers_[i]);
586  }
587 
588  void
589  testReopenDatabase(
590  std::string const& backendType,
591  std::uint64_t const seedValue)
592  {
593  using namespace test::jtx;
594 
595  beast::temp_dir shardDir;
596  {
597  Env env{
598  *this,
599  testConfig("reopenDatabase", backendType, shardDir.path())};
600  DatabaseShard* db = env.app().getShardStore();
601  BEAST_EXPECT(db);
602 
603  TestData data(seedValue, 4, 2);
604  if (!BEAST_EXPECT(data.makeLedgers(env)))
605  return;
606 
607  for (std::uint32_t i = 0; i < 2; ++i)
608  if (!createShard(data, *db, 2))
609  return;
610  }
611  {
612  Env env{*this, testConfig("", backendType, shardDir.path())};
613  DatabaseShard* db = env.app().getShardStore();
614  BEAST_EXPECT(db);
615 
616  TestData data(seedValue, 4, 2);
617  if (!BEAST_EXPECT(data.makeLedgers(env)))
618  return;
619 
620  for (std::uint32_t i = 1; i <= 2; ++i)
621  waitShard(*db, i);
622 
623  for (std::uint32_t i = 0; i < 2 * ledgersPerShard; ++i)
624  checkLedger(data, *db, *data.ledgers_[i]);
625  }
626  }
627 
628  void
629  testGetCompleteShards(
630  std::string const& backendType,
631  std::uint64_t const seedValue)
632  {
633  using namespace test::jtx;
634 
635  beast::temp_dir shardDir;
636  Env env{
637  *this,
638  testConfig("getCompleteShards", backendType, shardDir.path())};
639  DatabaseShard* db = env.app().getShardStore();
640  BEAST_EXPECT(db);
641 
642  TestData data(seedValue, 2, nTestShards);
643  if (!BEAST_EXPECT(data.makeLedgers(env)))
644  return;
645 
646  BEAST_EXPECT(db->getCompleteShards() == "");
647 
648  std::uint64_t bitMask = 0;
649 
650  for (std::uint32_t i = 0; i < nTestShards; ++i)
651  {
652  auto n = createShard(data, *db, nTestShards);
653  if (!BEAST_EXPECT(n && *n >= 1 && *n <= nTestShards))
654  return;
655  bitMask |= 1ll << *n;
656  BEAST_EXPECT(db->getCompleteShards() == bitmask2Rangeset(bitMask));
657  }
658  }
659 
660  void
661  testPrepareShard(
662  std::string const& backendType,
663  std::uint64_t const seedValue)
664  {
665  using namespace test::jtx;
666 
667  beast::temp_dir shardDir;
668  Env env{
669  *this, testConfig("prepareShard", backendType, shardDir.path())};
670  DatabaseShard* db = env.app().getShardStore();
671  BEAST_EXPECT(db);
672 
673  TestData data(seedValue, 1, nTestShards);
674  if (!BEAST_EXPECT(data.makeLedgers(env)))
675  return;
676 
677  std::uint64_t bitMask = 0;
678  BEAST_EXPECT(db->getPreShards() == "");
679 
680  for (std::uint32_t i = 0; i < nTestShards * 2; ++i)
681  {
682  std::uint32_t n = rand_int(data.rng_, nTestShards - 1) + 1;
683  if (bitMask & (1ll << n))
684  {
685  db->removePreShard(n);
686  bitMask &= ~(1ll << n);
687  }
688  else
689  {
690  db->prepareShard(n);
691  bitMask |= 1ll << n;
692  }
693  BEAST_EXPECT(db->getPreShards() == bitmask2Rangeset(bitMask));
694  }
695 
696  // test illegal cases
697  // adding shards with too large number
698  db->prepareShard(0);
699  BEAST_EXPECT(db->getPreShards() == bitmask2Rangeset(bitMask));
700  db->prepareShard(nTestShards + 1);
701  BEAST_EXPECT(db->getPreShards() == bitmask2Rangeset(bitMask));
702  db->prepareShard(nTestShards + 2);
703  BEAST_EXPECT(db->getPreShards() == bitmask2Rangeset(bitMask));
704 
705  // create shards which are not prepared for import
706  BEAST_EXPECT(db->getCompleteShards() == "");
707 
708  std::uint64_t bitMask2 = 0;
709 
710  for (std::uint32_t i = 0; i < nTestShards; ++i)
711  {
712  auto n = createShard(data, *db, nTestShards);
713  if (!BEAST_EXPECT(n && *n >= 1 && *n <= nTestShards))
714  return;
715  bitMask2 |= 1ll << *n;
716  BEAST_EXPECT(db->getPreShards() == bitmask2Rangeset(bitMask));
717  BEAST_EXPECT(db->getCompleteShards() == bitmask2Rangeset(bitMask2));
718  BEAST_EXPECT((bitMask & bitMask2) == 0);
719  if ((bitMask | bitMask2) == ((1ll << nTestShards) - 1) << 1)
720  break;
721  }
722 
723  // try to create another shard
724  BEAST_EXPECT(
725  db->prepareLedger((nTestShards + 1) * ledgersPerShard) ==
726  boost::none);
727  }
728 
729  void
730  testImportShard(
731  std::string const& backendType,
732  std::uint64_t const seedValue)
733  {
734  using namespace test::jtx;
735 
736  beast::temp_dir importDir;
737  TestData data(seedValue, 2);
738 
739  {
740  Env env{
741  *this,
742  testConfig("importShard", backendType, importDir.path())};
743  DatabaseShard* db = env.app().getShardStore();
744  BEAST_EXPECT(db);
745 
746  if (!BEAST_EXPECT(data.makeLedgers(env)))
747  return;
748 
749  if (!createShard(data, *db, 1))
750  return;
751 
752  for (std::uint32_t i = 0; i < ledgersPerShard; ++i)
753  checkLedger(data, *db, *data.ledgers_[i]);
754 
755  data.ledgers_.clear();
756  }
757 
758  boost::filesystem::path importPath(importDir.path());
759  importPath /= "1";
760 
761  {
762  beast::temp_dir shardDir;
763  Env env{*this, testConfig("", backendType, shardDir.path())};
764  DatabaseShard* db = env.app().getShardStore();
765  BEAST_EXPECT(db);
766 
767  if (!BEAST_EXPECT(data.makeLedgers(env)))
768  return;
769 
770  db->prepareShard(1);
771  BEAST_EXPECT(db->getPreShards() == bitmask2Rangeset(2));
772  if (!BEAST_EXPECT(db->importShard(1, importPath)))
773  return;
774  BEAST_EXPECT(db->getPreShards() == "");
775 
776  auto n = waitShard(*db, 1);
777  if (!BEAST_EXPECT(n && *n == 1))
778  return;
779 
780  for (std::uint32_t i = 0; i < ledgersPerShard; ++i)
781  checkLedger(data, *db, *data.ledgers_[i]);
782  }
783  }
784 
785  void
786  testCorruptedDatabase(
787  std::string const& backendType,
788  std::uint64_t const seedValue)
789  {
790  using namespace test::jtx;
791 
792  beast::temp_dir shardDir;
793  {
794  TestData data(seedValue, 4, 2);
795  {
796  Env env{
797  *this,
798  testConfig(
799  "corruptedDatabase", backendType, shardDir.path())};
800  DatabaseShard* db = env.app().getShardStore();
801  BEAST_EXPECT(db);
802 
803  if (!BEAST_EXPECT(data.makeLedgers(env)))
804  return;
805 
806  for (std::uint32_t i = 0; i < 2; ++i)
807  if (!BEAST_EXPECT(createShard(data, *db, 2)))
808  return;
809  }
810 
811  boost::filesystem::path path = shardDir.path();
812  path /= std::string("2");
813  path /= backendType + ".dat";
814 
815  FILE* f = fopen(path.string().c_str(), "r+b");
816  if (!BEAST_EXPECT(f))
817  return;
818  char buf[256];
819  beast::rngfill(buf, sizeof(buf), data.rng_);
820  BEAST_EXPECT(fwrite(buf, 1, 256, f) == 256);
821  fclose(f);
822  }
823  {
824  Env env{*this, testConfig("", backendType, shardDir.path())};
825  DatabaseShard* db = env.app().getShardStore();
826  BEAST_EXPECT(db);
827 
828  TestData data(seedValue, 4, 2);
829  if (!BEAST_EXPECT(data.makeLedgers(env)))
830  return;
831 
832  for (std::uint32_t i = 1; i <= 1; ++i)
833  waitShard(*db, i);
834 
835  BEAST_EXPECT(db->getCompleteShards() == bitmask2Rangeset(0x2));
836 
837  for (std::uint32_t i = 0; i < 1 * ledgersPerShard; ++i)
838  checkLedger(data, *db, *data.ledgers_[i]);
839  }
840  }
841 
842  void
843  testIllegalFinalKey(
844  std::string const& backendType,
845  std::uint64_t const seedValue)
846  {
847  using namespace test::jtx;
848 
849  for (int i = 0; i < 5; ++i)
850  {
851  beast::temp_dir shardDir;
852  {
853  Env env{
854  *this,
855  testConfig(
856  (i == 0 ? "illegalFinalKey" : ""),
857  backendType,
858  shardDir.path())};
859  DatabaseShard* db = env.app().getShardStore();
860  BEAST_EXPECT(db);
861 
862  TestData data(seedValue + i, 2);
863  if (!BEAST_EXPECT(data.makeLedgers(env)))
864  return;
865 
866  int shardNumber = -1;
867  for (std::uint32_t j = 0; j < ledgersPerShard; ++j)
868  {
869  auto ind = db->prepareLedger(2 * ledgersPerShard);
870  if (!BEAST_EXPECT(ind != boost::none))
871  return;
872  shardNumber = db->seqToShardIndex(*ind);
873  int arrInd = *ind - ledgersPerShard - 1;
874  BEAST_EXPECT(arrInd >= 0 && arrInd < ledgersPerShard);
875  BEAST_EXPECT(saveLedger(*db, *data.ledgers_[arrInd]));
876  if (arrInd % ledgersPerShard == (ledgersPerShard - 1))
877  {
878  uint256 const finalKey_{0};
879  Serializer s;
880  s.add32(Shard::version + (i == 0));
881  s.add32(db->firstLedgerSeq(shardNumber) + (i == 1));
882  s.add32(db->lastLedgerSeq(shardNumber) - (i == 3));
883  s.addRaw(
884  data.ledgers_[arrInd - (i == 4)]
885  ->info()
886  .hash.data(),
887  256 / 8);
888  db->store(
889  hotUNKNOWN,
890  std::move(s.modData()),
891  finalKey_,
892  *ind);
893  }
894  db->setStored(data.ledgers_[arrInd]);
895  }
896 
897  if (i == 2)
898  waitShard(*db, shardNumber);
899  else
900  {
901  boost::filesystem::path path(shardDir.path());
902  path /= "1";
903  boost::system::error_code ec;
904  auto start = std::chrono::system_clock::now();
905  auto end = start + shardStoreTimeout;
906  while (std::chrono::system_clock::now() < end &&
907  boost::filesystem::exists(path, ec))
908  {
909  std::this_thread::yield();
910  }
911  }
912 
913  BEAST_EXPECT(
914  db->getCompleteShards() ==
915  bitmask2Rangeset(i == 2 ? 2 : 0));
916  }
917 
918  {
919  Env env{*this, testConfig("", backendType, shardDir.path())};
920  DatabaseShard* db = env.app().getShardStore();
921  BEAST_EXPECT(db);
922 
923  TestData data(seedValue + i, 2);
924  if (!BEAST_EXPECT(data.makeLedgers(env)))
925  return;
926 
927  if (i == 2)
928  waitShard(*db, 1);
929 
930  BEAST_EXPECT(
931  db->getCompleteShards() ==
932  bitmask2Rangeset(i == 2 ? 2 : 0));
933 
934  if (i == 2)
935  {
936  for (std::uint32_t j = 0; j < ledgersPerShard; ++j)
937  checkLedger(data, *db, *data.ledgers_[j]);
938  }
939  }
940  }
941  }
942 
943  void
944  testImport(std::string const& backendType, std::uint64_t const seedValue)
945  {
946  using namespace test::jtx;
947 
948  beast::temp_dir shardDir;
949  {
950  beast::temp_dir nodeDir;
951  Env env{
952  *this,
953  testConfig(
954  "import", backendType, shardDir.path(), nodeDir.path())};
955  DatabaseShard* db = env.app().getShardStore();
956  Database& ndb = env.app().getNodeStore();
957  BEAST_EXPECT(db);
958 
959  TestData data(seedValue, 4, 2);
960  if (!BEAST_EXPECT(data.makeLedgers(env)))
961  return;
962 
963  for (std::uint32_t i = 0; i < 2 * ledgersPerShard; ++i)
964  BEAST_EXPECT(saveLedger(ndb, *data.ledgers_[i]));
965 
966  BEAST_EXPECT(db->getCompleteShards() == bitmask2Rangeset(0));
967  db->import(ndb);
968  for (std::uint32_t i = 1; i <= 2; ++i)
969  waitShard(*db, i);
970  BEAST_EXPECT(db->getCompleteShards() == bitmask2Rangeset(0x6));
971  }
972  {
973  Env env{*this, testConfig("", backendType, shardDir.path())};
974  DatabaseShard* db = env.app().getShardStore();
975  BEAST_EXPECT(db);
976 
977  TestData data(seedValue, 4, 2);
978  if (!BEAST_EXPECT(data.makeLedgers(env)))
979  return;
980 
981  for (std::uint32_t i = 1; i <= 2; ++i)
982  waitShard(*db, i);
983 
984  BEAST_EXPECT(db->getCompleteShards() == bitmask2Rangeset(0x6));
985 
986  for (std::uint32_t i = 0; i < 2 * ledgersPerShard; ++i)
987  checkLedger(data, *db, *data.ledgers_[i]);
988  }
989  }
990 
991  void
992  testImportWithHistoricalPaths(
993  std::string const& backendType,
994  std::uint64_t const seedValue)
995  {
996  using namespace test::jtx;
997 
998  // Test importing with multiple historical
999  // paths
1000  {
1001  beast::temp_dir shardDir;
1002  std::array<beast::temp_dir, 4> historicalDirs;
1003  std::array<boost::filesystem::path, 4> historicalPaths;
1004 
1005  std::transform(
1006  historicalDirs.begin(),
1007  historicalDirs.end(),
1008  historicalPaths.begin(),
1009  [](const beast::temp_dir& dir) { return dir.path(); });
1010 
1011  beast::temp_dir nodeDir;
1012  auto c = testConfig(
1013  "importWithHistoricalPaths",
1014  backendType,
1015  shardDir.path(),
1016  nodeDir.path());
1017 
1018  auto& historyPaths = c->section(SECTION_HISTORICAL_SHARD_PATHS);
1019  historyPaths.append(
1020  {historicalPaths[0].string(),
1021  historicalPaths[1].string(),
1022  historicalPaths[2].string(),
1023  historicalPaths[3].string()});
1024 
1025  Env env{*this, std::move(c)};
1026  DatabaseShard* db = env.app().getShardStore();
1027  Database& ndb = env.app().getNodeStore();
1028  BEAST_EXPECT(db);
1029 
1030  auto const ledgerCount = 4;
1031 
1032  TestData data(seedValue, 4, ledgerCount);
1033  if (!BEAST_EXPECT(data.makeLedgers(env)))
1034  return;
1035 
1036  for (std::uint32_t i = 0; i < ledgerCount * ledgersPerShard; ++i)
1037  BEAST_EXPECT(saveLedger(ndb, *data.ledgers_[i]));
1038 
1039  BEAST_EXPECT(db->getCompleteShards() == bitmask2Rangeset(0));
1040 
1041  db->import(ndb);
1042  for (std::uint32_t i = 1; i <= ledgerCount; ++i)
1043  waitShard(*db, i);
1044 
1045  BEAST_EXPECT(db->getCompleteShards() == bitmask2Rangeset(0b11110));
1046 
1047  auto const mainPathCount = std::distance(
1048  boost::filesystem::directory_iterator(shardDir.path()),
1049  boost::filesystem::directory_iterator());
1050 
1051  // Only the two most recent shards
1052  // should be stored at the main path
1053  BEAST_EXPECT(mainPathCount == 2);
1054 
1055  auto const historicalPathCount = std::accumulate(
1056  historicalPaths.begin(),
1057  historicalPaths.end(),
1058  0,
1059  [](int const sum, boost::filesystem::path const& path) {
1060  return sum +
1061  std::distance(
1062  boost::filesystem::directory_iterator(path),
1063  boost::filesystem::directory_iterator());
1064  });
1065 
1066  // All historical shards should be stored
1067  // at historical paths
1068  BEAST_EXPECT(historicalPathCount == ledgerCount - 2);
1069  }
1070 
1071  // Test importing with a single historical
1072  // path
1073  {
1074  beast::temp_dir shardDir;
1075  beast::temp_dir historicalDir;
1076  beast::temp_dir nodeDir;
1077 
1078  auto c = testConfig(
1079  "importWithSingleHistoricalPath",
1080  backendType,
1081  shardDir.path(),
1082  nodeDir.path());
1083 
1084  auto& historyPaths = c->section(SECTION_HISTORICAL_SHARD_PATHS);
1085  historyPaths.append({historicalDir.path()});
1086 
1087  Env env{*this, std::move(c)};
1088  DatabaseShard* db = env.app().getShardStore();
1089  Database& ndb = env.app().getNodeStore();
1090  BEAST_EXPECT(db);
1091 
1092  auto const ledgerCount = 4;
1093 
1094  TestData data(seedValue * 2, 4, ledgerCount);
1095  if (!BEAST_EXPECT(data.makeLedgers(env)))
1096  return;
1097 
1098  for (std::uint32_t i = 0; i < ledgerCount * ledgersPerShard; ++i)
1099  BEAST_EXPECT(saveLedger(ndb, *data.ledgers_[i]));
1100 
1101  BEAST_EXPECT(db->getCompleteShards() == bitmask2Rangeset(0));
1102 
1103  db->import(ndb);
1104  for (std::uint32_t i = 1; i <= ledgerCount; ++i)
1105  waitShard(*db, i);
1106 
1107  BEAST_EXPECT(db->getCompleteShards() == bitmask2Rangeset(0b11110));
1108 
1109  auto const mainPathCount = std::distance(
1110  boost::filesystem::directory_iterator(shardDir.path()),
1111  boost::filesystem::directory_iterator());
1112 
1113  // Only the two most recent shards
1114  // should be stored at the main path
1115  BEAST_EXPECT(mainPathCount == 2);
1116 
1117  auto const historicalPathCount = std::distance(
1118  boost::filesystem::directory_iterator(historicalDir.path()),
1119  boost::filesystem::directory_iterator());
1120 
1121  // All historical shards should be stored
1122  // at historical paths
1123  BEAST_EXPECT(historicalPathCount == ledgerCount - 2);
1124  }
1125  }
1126 
1127  void
1128  testPrepareWithHistoricalPaths(
1129  std::string const& backendType,
1130  std::uint64_t const seedValue)
1131  {
1132  using namespace test::jtx;
1133 
1134  // Test importing with multiple historical
1135  // paths
1136  {
1137  beast::temp_dir shardDir;
1138  std::array<beast::temp_dir, 4> historicalDirs;
1139  std::array<boost::filesystem::path, 4> historicalPaths;
1140 
1141  std::transform(
1142  historicalDirs.begin(),
1143  historicalDirs.end(),
1144  historicalPaths.begin(),
1145  [](const beast::temp_dir& dir) { return dir.path(); });
1146 
1147  beast::temp_dir nodeDir;
1148  auto c = testConfig(
1149  "prepareWithHistoricalPaths", backendType, shardDir.path());
1150 
1151  auto& historyPaths = c->section(SECTION_HISTORICAL_SHARD_PATHS);
1152  historyPaths.append(
1153  {historicalPaths[0].string(),
1154  historicalPaths[1].string(),
1155  historicalPaths[2].string(),
1156  historicalPaths[3].string()});
1157 
1158  Env env{*this, std::move(c)};
1159  DatabaseShard* db = env.app().getShardStore();
1160  BEAST_EXPECT(db);
1161 
1162  auto const ledgerCount = 4;
1163 
1164  TestData data(seedValue, 4, ledgerCount);
1165  if (!BEAST_EXPECT(data.makeLedgers(env)))
1166  return;
1167 
1168  BEAST_EXPECT(db->getCompleteShards() == "");
1169  std::uint64_t bitMask = 0;
1170 
1171  // Add ten shards to the Shard Database
1172  for (std::uint32_t i = 0; i < ledgerCount; ++i)
1173  {
1174  auto n = createShard(data, *db, ledgerCount);
1175  if (!BEAST_EXPECT(n && *n >= 1 && *n <= ledgerCount))
1176  return;
1177  bitMask |= 1ll << *n;
1178  BEAST_EXPECT(
1179  db->getCompleteShards() == bitmask2Rangeset(bitMask));
1180  }
1181 
1182  auto mainPathCount = std::distance(
1183  boost::filesystem::directory_iterator(shardDir.path()),
1184  boost::filesystem::directory_iterator());
1185 
1186  // Only the two most recent shards
1187  // should be stored at the main path
1188  BEAST_EXPECT(mainPathCount == 2);
1189 
1190  // Confirm recent shard locations
1191  std::set<boost::filesystem::path> mainPathShards{
1192  shardDir.path() / boost::filesystem::path("3"),
1193  shardDir.path() / boost::filesystem::path("4")};
1194  std::set<boost::filesystem::path> actual(
1195  boost::filesystem::directory_iterator(shardDir.path()),
1196  boost::filesystem::directory_iterator());
1197 
1198  BEAST_EXPECT(mainPathShards == actual);
1199 
1200  const auto generateHistoricalStems = [&historicalPaths, &actual] {
1201  for (auto const& path : historicalPaths)
1202  {
1203  for (auto const& shard :
1204  boost::filesystem::directory_iterator(path))
1205  {
1206  actual.insert(boost::filesystem::path(shard).stem());
1207  }
1208  }
1209  };
1210 
1211  // Confirm historical shard locations
1212  std::set<boost::filesystem::path> historicalPathShards;
1213  std::generate_n(
1214  std::inserter(
1215  historicalPathShards, historicalPathShards.begin()),
1216  2,
1217  [n = 1]() mutable { return std::to_string(n++); });
1218  actual.clear();
1219  generateHistoricalStems();
1220 
1221  BEAST_EXPECT(historicalPathShards == actual);
1222 
1223  auto historicalPathCount = std::accumulate(
1224  historicalPaths.begin(),
1225  historicalPaths.end(),
1226  0,
1227  [](int const sum, boost::filesystem::path const& path) {
1228  return sum +
1229  std::distance(
1230  boost::filesystem::directory_iterator(path),
1231  boost::filesystem::directory_iterator());
1232  });
1233 
1234  // All historical shards should be stored
1235  // at historical paths
1236  BEAST_EXPECT(historicalPathCount == ledgerCount - 2);
1237 
1238  data = TestData(seedValue * 2, 4, ledgerCount);
1239  if (!BEAST_EXPECT(data.makeLedgers(env, ledgerCount)))
1240  return;
1241 
1242  // Add ten more shards to the Shard Database
1243  // to exercise recent shard rotation
1244  for (std::uint32_t i = 0; i < ledgerCount; ++i)
1245  {
1246  auto n = createShard(data, *db, ledgerCount * 2, ledgerCount);
1247  if (!BEAST_EXPECT(
1248  n && *n >= 1 + ledgerCount && *n <= ledgerCount * 2))
1249  return;
1250  bitMask |= 1ll << *n;
1251  BEAST_EXPECT(
1252  db->getCompleteShards() == bitmask2Rangeset(bitMask));
1253  }
1254 
1255  mainPathCount = std::distance(
1256  boost::filesystem::directory_iterator(shardDir.path()),
1257  boost::filesystem::directory_iterator());
1258 
1259  // Only the two most recent shards
1260  // should be stored at the main path
1261  BEAST_EXPECT(mainPathCount == 2);
1262 
1263  // Confirm recent shard locations
1264  mainPathShards = {
1265  shardDir.path() / boost::filesystem::path("7"),
1266  shardDir.path() / boost::filesystem::path("8")};
1267  actual = {
1268  boost::filesystem::directory_iterator(shardDir.path()),
1269  boost::filesystem::directory_iterator()};
1270 
1271  BEAST_EXPECT(mainPathShards == actual);
1272 
1273  // Confirm historical shard locations
1274  historicalPathShards.clear();
1275  std::generate_n(
1276  std::inserter(
1277  historicalPathShards, historicalPathShards.begin()),
1278  6,
1279  [n = 1]() mutable { return std::to_string(n++); });
1280  actual.clear();
1281  generateHistoricalStems();
1282 
1283  BEAST_EXPECT(historicalPathShards == actual);
1284 
1285  historicalPathCount = std::accumulate(
1286  historicalPaths.begin(),
1287  historicalPaths.end(),
1288  0,
1289  [](int const sum, boost::filesystem::path const& path) {
1290  return sum +
1291  std::distance(
1292  boost::filesystem::directory_iterator(path),
1293  boost::filesystem::directory_iterator());
1294  });
1295 
1296  // All historical shards should be stored
1297  // at historical paths
1298  BEAST_EXPECT(historicalPathCount == (ledgerCount * 2) - 2);
1299  }
1300  }
1301 
1302  void
1303  testAll(std::string const& backendType)
1304  {
1305  std::uint64_t const seedValue = 51;
1306  testStandalone(backendType);
1307  testCreateShard(backendType, seedValue);
1308  testReopenDatabase(backendType, seedValue + 5);
1309  testGetCompleteShards(backendType, seedValue + 10);
1310  testPrepareShard(backendType, seedValue + 20);
1311  testImportShard(backendType, seedValue + 30);
1312  testCorruptedDatabase(backendType, seedValue + 40);
1313  testIllegalFinalKey(backendType, seedValue + 50);
1314  testImport(backendType, seedValue + 60);
1315  testImportWithHistoricalPaths(backendType, seedValue + 80);
1316  testPrepareWithHistoricalPaths(backendType, seedValue + 90);
1317  }
1318 
1319 public:
1320  DatabaseShard_test() : journal_("DatabaseShard_test", *this)
1321  {
1322  }
1323 
1324  void
1325  run() override
1326  {
1327  testAll("nudb");
1328 
1329 #if RIPPLE_ROCKSDB_AVAILABLE
1330 // testAll ("rocksdb");
1331 #endif
1332 
1333 #if RIPPLE_ENABLE_SQLITE_BACKEND_TESTS
1334  testAll("sqlite");
1335 #endif
1336  }
1337 };
1338 
1339 BEAST_DEFINE_TESTSUITE(DatabaseShard, NodeStore, ripple);
1340 
1341 } // namespace NodeStore
1342 } // namespace ripple
ripple::NodeStore::make_ShardStore
std::unique_ptr< DatabaseShard > make_ShardStore(Application &app, Stoppable &parent, Scheduler &scheduler, int readThreads, beast::Journal j)
Definition: DatabaseShardImp.cpp:1948
ripple::NodeStore::DummyScheduler
Simple NodeStore Scheduler that just peforms the tasks synchronously.
Definition: DummyScheduler.h:29
ripple::hotUNKNOWN
@ hotUNKNOWN
Definition: NodeObject.h:33
ripple::HashPrefix::ledgerMaster
@ ledgerMaster
ledger master data for signing
ripple::SHAMap::isValid
bool isValid() const
Definition: SHAMap.h:522
std::generate_n
T generate_n(T... args)
ripple::NodeStore::DatabaseShard_test::testLedgerData
void testLedgerData(TestData &data, std::shared_ptr< Ledger > ledger, std::uint32_t seq)
Definition: DatabaseShard_test.cpp:184
ripple::NodeStore::DatabaseShard_test::TestData::nAccounts_
std::vector< int > nAccounts_
Definition: DatabaseShard_test.cpp:62
ripple::NodeStore::Database
Persistency layer for NodeObject.
Definition: Database.h:53
ripple::NodeStore::DatabaseShard_test::testGetCompleteShards
void testGetCompleteShards(std::string const &backendType, std::uint64_t const seedValue)
Definition: DatabaseShard_test.cpp:629
ripple::NodeStore::DatabaseShard_test::ledgersPerShard
static constexpr std::uint32_t ledgersPerShard
Definition: DatabaseShard_test.cpp:42
ripple::NodeStore::TestBase
Definition: TestBase.h:68
std::string
STL class.
std::shared_ptr
STL class.
ripple::SHAMap::getHash
SHAMapHash getHash() const
Definition: SHAMap.cpp:791
ripple::base_uint::isNonZero
bool isNonZero() const
Definition: base_uint.h:480
ripple::NodeStore::DatabaseShard::seqToShardIndex
virtual std::uint32_t seqToShardIndex(std::uint32_t seq) const =0
Calculates the shard index for a given ledger sequence.
ripple::Serializer::modData
Blob & modData()
Definition: Serializer.h:176
ripple::NodeStore::DatabaseShard_test::TestData::ledgers_
std::vector< std::shared_ptr< const Ledger > > ledgers_
Definition: DatabaseShard_test.cpp:71
std::vector::reserve
T reserve(T... args)
ripple::LedgerInfo::hash
uint256 hash
Definition: ReadView.h:100
ripple::NodeStore::DatabaseShard_test::journal_
test::SuiteJournal journal_
Definition: DatabaseShard_test.cpp:49
ripple::NodeStore::DatabaseShard_test::TestData::accounts_
std::vector< test::jtx::Account > accounts_
Definition: DatabaseShard_test.cpp:59
ripple::hotACCOUNT_NODE
@ hotACCOUNT_NODE
Definition: NodeObject.h:35
std::vector
STL class.
ripple::ConfigSection::shardDatabase
static std::string shardDatabase()
Definition: ConfigSections.h:38
std::vector::size
T size(T... args)
ripple::sfSequence
const SF_U32 sfSequence(access, STI_UINT32, 4, "Sequence")
Definition: SField.h:356
ripple::snfPREFIX
@ snfPREFIX
Definition: SHAMapTreeNode.h:36
ripple::NodeObject::createObject
static std::shared_ptr< NodeObject > createObject(NodeObjectType type, Blob &&data, uint256 const &hash)
Create an object from fields.
Definition: NodeObject.cpp:37
ripple::sfAccount
const SF_Account sfAccount(access, STI_ACCOUNT, 1, "Account")
Definition: SField.h:481
std::chrono::seconds
ripple::NodeStore::DatabaseShard_test::TestData::TestData
TestData(std::uint64_t const seedValue, int dataSize=dataSizeMax, int nShards=1)
Definition: DatabaseShard_test.cpp:73
ripple::NodeStore::DatabaseShard_test::testConfig
std::unique_ptr< Config > testConfig(std::string const &testName, std::string const &backendType, std::string const &shardDir, std::string const &nodeDir=std::string())
Definition: DatabaseShard_test.cpp:434
ripple::NodeStore::DatabaseShard_test::TestData::payAccounts_
std::vector< std::vector< std::pair< int, int > > > payAccounts_
Definition: DatabaseShard_test.cpp:66
ripple::NodeStore::DatabaseShard::lastLedgerSeq
virtual std::uint32_t lastLedgerSeq(std::uint32_t shardIndex) const =0
Calculates the last ledger sequence for a given shard index.
std::distance
T distance(T... args)
ripple::NodeStore::DatabaseShard_test::testAll
void testAll(std::string const &backendType)
Definition: DatabaseShard_test.cpp:1303
ripple::NodeStore::DatabaseShard_test::checkLedger
void checkLedger(TestData &data, DatabaseShard &db, Ledger const &ledger)
Definition: DatabaseShard_test.cpp:338
ripple::addRaw
void addRaw(LedgerInfo const &info, Serializer &s)
Definition: View.cpp:43
ripple::NodeStore::DatabaseShard_test::maxHistoricalShards
static constexpr std::uint32_t maxHistoricalShards
Definition: DatabaseShard_test.cpp:41
ripple::from_string
bool from_string(RangeSet< T > &rs, std::string const &s)
Convert the given styled string to a RangeSet.
Definition: RangeSet.h:126
ripple::NodeStore::DatabaseShard_test::earliestSeq
static constexpr std::uint32_t earliestSeq
Definition: DatabaseShard_test.cpp:43
ripple::test::jtx::Env::app
Application & app()
Definition: Env.h:240
ripple::LedgerInfo::seq
LedgerIndex seq
Definition: ReadView.h:92
ripple::to_string
std::string to_string(ListDisposition disposition)
Definition: ValidatorList.cpp:42
ripple::SHAMapHash::isNonZero
bool isNonZero() const
Definition: SHAMapTreeNode.h:69
ripple::NodeStore::DatabaseShard_test::TestData
Definition: DatabaseShard_test.cpp:52
ripple::hotTRANSACTION_NODE
@ hotTRANSACTION_NODE
Definition: NodeObject.h:36
ripple::sfAmount
const SF_Amount sfAmount(access, STI_AMOUNT, 1, "Amount")
Definition: SField.h:441
ripple::NodeStore::DatabaseShard_test::nTestShards
static constexpr std::uint32_t nTestShards
Definition: DatabaseShard_test.cpp:46
ripple::NodeStore::DatabaseShard_test::run
void run() override
Definition: DatabaseShard_test.cpp:1325
ripple::SHAMap::snapShot
std::shared_ptr< SHAMap > snapShot(bool isMutable) const
Definition: SHAMap.cpp:51
ripple::NodeStore::DatabaseShard_test::iniAmount
static constexpr std::uint32_t iniAmount
Definition: DatabaseShard_test.cpp:45
ripple::NodeStore::DatabaseShard::getCompleteShards
virtual std::string getCompleteShards()=0
Query which complete shards are stored.
ripple::NodeStore::DatabaseShard_test::TestData::nShards_
int nShards_
Definition: DatabaseShard_test.cpp:57
ripple::LedgerFill::expand
@ expand
Definition: LedgerToJson.h:46
ripple::LedgerInfo::txHash
uint256 txHash
Definition: ReadView.h:101
ripple::ttPAYMENT
@ ttPAYMENT
Definition: TxFormats.h:36
std::set::clear
T clear(T... args)
ripple::NodeStore::Shard::version
static constexpr std::uint32_t version
Definition: Shard.h:163
ripple::NodeStore::DatabaseShard_test::createShard
std::optional< int > createShard(TestData &data, DatabaseShard &db, int maxShardNumber=1, int ledgerOffset=0)
Definition: DatabaseShard_test.cpp:503
std::vector::push_back
T push_back(T... args)
ripple::base_uint< 256 >
ripple::NodeStore::DatabaseShard_test::TestData::makeLedgers
bool makeLedgers(test::jtx::Env &env_, std::uint32_t startIndex=0)
Definition: DatabaseShard_test.cpp:150
ripple::NodeStore::DatabaseShard_test::DatabaseShard_test
DatabaseShard_test()
Definition: DatabaseShard_test.cpp:1320
ripple::Ledger::info
LedgerInfo const & info() const override
Returns information about the ledger.
Definition: Ledger.h:155
ripple::NodeStore::DatabaseShard_test::shardStoreTimeout
static constexpr std::chrono::seconds shardStoreTimeout
Definition: DatabaseShard_test.cpp:47
ripple::NodeStore::BEAST_DEFINE_TESTSUITE
BEAST_DEFINE_TESTSUITE(Backend, ripple_core, ripple)
ripple::NodeStore::Database::store
virtual void store(NodeObjectType type, Blob &&data, uint256 const &hash, std::uint32_t seq)=0
Store the object.
ripple::rand_int
std::enable_if_t< std::is_integral< Integral >::value &&detail::is_engine< Engine >::value, Integral > rand_int(Engine &engine, Integral min, Integral max)
Return a uniformly distributed random integer.
Definition: ripple/basics/random.h:115
ripple::RootStoppable
Definition: Stoppable.h:352
ripple::Ledger
Holds a ledger.
Definition: Ledger.h:77
ripple::Application::getLedgerMaster
virtual LedgerMaster & getLedgerMaster()=0
ripple::NodeStore::Database::fetch
virtual std::shared_ptr< NodeObject > fetch(uint256 const &hash, std::uint32_t seq)=0
Fetch an object.
chrono
ripple::NodeStore::DatabaseShard_test::testCorruptedDatabase
void testCorruptedDatabase(std::string const &backendType, std::uint64_t const seedValue)
Definition: DatabaseShard_test.cpp:786
ripple::NodeStore::DatabaseShard_test::defNodeDir
beast::temp_dir defNodeDir
Definition: DatabaseShard_test.cpp:50
ripple::Ledger::stateMap
SHAMap const & stateMap() const
Definition: Ledger.h:289
ripple::NodeStore::DatabaseShard_test::waitShard
std::optional< int > waitShard(DatabaseShard &db, int shardNumber, std::chrono::seconds timeout=shardStoreTimeout)
Definition: DatabaseShard_test.cpp:483
ripple::LedgerFill::full
@ full
Definition: LedgerToJson.h:47
ripple::NodeStore::DatabaseShard
A collection of historical shards.
Definition: DatabaseShard.h:37
ripple::Serializer::addRaw
int addRaw(Blob const &vector)
Definition: Serializer.cpp:100
std::to_string
T to_string(T... args)
ripple::set
bool set(T &target, std::string const &name, Section const &section)
Set a value from a configuration Section If the named value is not found or doesn't parse as a T,...
Definition: BasicConfig.h:276
std::array
STL class.
ripple::NodeStore::DatabaseShard_test::testStandalone
void testStandalone(std::string const &backendType)
Definition: DatabaseShard_test.cpp:539
ripple::NodeStore::DatabaseShard_test::testIllegalFinalKey
void testIllegalFinalKey(std::string const &backendType, std::uint64_t const seedValue)
Definition: DatabaseShard_test.cpp:843
ripple::NodeStore::DatabaseShard_test::TestData::rng_
beast::xor_shift_engine rng_
Definition: DatabaseShard_test.cpp:55
ripple::NodeStore::DatabaseShard_test
Definition: DatabaseShard_test.cpp:38
std::accumulate
T accumulate(T... args)
std::uint32_t
ripple::NodeStore::DatabaseShard::firstLedgerSeq
virtual std::uint32_t firstLedgerSeq(std::uint32_t shardIndex) const =0
Calculates the first ledger sequence for a given shard index.
std::transform
T transform(T... args)
ripple::test::SuiteJournal
Definition: SuiteJournal.h:88
beast::temp_dir::path
std::string path() const
Get the native path for the temporary directory.
Definition: temp_dir.h:66
ripple::Serializer
Definition: Serializer.h:39
ripple::LedgerFill::binary
@ binary
Definition: LedgerToJson.h:48
ripple::getJson
Json::Value getJson(LedgerFill const &fill)
Return a new Json::Value representing the ledger with given options.
Definition: LedgerToJson.cpp:280
ripple::Ledger::txMap
SHAMap const & txMap() const
Definition: Ledger.h:301
ripple::NodeStore::DatabaseShard::fetchLedger
virtual std::shared_ptr< Ledger > fetchLedger(uint256 const &hash, std::uint32_t seq)=0
Fetch a ledger from the shard store.
ripple::LedgerMaster::getClosedLedger
std::shared_ptr< Ledger const > getClosedLedger()
Definition: LedgerMaster.h:87
ripple::NodeStore::DatabaseShard_test::testReopenDatabase
void testReopenDatabase(std::string const &backendType, std::uint64_t const seedValue)
Definition: DatabaseShard_test.cpp:589
ripple::NodeStore::DatabaseShard::importShard
virtual bool importShard(std::uint32_t shardIndex, boost::filesystem::path const &srcDir)=0
Import a shard into the shard database.
ripple
Use hash_* containers for keys that do not need a cryptographically secure hashing algorithm.
Definition: RCLCensorshipDetector.h:29
ripple::NodeStore::DatabaseShard_test::testImportShard
void testImportShard(std::string const &backendType, std::uint64_t const seedValue)
Definition: DatabaseShard_test.cpp:730
ripple::ttACCOUNT_SET
@ ttACCOUNT_SET
Definition: TxFormats.h:39
ripple::test::jtx::Env::close
bool close(NetClock::time_point closeTime, boost::optional< std::chrono::milliseconds > consensusDelay=boost::none)
Close and advance the ledger.
Definition: Env.cpp:121
std::array::begin
T begin(T... args)
ripple::NodeStore::DatabaseShard_test::saveLedger
bool saveLedger(Database &db, Ledger const &ledger, std::shared_ptr< Ledger const > const &next={})
Definition: DatabaseShard_test.cpp:268
beast::rngfill
void rngfill(void *buffer, std::size_t bytes, Generator &g)
Definition: rngfill.h:32
ripple::NodeStore::DatabaseShard::prepareShard
virtual bool prepareShard(std::uint32_t shardIndex)=0
Prepare a shard index to be imported into the database.
std::set::insert
T insert(T... args)
ripple::test::jtx::Env::fund
void fund(bool setDefaultRipple, STAmount const &amount, Account const &account)
Definition: Env.cpp:219
ripple::NodeStore::isSame
bool isSame(std::shared_ptr< NodeObject > const &lhs, std::shared_ptr< NodeObject > const &rhs)
Returns true if objects are identical.
Definition: TestBase.h:57
ripple::NodeStore::DatabaseShard_test::TestData::isNewAccounts
bool isNewAccounts(int seq)
Definition: DatabaseShard_test.cpp:127
ripple::NodeStore::DatabaseShard::getPreShards
virtual std::string getPreShards()=0
Get shard indexes being imported.
ripple::NodeStore::DatabaseShard::setStored
virtual void setStored(std::shared_ptr< Ledger const > const &ledger)=0
Notifies the database that the given ledger has been fully acquired and stored.
ripple::NodeStore::DatabaseShard_test::testCreateShard
void testCreateShard(std::string const &backendType, std::uint64_t const seedValue)
Definition: DatabaseShard_test.cpp:566
ripple::NodeStore::DatabaseShard_test::dataSizeMax
static constexpr std::uint32_t dataSizeMax
Definition: DatabaseShard_test.cpp:44
ripple::NodeStore::DatabaseShard_test::testPrepareWithHistoricalPaths
void testPrepareWithHistoricalPaths(std::string const &backendType, std::uint64_t const seedValue)
Definition: DatabaseShard_test.cpp:1128
ripple::NodeStore::DatabaseShard_test::testImport
void testImport(std::string const &backendType, std::uint64_t const seedValue)
Definition: DatabaseShard_test.cpp:944
ripple::SHAMapAbstractNode
Definition: SHAMapTreeNode.h:122
ripple::NodeStore::DatabaseShard_test::testImportWithHistoricalPaths
void testImportWithHistoricalPaths(std::string const &backendType, std::uint64_t const seedValue)
Definition: DatabaseShard_test.cpp:992
ripple::NodeStore::DatabaseShard::prepareLedger
virtual boost::optional< std::uint32_t > prepareLedger(std::uint32_t validLedgerSeq)=0
Prepare to store a new ledger in the shard being acquired.
std::optional< int >
ripple::hotLEDGER
@ hotLEDGER
Definition: NodeObject.h:34
std::make_pair
T make_pair(T... args)
beast::detail::xor_shift_engine
Definition: xor_shift_engine.h:32
ripple::Serializer::add32
int add32(std::uint32_t i)
Definition: Serializer.cpp:38
std::array::end
T end(T... args)
ripple::RangeSet
boost::icl::interval_set< T, std::less, ClosedInterval< T > > RangeSet
A set of closed intervals over the domain T.
Definition: RangeSet.h:69
ripple::NodeStore::DatabaseShard_test::TestData::makeLedgerData
void makeLedgerData(test::jtx::Env &env_, std::uint32_t seq)
Definition: DatabaseShard_test.cpp:133
numeric
ripple::NodeStore::DatabaseShard::removePreShard
virtual void removePreShard(std::uint32_t shardIndex)=0
Remove a previously prepared shard index for import.
ripple::NodeStore::DatabaseShard_test::testPrepareShard
void testPrepareShard(std::string const &backendType, std::uint64_t const seedValue)
Definition: DatabaseShard_test.cpp:661
std::inserter
T inserter(T... args)
ripple::ltACCOUNT_ROOT
@ ltACCOUNT_ROOT
Definition: LedgerFormats.h:53
std::unique_ptr
STL class.
ripple::sum
static auto sum(TCollection const &col)
Definition: BookStep.cpp:693
ripple::NodeStore::Database::import
virtual void import(Database &source)=0
Import objects from another database.
ripple::LedgerFill
Definition: LedgerToJson.h:32
ripple::NodeStore::DatabaseShard_test::bitmask2Rangeset
std::string bitmask2Rangeset(std::uint64_t bitmask)
Definition: DatabaseShard_test.cpp:410
std::set
STL class.
ripple::test::jtx::Env
A transaction testing environment.
Definition: Env.h:115
ripple::ConfigSection::nodeDatabase
static std::string nodeDatabase()
Definition: ConfigSections.h:33
std::this_thread::yield
T yield(T... args)
beast::temp_dir
RAII temporary directory.
Definition: temp_dir.h:33
ripple::NodeStore::DatabaseShard_test::TestData::xrpAmount_
std::vector< int > xrpAmount_
Definition: DatabaseShard_test.cpp:68
ripple::NodeStore::DatabaseShard_test::maxSizeGb
static constexpr std::uint32_t maxSizeGb
Definition: DatabaseShard_test.cpp:40
std::chrono::system_clock::now
T now(T... args)
Generated by   doxygen 1.8.17