rippled
Database_test.cpp
1 //------------------------------------------------------------------------------
2 /*
3  This file is part of rippled: https://github.com/ripple/rippled
4  Copyright (c) 2012, 2013 Ripple Labs Inc.
5 
6  Permission to use, copy, modify, and/or distribute this software for any
7  purpose with or without fee is hereby granted, provided that the above
8  copyright notice and this permission notice appear in all copies.
9 
10  THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11  WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12  MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13  ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14  WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15  ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16  OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18 //==============================================================================
19 
20 #include <ripple/beast/utility/temp_dir.h>
21 #include <ripple/core/DatabaseCon.h>
22 #include <ripple/nodestore/DummyScheduler.h>
23 #include <ripple/nodestore/Manager.h>
24 #include <test/jtx.h>
25 #include <test/jtx/CheckMessageLogs.h>
26 #include <test/jtx/envconfig.h>
27 #include <test/nodestore/TestBase.h>
28 #include <test/unit_test/SuiteJournal.h>
29 
30 namespace ripple {
31 
32 namespace NodeStore {
33 
34 class Database_test : public TestBase
35 {
37 
38 public:
39  Database_test() : journal_("Database_test", *this)
40  {
41  }
42 
43  void
45  {
46  testcase("Config");
47 
48  using namespace ripple::test;
49  using namespace ripple::test::jtx;
50 
51  auto const integrityWarning =
52  "reducing the data integrity guarantees from the "
53  "default [sqlite] behavior is not recommended for "
54  "nodes storing large amounts of history, because of the "
55  "difficulty inherent in rebuilding corrupted data.";
56  {
57  // defaults
58  Env env(*this);
59 
60  auto const s = setup_DatabaseCon(env.app().config());
61 
62  if (BEAST_EXPECT(s.globalPragma->size() == 3))
63  {
64  BEAST_EXPECT(
65  s.globalPragma->at(0) == "PRAGMA journal_mode=wal;");
66  BEAST_EXPECT(
67  s.globalPragma->at(1) == "PRAGMA synchronous=normal;");
68  BEAST_EXPECT(
69  s.globalPragma->at(2) == "PRAGMA temp_store=file;");
70  }
71  }
72  {
73  // High safety level
75 
76  bool found = false;
77  Env env = [&]() {
78  auto p = test::jtx::envconfig();
79  {
80  auto& section = p->section("sqlite");
81  section.set("safety_level", "high");
82  }
83  p->LEDGER_HISTORY = 100'000'000;
84 
85  return Env(
86  *this,
87  std::move(p),
88  std::make_unique<CheckMessageLogs>(
89  integrityWarning, &found),
91  }();
92 
93  BEAST_EXPECT(!found);
94  auto const s = setup_DatabaseCon(env.app().config());
95  if (BEAST_EXPECT(s.globalPragma->size() == 3))
96  {
97  BEAST_EXPECT(
98  s.globalPragma->at(0) == "PRAGMA journal_mode=wal;");
99  BEAST_EXPECT(
100  s.globalPragma->at(1) == "PRAGMA synchronous=normal;");
101  BEAST_EXPECT(
102  s.globalPragma->at(2) == "PRAGMA temp_store=file;");
103  }
104  }
105  {
106  // Low safety level
108 
109  bool found = false;
110  Env env = [&]() {
111  auto p = test::jtx::envconfig();
112  {
113  auto& section = p->section("sqlite");
114  section.set("safety_level", "low");
115  }
116  p->LEDGER_HISTORY = 100'000'000;
117 
118  return Env(
119  *this,
120  std::move(p),
121  std::make_unique<CheckMessageLogs>(
122  integrityWarning, &found),
124  }();
125 
126  BEAST_EXPECT(found);
127  auto const s = setup_DatabaseCon(env.app().config());
128  if (BEAST_EXPECT(s.globalPragma->size() == 3))
129  {
130  BEAST_EXPECT(
131  s.globalPragma->at(0) == "PRAGMA journal_mode=memory;");
132  BEAST_EXPECT(
133  s.globalPragma->at(1) == "PRAGMA synchronous=off;");
134  BEAST_EXPECT(
135  s.globalPragma->at(2) == "PRAGMA temp_store=memory;");
136  }
137  }
138  {
139  // Override individual settings
141 
142  bool found = false;
143  Env env = [&]() {
144  auto p = test::jtx::envconfig();
145  {
146  auto& section = p->section("sqlite");
147  section.set("journal_mode", "off");
148  section.set("synchronous", "extra");
149  section.set("temp_store", "default");
150  }
151 
152  return Env(
153  *this,
154  std::move(p),
155  std::make_unique<CheckMessageLogs>(
156  integrityWarning, &found),
158  }();
159 
160  // No warning, even though higher risk settings were used because
161  // LEDGER_HISTORY is small
162  BEAST_EXPECT(!found);
163  auto const s = setup_DatabaseCon(env.app().config());
164  if (BEAST_EXPECT(s.globalPragma->size() == 3))
165  {
166  BEAST_EXPECT(
167  s.globalPragma->at(0) == "PRAGMA journal_mode=off;");
168  BEAST_EXPECT(
169  s.globalPragma->at(1) == "PRAGMA synchronous=extra;");
170  BEAST_EXPECT(
171  s.globalPragma->at(2) == "PRAGMA temp_store=default;");
172  }
173  }
174  {
175  // Override individual settings with large history
177 
178  bool found = false;
179  Env env = [&]() {
180  auto p = test::jtx::envconfig();
181  {
182  auto& section = p->section("sqlite");
183  section.set("journal_mode", "off");
184  section.set("synchronous", "extra");
185  section.set("temp_store", "default");
186  }
187  p->LEDGER_HISTORY = 50'000'000;
188 
189  return Env(
190  *this,
191  std::move(p),
192  std::make_unique<CheckMessageLogs>(
193  integrityWarning, &found),
195  }();
196 
197  // No warning, even though higher risk settings were used because
198  // LEDGER_HISTORY is small
199  BEAST_EXPECT(found);
200  auto const s = setup_DatabaseCon(env.app().config());
201  if (BEAST_EXPECT(s.globalPragma->size() == 3))
202  {
203  BEAST_EXPECT(
204  s.globalPragma->at(0) == "PRAGMA journal_mode=off;");
205  BEAST_EXPECT(
206  s.globalPragma->at(1) == "PRAGMA synchronous=extra;");
207  BEAST_EXPECT(
208  s.globalPragma->at(2) == "PRAGMA temp_store=default;");
209  }
210  }
211  {
212  // Error: Mix safety_level and individual settings
214  auto const expected =
215  "Failed to initialize SQL databases: "
216  "Configuration file may not define both \"safety_level\" and "
217  "\"journal_mode\"";
218  bool found = false;
219 
220  auto p = test::jtx::envconfig();
221  {
222  auto& section = p->section("sqlite");
223  section.set("safety_level", "low");
224  section.set("journal_mode", "off");
225  section.set("synchronous", "extra");
226  section.set("temp_store", "default");
227  }
228 
229  try
230  {
231  Env env(
232  *this,
233  std::move(p),
234  std::make_unique<CheckMessageLogs>(expected, &found),
236  fail();
237  }
238  catch (...)
239  {
240  BEAST_EXPECT(found);
241  }
242  }
243  {
244  // Error: Mix safety_level and one setting (gotta catch 'em all)
246  auto const expected =
247  "Failed to initialize SQL databases: Configuration file may "
248  "not define both \"safety_level\" and \"journal_mode\"";
249  bool found = false;
250 
251  auto p = test::jtx::envconfig();
252  {
253  auto& section = p->section("sqlite");
254  section.set("safety_level", "high");
255  section.set("journal_mode", "off");
256  }
257 
258  try
259  {
260  Env env(
261  *this,
262  std::move(p),
263  std::make_unique<CheckMessageLogs>(expected, &found),
265  fail();
266  }
267  catch (...)
268  {
269  BEAST_EXPECT(found);
270  }
271  }
272  {
273  // Error: Mix safety_level and one setting (gotta catch 'em all)
275  auto const expected =
276  "Failed to initialize SQL databases: Configuration file may "
277  "not define both \"safety_level\" and \"synchronous\"";
278  bool found = false;
279 
280  auto p = test::jtx::envconfig();
281  {
282  auto& section = p->section("sqlite");
283  section.set("safety_level", "low");
284  section.set("synchronous", "extra");
285  }
286 
287  try
288  {
289  Env env(
290  *this,
291  std::move(p),
292  std::make_unique<CheckMessageLogs>(expected, &found),
294  fail();
295  }
296  catch (...)
297  {
298  BEAST_EXPECT(found);
299  }
300  }
301  {
302  // Error: Mix safety_level and one setting (gotta catch 'em all)
304  auto const expected =
305  "Failed to initialize SQL databases: Configuration file may "
306  "not define both \"safety_level\" and \"temp_store\"";
307  bool found = false;
308 
309  auto p = test::jtx::envconfig();
310  {
311  auto& section = p->section("sqlite");
312  section.set("safety_level", "high");
313  section.set("temp_store", "default");
314  }
315 
316  try
317  {
318  Env env(
319  *this,
320  std::move(p),
321  std::make_unique<CheckMessageLogs>(expected, &found),
323  fail();
324  }
325  catch (...)
326  {
327  BEAST_EXPECT(found);
328  }
329  }
330  {
331  // Error: Invalid value
333  auto const expected =
334  "Failed to initialize SQL databases: Invalid safety_level "
335  "value: slow";
336  bool found = false;
337 
338  auto p = test::jtx::envconfig();
339  {
340  auto& section = p->section("sqlite");
341  section.set("safety_level", "slow");
342  }
343 
344  try
345  {
346  Env env(
347  *this,
348  std::move(p),
349  std::make_unique<CheckMessageLogs>(expected, &found),
351  fail();
352  }
353  catch (...)
354  {
355  BEAST_EXPECT(found);
356  }
357  }
358  {
359  // Error: Invalid value
361  auto const expected =
362  "Failed to initialize SQL databases: Invalid journal_mode "
363  "value: fast";
364  bool found = false;
365 
366  auto p = test::jtx::envconfig();
367  {
368  auto& section = p->section("sqlite");
369  section.set("journal_mode", "fast");
370  }
371 
372  try
373  {
374  Env env(
375  *this,
376  std::move(p),
377  std::make_unique<CheckMessageLogs>(expected, &found),
379  fail();
380  }
381  catch (...)
382  {
383  BEAST_EXPECT(found);
384  }
385  }
386  {
387  // Error: Invalid value
389  auto const expected =
390  "Failed to initialize SQL databases: Invalid synchronous "
391  "value: instant";
392  bool found = false;
393 
394  auto p = test::jtx::envconfig();
395  {
396  auto& section = p->section("sqlite");
397  section.set("synchronous", "instant");
398  }
399 
400  try
401  {
402  Env env(
403  *this,
404  std::move(p),
405  std::make_unique<CheckMessageLogs>(expected, &found),
407  fail();
408  }
409  catch (...)
410  {
411  BEAST_EXPECT(found);
412  }
413  }
414  {
415  // Error: Invalid value
417  auto const expected =
418  "Failed to initialize SQL databases: Invalid temp_store "
419  "value: network";
420  bool found = false;
421 
422  auto p = test::jtx::envconfig();
423  {
424  auto& section = p->section("sqlite");
425  section.set("temp_store", "network");
426  }
427 
428  try
429  {
430  Env env(
431  *this,
432  std::move(p),
433  std::make_unique<CheckMessageLogs>(expected, &found),
435  fail();
436  }
437  catch (...)
438  {
439  BEAST_EXPECT(found);
440  }
441  }
442  }
443 
444  //--------------------------------------------------------------------------
445 
446  void
448  std::string const& destBackendType,
449  std::string const& srcBackendType,
450  std::int64_t seedValue)
451  {
452  DummyScheduler scheduler;
453  RootStoppable parent("TestRootStoppable");
454 
455  beast::temp_dir node_db;
456  Section srcParams;
457  srcParams.set("type", srcBackendType);
458  srcParams.set("path", node_db.path());
459 
460  // Create a batch
461  auto batch = createPredictableBatch(numObjectsToTest, seedValue);
462 
463  // Write to source db
464  {
466  "test",
467  megabytes(4),
468  scheduler,
469  2,
470  parent,
471  srcParams,
472  journal_);
473  storeBatch(*src, batch);
474  }
475 
476  Batch copy;
477 
478  {
479  // Re-open the db
481  "test",
482  megabytes(4),
483  scheduler,
484  2,
485  parent,
486  srcParams,
487  journal_);
488 
489  // Set up the destination database
490  beast::temp_dir dest_db;
491  Section destParams;
492  destParams.set("type", destBackendType);
493  destParams.set("path", dest_db.path());
494 
496  "test",
497  megabytes(4),
498  scheduler,
499  2,
500  parent,
501  destParams,
502  journal_);
503 
504  testcase(
505  "import into '" + destBackendType + "' from '" +
506  srcBackendType + "'");
507 
508  // Do the import
509  dest->import(*src);
510 
511  // Get the results of the import
512  fetchCopyOfBatch(*dest, &copy, batch);
513  }
514 
515  // Canonicalize the source and destination batches
516  std::sort(batch.begin(), batch.end(), LessThan{});
517  std::sort(copy.begin(), copy.end(), LessThan{});
518  BEAST_EXPECT(areBatchesEqual(batch, copy));
519  }
520 
521  //--------------------------------------------------------------------------
522 
523  void
525  std::string const& type,
526  bool const testPersistence,
527  std::int64_t const seedValue,
528  int numObjsToTest = 2000)
529  {
530  DummyScheduler scheduler;
531  RootStoppable parent("TestRootStoppable");
532 
533  std::string s = "NodeStore backend '" + type + "'";
534 
535  testcase(s);
536 
537  beast::temp_dir node_db;
538  Section nodeParams;
539  nodeParams.set("type", type);
540  nodeParams.set("path", node_db.path());
541 
542  beast::xor_shift_engine rng(seedValue);
543 
544  // Create a batch
545  auto batch = createPredictableBatch(numObjsToTest, rng());
546 
547  {
548  // Open the database
550  "test",
551  megabytes(4),
552  scheduler,
553  2,
554  parent,
555  nodeParams,
556  journal_);
557 
558  // Write the batch
559  storeBatch(*db, batch);
560 
561  {
562  // Read it back in
563  Batch copy;
564  fetchCopyOfBatch(*db, &copy, batch);
565  BEAST_EXPECT(areBatchesEqual(batch, copy));
566  }
567 
568  {
569  // Reorder and read the copy again
570  std::shuffle(batch.begin(), batch.end(), rng);
571  Batch copy;
572  fetchCopyOfBatch(*db, &copy, batch);
573  BEAST_EXPECT(areBatchesEqual(batch, copy));
574  }
575  }
576 
577  if (testPersistence)
578  {
579  // Re-open the database without the ephemeral DB
581  "test",
582  megabytes(4),
583  scheduler,
584  2,
585  parent,
586  nodeParams,
587  journal_);
588 
589  // Read it back in
590  Batch copy;
591  fetchCopyOfBatch(*db, &copy, batch);
592 
593  // Canonicalize the source and destination batches
594  std::sort(batch.begin(), batch.end(), LessThan{});
595  std::sort(copy.begin(), copy.end(), LessThan{});
596  BEAST_EXPECT(areBatchesEqual(batch, copy));
597  }
598 
599  if (type == "memory")
600  {
601  // Earliest ledger sequence tests
602  {
603  // Verify default earliest ledger sequence
606  "test",
607  megabytes(4),
608  scheduler,
609  2,
610  parent,
611  nodeParams,
612  journal_);
613  BEAST_EXPECT(
614  db->earliestLedgerSeq() == XRP_LEDGER_EARLIEST_SEQ);
615  }
616 
617  // Set an invalid earliest ledger sequence
618  try
619  {
620  nodeParams.set("earliest_seq", "0");
623  "test",
624  megabytes(4),
625  scheduler,
626  2,
627  parent,
628  nodeParams,
629  journal_);
630  }
631  catch (std::runtime_error const& e)
632  {
633  BEAST_EXPECT(
634  std::strcmp(e.what(), "Invalid earliest_seq") == 0);
635  }
636 
637  {
638  // Set a valid earliest ledger sequence
639  nodeParams.set("earliest_seq", "1");
642  "test",
643  megabytes(4),
644  scheduler,
645  2,
646  parent,
647  nodeParams,
648  journal_);
649 
650  // Verify database uses the earliest ledger sequence setting
651  BEAST_EXPECT(db->earliestLedgerSeq() == 1);
652  }
653 
654  // Create another database that attempts to set the value again
655  try
656  {
657  // Set to default earliest ledger sequence
658  nodeParams.set(
659  "earliest_seq", std::to_string(XRP_LEDGER_EARLIEST_SEQ));
662  "test",
663  megabytes(4),
664  scheduler,
665  2,
666  parent,
667  nodeParams,
668  journal_);
669  }
670  catch (std::runtime_error const& e)
671  {
672  BEAST_EXPECT(
673  std::strcmp(e.what(), "earliest_seq set more than once") ==
674  0);
675  }
676  }
677  }
678 
679  //--------------------------------------------------------------------------
680 
681  void
682  run() override
683  {
684  std::int64_t const seedValue = 50;
685 
686  testConfig();
687 
688  testNodeStore("memory", false, seedValue);
689 
690  // Persistent backend tests
691  {
692  testNodeStore("nudb", true, seedValue);
693 
694 #if RIPPLE_ROCKSDB_AVAILABLE
695  testNodeStore("rocksdb", true, seedValue);
696 #endif
697  }
698 
699  // Import tests
700  {
701  testImport("nudb", "nudb", seedValue);
702 
703 #if RIPPLE_ROCKSDB_AVAILABLE
704  testImport("rocksdb", "rocksdb", seedValue);
705 #endif
706 
707 #if RIPPLE_ENABLE_SQLITE_BACKEND_TESTS
708  testImport("sqlite", "sqlite", seedValue);
709 #endif
710  }
711  }
712 };
713 
714 BEAST_DEFINE_TESTSUITE(Database, NodeStore, ripple);
715 
716 } // namespace NodeStore
717 } // namespace ripple
ripple::NodeStore::DummyScheduler
Simple NodeStore Scheduler that just peforms the tasks synchronously.
Definition: DummyScheduler.h:29
ripple::Section
Holds a collection of configuration values.
Definition: BasicConfig.h:43
std::strcmp
T strcmp(T... args)
ripple::DatabaseCon::Setup::globalPragma
static std::unique_ptr< std::vector< std::string > const > globalPragma
Definition: DatabaseCon.h:104
ripple::NodeStore::TestBase
Definition: TestBase.h:68
std::string
STL class.
ripple::NodeStore::Database_test::Database_test
Database_test()
Definition: Database_test.cpp:39
ripple::NodeStore::Manager::make_Database
virtual std::unique_ptr< Database > make_Database(std::string const &name, std::size_t burstSize, Scheduler &scheduler, int readThreads, Stoppable &parent, Section const &backendParameters, beast::Journal journal)=0
Construct a NodeStore database.
ripple::setup_DatabaseCon
DatabaseCon::Setup setup_DatabaseCon(Config const &c, std::optional< beast::Journal > j=std::nullopt)
Definition: DatabaseCon.cpp:106
std::vector< std::shared_ptr< NodeObject > >
ripple::NodeStore::LessThan
Binary function that satisfies the strict-weak-ordering requirement.
Definition: TestBase.h:44
ripple::NodeStore::Database_test::testNodeStore
void testNodeStore(std::string const &type, bool const testPersistence, std::int64_t const seedValue, int numObjsToTest=2000)
Definition: Database_test.cpp:524
ripple::NodeStore::Database_test::testImport
void testImport(std::string const &destBackendType, std::string const &srcBackendType, std::int64_t seedValue)
Definition: Database_test.cpp:447
ripple::test::jtx::Env::app
Application & app()
Definition: Env.h:240
ripple::test::jtx::envconfig
std::unique_ptr< Config > envconfig()
creates and initializes a default configuration for jtx::Env
Definition: envconfig.h:49
ripple::NodeStore::Database_test::run
void run() override
Definition: Database_test.cpp:682
std::sort
T sort(T... args)
std::unique_ptr::reset
T reset(T... args)
ripple::NodeStore::TestBase::areBatchesEqual
static bool areBatchesEqual(Batch const &lhs, Batch const &rhs)
Definition: TestBase.h:120
ripple::NodeStore::BEAST_DEFINE_TESTSUITE
BEAST_DEFINE_TESTSUITE(Backend, ripple_core, ripple)
ripple::RootStoppable
Definition: Stoppable.h:354
ripple::Application::config
virtual Config & config()=0
ripple::megabytes
constexpr auto megabytes(T value) noexcept
Definition: ByteUtilities.h:34
ripple::NodeStore::TestBase::fetchCopyOfBatch
void fetchCopyOfBatch(Backend &backend, Batch *pCopy, Batch const &batch)
Definition: TestBase.h:155
std::to_string
T to_string(T... args)
std::runtime_error
STL class.
std::int64_t
ripple::test::SuiteJournal
Definition: SuiteJournal.h:88
beast::temp_dir::path
std::string path() const
Get the native path for the temporary directory.
Definition: temp_dir.h:66
ripple::NodeStore::TestBase::numObjectsToTest
static const int numObjectsToTest
Definition: TestBase.h:75
ripple::test::jtx
Definition: Check_test.cpp:26
ripple::NodeStore::TestBase::createPredictableBatch
static Batch createPredictableBatch(int numObjects, std::uint64_t seed)
Definition: TestBase.h:80
ripple
Use hash_* containers for keys that do not need a cryptographically secure hashing algorithm.
Definition: RCLCensorshipDetector.h:29
ripple::test
Definition: LedgerDeltaAcquire.h:35
ripple::NodeStore::Database_test::journal_
test::SuiteJournal journal_
Definition: Database_test.cpp:36
ripple::Section::set
void set(std::string const &key, std::string const &value)
Set a key/value pair.
Definition: BasicConfig.cpp:32
beast::severities::kWarning
@ kWarning
Definition: Journal.h:37
ripple::XRP_LEDGER_EARLIEST_SEQ
static constexpr std::uint32_t XRP_LEDGER_EARLIEST_SEQ
The XRP ledger network's earliest allowed sequence.
Definition: SystemParameters.h:61
beast::detail::xor_shift_engine
Definition: xor_shift_engine.h:32
ripple::NodeStore::TestBase::storeBatch
void storeBatch(Backend &backend, Batch const &batch)
Definition: TestBase.h:145
ripple::NodeStore::Manager::instance
static Manager & instance()
Returns the instance of the manager singleton.
Definition: ManagerImp.cpp:128
ripple::NodeStore::Database_test::testConfig
void testConfig()
Definition: Database_test.cpp:44
std::unique_ptr
STL class.
std::shuffle
T shuffle(T... args)
ripple::test::jtx::Env
A transaction testing environment.
Definition: Env.h:115
beast::temp_dir
RAII temporary directory.
Definition: temp_dir.h:33
std::runtime_error::what
T what(T... args)
ripple::NodeStore::Database_test
Definition: Database_test.cpp:34