rippled
Database_test.cpp
1 //------------------------------------------------------------------------------
2 /*
3  This file is part of rippled: https://github.com/ripple/rippled
4  Copyright (c) 2012, 2013 Ripple Labs Inc.
5 
6  Permission to use, copy, modify, and/or distribute this software for any
7  purpose with or without fee is hereby granted, provided that the above
8  copyright notice and this permission notice appear in all copies.
9 
10  THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11  WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12  MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13  ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14  WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15  ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16  OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18 //==============================================================================
19 
20 #include <ripple/beast/utility/temp_dir.h>
21 #include <ripple/core/DatabaseCon.h>
22 #include <ripple/nodestore/DummyScheduler.h>
23 #include <ripple/nodestore/Manager.h>
24 #include <test/jtx.h>
25 #include <test/jtx/CheckMessageLogs.h>
26 #include <test/jtx/envconfig.h>
27 #include <test/nodestore/TestBase.h>
28 #include <test/unit_test/SuiteJournal.h>
29 
30 namespace ripple {
31 namespace NodeStore {
32 
33 class Database_test : public TestBase
34 {
36 
37 public:
38  Database_test() : journal_("Database_test", *this)
39  {
40  }
41 
42  void
44  {
45  testcase("Config");
46 
47  using namespace ripple::test;
48  using namespace ripple::test::jtx;
49 
50  auto const integrityWarning =
51  "reducing the data integrity guarantees from the "
52  "default [sqlite] behavior is not recommended for "
53  "nodes storing large amounts of history, because of the "
54  "difficulty inherent in rebuilding corrupted data.";
55  {
56  // defaults
57  Env env(*this);
58 
59  auto const s = setup_DatabaseCon(env.app().config());
60 
61  if (BEAST_EXPECT(s.globalPragma->size() == 3))
62  {
63  BEAST_EXPECT(
64  s.globalPragma->at(0) == "PRAGMA journal_mode=wal;");
65  BEAST_EXPECT(
66  s.globalPragma->at(1) == "PRAGMA synchronous=normal;");
67  BEAST_EXPECT(
68  s.globalPragma->at(2) == "PRAGMA temp_store=file;");
69  }
70  }
71  {
72  // High safety level
74 
75  bool found = false;
76  Env env = [&]() {
77  auto p = test::jtx::envconfig();
78  {
79  auto& section = p->section("sqlite");
80  section.set("safety_level", "high");
81  }
82  p->LEDGER_HISTORY = 100'000'000;
83 
84  return Env(
85  *this,
86  std::move(p),
87  std::make_unique<CheckMessageLogs>(
88  integrityWarning, &found),
90  }();
91 
92  BEAST_EXPECT(!found);
93  auto const s = setup_DatabaseCon(env.app().config());
94  if (BEAST_EXPECT(s.globalPragma->size() == 3))
95  {
96  BEAST_EXPECT(
97  s.globalPragma->at(0) == "PRAGMA journal_mode=wal;");
98  BEAST_EXPECT(
99  s.globalPragma->at(1) == "PRAGMA synchronous=normal;");
100  BEAST_EXPECT(
101  s.globalPragma->at(2) == "PRAGMA temp_store=file;");
102  }
103  }
104  {
105  // Low safety level
107 
108  bool found = false;
109  Env env = [&]() {
110  auto p = test::jtx::envconfig();
111  {
112  auto& section = p->section("sqlite");
113  section.set("safety_level", "low");
114  }
115  p->LEDGER_HISTORY = 100'000'000;
116 
117  return Env(
118  *this,
119  std::move(p),
120  std::make_unique<CheckMessageLogs>(
121  integrityWarning, &found),
123  }();
124 
125  BEAST_EXPECT(found);
126  auto const s = setup_DatabaseCon(env.app().config());
127  if (BEAST_EXPECT(s.globalPragma->size() == 3))
128  {
129  BEAST_EXPECT(
130  s.globalPragma->at(0) == "PRAGMA journal_mode=memory;");
131  BEAST_EXPECT(
132  s.globalPragma->at(1) == "PRAGMA synchronous=off;");
133  BEAST_EXPECT(
134  s.globalPragma->at(2) == "PRAGMA temp_store=memory;");
135  }
136  }
137  {
138  // Override individual settings
140 
141  bool found = false;
142  Env env = [&]() {
143  auto p = test::jtx::envconfig();
144  {
145  auto& section = p->section("sqlite");
146  section.set("journal_mode", "off");
147  section.set("synchronous", "extra");
148  section.set("temp_store", "default");
149  }
150 
151  return Env(
152  *this,
153  std::move(p),
154  std::make_unique<CheckMessageLogs>(
155  integrityWarning, &found),
157  }();
158 
159  // No warning, even though higher risk settings were used because
160  // LEDGER_HISTORY is small
161  BEAST_EXPECT(!found);
162  auto const s = setup_DatabaseCon(env.app().config());
163  if (BEAST_EXPECT(s.globalPragma->size() == 3))
164  {
165  BEAST_EXPECT(
166  s.globalPragma->at(0) == "PRAGMA journal_mode=off;");
167  BEAST_EXPECT(
168  s.globalPragma->at(1) == "PRAGMA synchronous=extra;");
169  BEAST_EXPECT(
170  s.globalPragma->at(2) == "PRAGMA temp_store=default;");
171  }
172  }
173  {
174  // Override individual settings with large history
176 
177  bool found = false;
178  Env env = [&]() {
179  auto p = test::jtx::envconfig();
180  {
181  auto& section = p->section("sqlite");
182  section.set("journal_mode", "off");
183  section.set("synchronous", "extra");
184  section.set("temp_store", "default");
185  }
186  p->LEDGER_HISTORY = 50'000'000;
187 
188  return Env(
189  *this,
190  std::move(p),
191  std::make_unique<CheckMessageLogs>(
192  integrityWarning, &found),
194  }();
195 
196  // No warning, even though higher risk settings were used because
197  // LEDGER_HISTORY is small
198  BEAST_EXPECT(found);
199  auto const s = setup_DatabaseCon(env.app().config());
200  if (BEAST_EXPECT(s.globalPragma->size() == 3))
201  {
202  BEAST_EXPECT(
203  s.globalPragma->at(0) == "PRAGMA journal_mode=off;");
204  BEAST_EXPECT(
205  s.globalPragma->at(1) == "PRAGMA synchronous=extra;");
206  BEAST_EXPECT(
207  s.globalPragma->at(2) == "PRAGMA temp_store=default;");
208  }
209  }
210  {
211  // Error: Mix safety_level and individual settings
213  auto const expected =
214  "Failed to initialize SQLite databases: "
215  "Configuration file may not define both \"safety_level\" and "
216  "\"journal_mode\"";
217  bool found = false;
218 
219  auto p = test::jtx::envconfig();
220  {
221  auto& section = p->section("sqlite");
222  section.set("safety_level", "low");
223  section.set("journal_mode", "off");
224  section.set("synchronous", "extra");
225  section.set("temp_store", "default");
226  }
227 
228  try
229  {
230  Env env(
231  *this,
232  std::move(p),
233  std::make_unique<CheckMessageLogs>(expected, &found),
235  fail();
236  }
237  catch (...)
238  {
239  BEAST_EXPECT(found);
240  }
241  }
242  {
243  // Error: Mix safety_level and one setting (gotta catch 'em all)
245  auto const expected =
246  "Failed to initialize SQLite databases: Configuration file may "
247  "not define both \"safety_level\" and \"journal_mode\"";
248  bool found = false;
249 
250  auto p = test::jtx::envconfig();
251  {
252  auto& section = p->section("sqlite");
253  section.set("safety_level", "high");
254  section.set("journal_mode", "off");
255  }
256 
257  try
258  {
259  Env env(
260  *this,
261  std::move(p),
262  std::make_unique<CheckMessageLogs>(expected, &found),
264  fail();
265  }
266  catch (...)
267  {
268  BEAST_EXPECT(found);
269  }
270  }
271  {
272  // Error: Mix safety_level and one setting (gotta catch 'em all)
274  auto const expected =
275  "Failed to initialize SQLite databases: Configuration file may "
276  "not define both \"safety_level\" and \"synchronous\"";
277  bool found = false;
278 
279  auto p = test::jtx::envconfig();
280  {
281  auto& section = p->section("sqlite");
282  section.set("safety_level", "low");
283  section.set("synchronous", "extra");
284  }
285 
286  try
287  {
288  Env env(
289  *this,
290  std::move(p),
291  std::make_unique<CheckMessageLogs>(expected, &found),
293  fail();
294  }
295  catch (...)
296  {
297  BEAST_EXPECT(found);
298  }
299  }
300  {
301  // Error: Mix safety_level and one setting (gotta catch 'em all)
303  auto const expected =
304  "Failed to initialize SQLite databases: Configuration file may "
305  "not define both \"safety_level\" and \"temp_store\"";
306  bool found = false;
307 
308  auto p = test::jtx::envconfig();
309  {
310  auto& section = p->section("sqlite");
311  section.set("safety_level", "high");
312  section.set("temp_store", "default");
313  }
314 
315  try
316  {
317  Env env(
318  *this,
319  std::move(p),
320  std::make_unique<CheckMessageLogs>(expected, &found),
322  fail();
323  }
324  catch (...)
325  {
326  BEAST_EXPECT(found);
327  }
328  }
329  {
330  // Error: Invalid value
332  auto const expected =
333  "Failed to initialize SQLite databases: Invalid safety_level "
334  "value: slow";
335  bool found = false;
336 
337  auto p = test::jtx::envconfig();
338  {
339  auto& section = p->section("sqlite");
340  section.set("safety_level", "slow");
341  }
342 
343  try
344  {
345  Env env(
346  *this,
347  std::move(p),
348  std::make_unique<CheckMessageLogs>(expected, &found),
350  fail();
351  }
352  catch (...)
353  {
354  BEAST_EXPECT(found);
355  }
356  }
357  {
358  // Error: Invalid value
360  auto const expected =
361  "Failed to initialize SQLite databases: Invalid journal_mode "
362  "value: fast";
363  bool found = false;
364 
365  auto p = test::jtx::envconfig();
366  {
367  auto& section = p->section("sqlite");
368  section.set("journal_mode", "fast");
369  }
370 
371  try
372  {
373  Env env(
374  *this,
375  std::move(p),
376  std::make_unique<CheckMessageLogs>(expected, &found),
378  fail();
379  }
380  catch (...)
381  {
382  BEAST_EXPECT(found);
383  }
384  }
385  {
386  // Error: Invalid value
388  auto const expected =
389  "Failed to initialize SQLite databases: Invalid synchronous "
390  "value: instant";
391  bool found = false;
392 
393  auto p = test::jtx::envconfig();
394  {
395  auto& section = p->section("sqlite");
396  section.set("synchronous", "instant");
397  }
398 
399  try
400  {
401  Env env(
402  *this,
403  std::move(p),
404  std::make_unique<CheckMessageLogs>(expected, &found),
406  fail();
407  }
408  catch (...)
409  {
410  BEAST_EXPECT(found);
411  }
412  }
413  {
414  // Error: Invalid value
416  auto const expected =
417  "Failed to initialize SQLite databases: Invalid temp_store "
418  "value: network";
419  bool found = false;
420 
421  auto p = test::jtx::envconfig();
422  {
423  auto& section = p->section("sqlite");
424  section.set("temp_store", "network");
425  }
426 
427  try
428  {
429  Env env(
430  *this,
431  std::move(p),
432  std::make_unique<CheckMessageLogs>(expected, &found),
434  fail();
435  }
436  catch (...)
437  {
438  BEAST_EXPECT(found);
439  }
440  }
441  }
442 
443  //--------------------------------------------------------------------------
444 
445  void
447  std::string const& destBackendType,
448  std::string const& srcBackendType,
449  std::int64_t seedValue)
450  {
451  DummyScheduler scheduler;
452  RootStoppable parent("TestRootStoppable");
453 
454  beast::temp_dir node_db;
455  Section srcParams;
456  srcParams.set("type", srcBackendType);
457  srcParams.set("path", node_db.path());
458 
459  // Create a batch
460  auto batch = createPredictableBatch(numObjectsToTest, seedValue);
461 
462  // Write to source db
463  {
465  "test",
466  megabytes(4),
467  scheduler,
468  2,
469  parent,
470  srcParams,
471  journal_);
472  storeBatch(*src, batch);
473  }
474 
475  Batch copy;
476 
477  {
478  // Re-open the db
480  "test",
481  megabytes(4),
482  scheduler,
483  2,
484  parent,
485  srcParams,
486  journal_);
487 
488  // Set up the destination database
489  beast::temp_dir dest_db;
490  Section destParams;
491  destParams.set("type", destBackendType);
492  destParams.set("path", dest_db.path());
493 
495  "test",
496  megabytes(4),
497  scheduler,
498  2,
499  parent,
500  destParams,
501  journal_);
502 
503  testcase(
504  "import into '" + destBackendType + "' from '" +
505  srcBackendType + "'");
506 
507  // Do the import
508  dest->import(*src);
509 
510  // Get the results of the import
511  fetchCopyOfBatch(*dest, &copy, batch);
512  }
513 
514  // Canonicalize the source and destination batches
515  std::sort(batch.begin(), batch.end(), LessThan{});
516  std::sort(copy.begin(), copy.end(), LessThan{});
517  BEAST_EXPECT(areBatchesEqual(batch, copy));
518  }
519 
520  //--------------------------------------------------------------------------
521 
522  void
524  std::string const& type,
525  bool const testPersistence,
526  std::int64_t const seedValue,
527  int numObjsToTest = 2000)
528  {
529  DummyScheduler scheduler;
530  RootStoppable parent("TestRootStoppable");
531 
532  std::string s = "NodeStore backend '" + type + "'";
533 
534  testcase(s);
535 
536  beast::temp_dir node_db;
537  Section nodeParams;
538  nodeParams.set("type", type);
539  nodeParams.set("path", node_db.path());
540 
541  beast::xor_shift_engine rng(seedValue);
542 
543  // Create a batch
544  auto batch = createPredictableBatch(numObjsToTest, rng());
545 
546  {
547  // Open the database
549  "test",
550  megabytes(4),
551  scheduler,
552  2,
553  parent,
554  nodeParams,
555  journal_);
556 
557  // Write the batch
558  storeBatch(*db, batch);
559 
560  {
561  // Read it back in
562  Batch copy;
563  fetchCopyOfBatch(*db, &copy, batch);
564  BEAST_EXPECT(areBatchesEqual(batch, copy));
565  }
566 
567  {
568  // Reorder and read the copy again
569  std::shuffle(batch.begin(), batch.end(), rng);
570  Batch copy;
571  fetchCopyOfBatch(*db, &copy, batch);
572  BEAST_EXPECT(areBatchesEqual(batch, copy));
573  }
574  }
575 
576  if (testPersistence)
577  {
578  // Re-open the database without the ephemeral DB
580  "test",
581  megabytes(4),
582  scheduler,
583  2,
584  parent,
585  nodeParams,
586  journal_);
587 
588  // Read it back in
589  Batch copy;
590  fetchCopyOfBatch(*db, &copy, batch);
591 
592  // Canonicalize the source and destination batches
593  std::sort(batch.begin(), batch.end(), LessThan{});
594  std::sort(copy.begin(), copy.end(), LessThan{});
595  BEAST_EXPECT(areBatchesEqual(batch, copy));
596  }
597 
598  if (type == "memory")
599  {
600  // Earliest ledger sequence tests
601  {
602  // Verify default earliest ledger sequence
605  "test",
606  megabytes(4),
607  scheduler,
608  2,
609  parent,
610  nodeParams,
611  journal_);
612  BEAST_EXPECT(
613  db->earliestLedgerSeq() == XRP_LEDGER_EARLIEST_SEQ);
614  }
615 
616  // Set an invalid earliest ledger sequence
617  try
618  {
619  nodeParams.set("earliest_seq", "0");
622  "test",
623  megabytes(4),
624  scheduler,
625  2,
626  parent,
627  nodeParams,
628  journal_);
629  }
630  catch (std::runtime_error const& e)
631  {
632  BEAST_EXPECT(
633  std::strcmp(e.what(), "Invalid earliest_seq") == 0);
634  }
635 
636  {
637  // Set a valid earliest ledger sequence
638  nodeParams.set("earliest_seq", "1");
641  "test",
642  megabytes(4),
643  scheduler,
644  2,
645  parent,
646  nodeParams,
647  journal_);
648 
649  // Verify database uses the earliest ledger sequence setting
650  BEAST_EXPECT(db->earliestLedgerSeq() == 1);
651  }
652 
653  // Create another database that attempts to set the value again
654  try
655  {
656  // Set to default earliest ledger sequence
657  nodeParams.set(
658  "earliest_seq", std::to_string(XRP_LEDGER_EARLIEST_SEQ));
661  "test",
662  megabytes(4),
663  scheduler,
664  2,
665  parent,
666  nodeParams,
667  journal_);
668  }
669  catch (std::runtime_error const& e)
670  {
671  BEAST_EXPECT(
672  std::strcmp(e.what(), "earliest_seq set more than once") ==
673  0);
674  }
675  }
676  }
677 
678  //--------------------------------------------------------------------------
679 
680  void
681  run() override
682  {
683  std::int64_t const seedValue = 50;
684 
685  testConfig();
686 
687  testNodeStore("memory", false, seedValue);
688 
689  // Persistent backend tests
690  {
691  testNodeStore("nudb", true, seedValue);
692 
693 #if RIPPLE_ROCKSDB_AVAILABLE
694  testNodeStore("rocksdb", true, seedValue);
695 #endif
696  }
697 
698  // Import tests
699  {
700  testImport("nudb", "nudb", seedValue);
701 
702 #if RIPPLE_ROCKSDB_AVAILABLE
703  testImport("rocksdb", "rocksdb", seedValue);
704 #endif
705 
706 #if RIPPLE_ENABLE_SQLITE_BACKEND_TESTS
707  testImport("sqlite", "sqlite", seedValue);
708 #endif
709  }
710  }
711 };
712 
713 BEAST_DEFINE_TESTSUITE(Database, NodeStore, ripple);
714 
715 } // namespace NodeStore
716 } // namespace ripple
ripple::NodeStore::DummyScheduler
Simple NodeStore Scheduler that just peforms the tasks synchronously.
Definition: DummyScheduler.h:29
ripple::Section
Holds a collection of configuration values.
Definition: BasicConfig.h:43
std::strcmp
T strcmp(T... args)
ripple::DatabaseCon::Setup::globalPragma
static std::unique_ptr< std::vector< std::string > const > globalPragma
Definition: DatabaseCon.h:103
ripple::NodeStore::TestBase
Definition: TestBase.h:68
std::string
STL class.
ripple::NodeStore::Database_test::Database_test
Database_test()
Definition: Database_test.cpp:38
ripple::NodeStore::Manager::make_Database
virtual std::unique_ptr< Database > make_Database(std::string const &name, std::size_t burstSize, Scheduler &scheduler, int readThreads, Stoppable &parent, Section const &backendParameters, beast::Journal journal)=0
Construct a NodeStore database.
std::vector< std::shared_ptr< NodeObject > >
ripple::NodeStore::LessThan
Binary function that satisfies the strict-weak-ordering requirement.
Definition: TestBase.h:44
ripple::NodeStore::Database_test::testNodeStore
void testNodeStore(std::string const &type, bool const testPersistence, std::int64_t const seedValue, int numObjsToTest=2000)
Definition: Database_test.cpp:523
ripple::NodeStore::Database_test::testImport
void testImport(std::string const &destBackendType, std::string const &srcBackendType, std::int64_t seedValue)
Definition: Database_test.cpp:446
ripple::test::jtx::Env::app
Application & app()
Definition: Env.h:240
ripple::test::jtx::envconfig
std::unique_ptr< Config > envconfig()
creates and initializes a default configuration for jtx::Env
Definition: envconfig.h:49
ripple::NodeStore::Database_test::run
void run() override
Definition: Database_test.cpp:681
std::sort
T sort(T... args)
std::unique_ptr::reset
T reset(T... args)
ripple::NodeStore::TestBase::areBatchesEqual
static bool areBatchesEqual(Batch const &lhs, Batch const &rhs)
Definition: TestBase.h:120
ripple::NodeStore::BEAST_DEFINE_TESTSUITE
BEAST_DEFINE_TESTSUITE(Backend, ripple_core, ripple)
ripple::RootStoppable
Definition: Stoppable.h:354
ripple::Application::config
virtual Config & config()=0
ripple::megabytes
constexpr auto megabytes(T value) noexcept
Definition: ByteUtilities.h:34
ripple::NodeStore::TestBase::fetchCopyOfBatch
void fetchCopyOfBatch(Backend &backend, Batch *pCopy, Batch const &batch)
Definition: TestBase.h:155
std::to_string
T to_string(T... args)
std::runtime_error
STL class.
std::int64_t
ripple::test::SuiteJournal
Definition: SuiteJournal.h:88
beast::temp_dir::path
std::string path() const
Get the native path for the temporary directory.
Definition: temp_dir.h:66
ripple::NodeStore::TestBase::numObjectsToTest
static const int numObjectsToTest
Definition: TestBase.h:75
ripple::test::jtx
Definition: Check_test.cpp:26
ripple::NodeStore::TestBase::createPredictableBatch
static Batch createPredictableBatch(int numObjects, std::uint64_t seed)
Definition: TestBase.h:80
ripple::setup_DatabaseCon
DatabaseCon::Setup setup_DatabaseCon(Config const &c, boost::optional< beast::Journal > j=boost::none)
Definition: DatabaseCon.cpp:106
ripple
Use hash_* containers for keys that do not need a cryptographically secure hashing algorithm.
Definition: RCLCensorshipDetector.h:29
ripple::test
Definition: NegativeUNLVote.h:38
ripple::NodeStore::Database_test::journal_
test::SuiteJournal journal_
Definition: Database_test.cpp:35
ripple::Section::set
void set(std::string const &key, std::string const &value)
Set a key/value pair.
Definition: BasicConfig.cpp:32
beast::severities::kWarning
@ kWarning
Definition: Journal.h:37
ripple::XRP_LEDGER_EARLIEST_SEQ
static constexpr std::uint32_t XRP_LEDGER_EARLIEST_SEQ
The XRP ledger network's earliest allowed sequence.
Definition: SystemParameters.h:61
beast::detail::xor_shift_engine
Definition: xor_shift_engine.h:32
ripple::NodeStore::TestBase::storeBatch
void storeBatch(Backend &backend, Batch const &batch)
Definition: TestBase.h:145
ripple::NodeStore::Manager::instance
static Manager & instance()
Returns the instance of the manager singleton.
Definition: ManagerImp.cpp:119
ripple::NodeStore::Database_test::testConfig
void testConfig()
Definition: Database_test.cpp:43
std::unique_ptr
STL class.
std::shuffle
T shuffle(T... args)
ripple::test::jtx::Env
A transaction testing environment.
Definition: Env.h:115
beast::temp_dir
RAII temporary directory.
Definition: temp_dir.h:33
std::runtime_error::what
T what(T... args)
ripple::NodeStore::Database_test
Definition: Database_test.cpp:33