rippled
Database_test.cpp
1 //------------------------------------------------------------------------------
2 /*
3  This file is part of rippled: https://github.com/ripple/rippled
4  Copyright (c) 2012, 2013 Ripple Labs Inc.
5 
6  Permission to use, copy, modify, and/or distribute this software for any
7  purpose with or without fee is hereby granted, provided that the above
8  copyright notice and this permission notice appear in all copies.
9 
10  THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11  WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12  MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13  ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14  WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15  ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16  OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18 //==============================================================================
19 
20 #include <ripple/beast/utility/temp_dir.h>
21 #include <ripple/core/DatabaseCon.h>
22 #include <ripple/nodestore/DummyScheduler.h>
23 #include <ripple/nodestore/Manager.h>
24 #include <test/jtx.h>
25 #include <test/jtx/CheckMessageLogs.h>
26 #include <test/jtx/envconfig.h>
27 #include <test/nodestore/TestBase.h>
28 #include <test/unit_test/SuiteJournal.h>
29 
30 namespace ripple {
31 namespace NodeStore {
32 
33 class Database_test : public TestBase
34 {
36 
37 public:
38  Database_test() : journal_("Database_test", *this)
39  {
40  }
41 
42  void
44  {
45  testcase("Config");
46 
47  using namespace ripple::test;
48  using namespace ripple::test::jtx;
49 
50  auto const integrityWarning =
51  "reducing the data integrity guarantees from the "
52  "default [sqlite] behavior is not recommended for "
53  "nodes storing large amounts of history, because of the "
54  "difficulty inherent in rebuilding corrupted data.";
55  {
56  // defaults
57  Env env(*this);
58 
59  auto const s = setup_DatabaseCon(env.app().config());
60 
61  if (BEAST_EXPECT(s.globalPragma->size() == 3))
62  {
63  BEAST_EXPECT(
64  s.globalPragma->at(0) == "PRAGMA journal_mode=wal;");
65  BEAST_EXPECT(
66  s.globalPragma->at(1) == "PRAGMA synchronous=normal;");
67  BEAST_EXPECT(
68  s.globalPragma->at(2) == "PRAGMA temp_store=file;");
69  }
70  }
71  {
72  // High safety level
74 
75  bool found = false;
76  Env env = [&]() {
77  auto p = test::jtx::envconfig();
78  {
79  auto& section = p->section("sqlite");
80  section.set("safety_level", "high");
81  }
82  p->LEDGER_HISTORY = 100'000'000;
83 
84  return Env(
85  *this,
86  std::move(p),
87  std::make_unique<CheckMessageLogs>(
88  integrityWarning, &found),
90  }();
91 
92  BEAST_EXPECT(!found);
93  auto const s = setup_DatabaseCon(env.app().config());
94  if (BEAST_EXPECT(s.globalPragma->size() == 3))
95  {
96  BEAST_EXPECT(
97  s.globalPragma->at(0) == "PRAGMA journal_mode=wal;");
98  BEAST_EXPECT(
99  s.globalPragma->at(1) == "PRAGMA synchronous=normal;");
100  BEAST_EXPECT(
101  s.globalPragma->at(2) == "PRAGMA temp_store=file;");
102  }
103  }
104  {
105  // Low safety level
107 
108  bool found = false;
109  Env env = [&]() {
110  auto p = test::jtx::envconfig();
111  {
112  auto& section = p->section("sqlite");
113  section.set("safety_level", "low");
114  }
115  p->LEDGER_HISTORY = 100'000'000;
116 
117  return Env(
118  *this,
119  std::move(p),
120  std::make_unique<CheckMessageLogs>(
121  integrityWarning, &found),
123  }();
124 
125  BEAST_EXPECT(found);
126  auto const s = setup_DatabaseCon(env.app().config());
127  if (BEAST_EXPECT(s.globalPragma->size() == 3))
128  {
129  BEAST_EXPECT(
130  s.globalPragma->at(0) == "PRAGMA journal_mode=memory;");
131  BEAST_EXPECT(
132  s.globalPragma->at(1) == "PRAGMA synchronous=off;");
133  BEAST_EXPECT(
134  s.globalPragma->at(2) == "PRAGMA temp_store=memory;");
135  }
136  }
137  {
138  // Override individual settings
140 
141  bool found = false;
142  Env env = [&]() {
143  auto p = test::jtx::envconfig();
144  {
145  auto& section = p->section("sqlite");
146  section.set("journal_mode", "off");
147  section.set("synchronous", "extra");
148  section.set("temp_store", "default");
149  }
150 
151  return Env(
152  *this,
153  std::move(p),
154  std::make_unique<CheckMessageLogs>(
155  integrityWarning, &found),
157  }();
158 
159  // No warning, even though higher risk settings were used because
160  // LEDGER_HISTORY is small
161  BEAST_EXPECT(!found);
162  auto const s = setup_DatabaseCon(env.app().config());
163  if (BEAST_EXPECT(s.globalPragma->size() == 3))
164  {
165  BEAST_EXPECT(
166  s.globalPragma->at(0) == "PRAGMA journal_mode=off;");
167  BEAST_EXPECT(
168  s.globalPragma->at(1) == "PRAGMA synchronous=extra;");
169  BEAST_EXPECT(
170  s.globalPragma->at(2) == "PRAGMA temp_store=default;");
171  }
172  }
173  {
174  // Override individual settings with large history
176 
177  bool found = false;
178  Env env = [&]() {
179  auto p = test::jtx::envconfig();
180  {
181  auto& section = p->section("sqlite");
182  section.set("journal_mode", "off");
183  section.set("synchronous", "extra");
184  section.set("temp_store", "default");
185  }
186  p->LEDGER_HISTORY = 50'000'000;
187 
188  return Env(
189  *this,
190  std::move(p),
191  std::make_unique<CheckMessageLogs>(
192  integrityWarning, &found),
194  }();
195 
196  // No warning, even though higher risk settings were used because
197  // LEDGER_HISTORY is small
198  BEAST_EXPECT(found);
199  auto const s = setup_DatabaseCon(env.app().config());
200  if (BEAST_EXPECT(s.globalPragma->size() == 3))
201  {
202  BEAST_EXPECT(
203  s.globalPragma->at(0) == "PRAGMA journal_mode=off;");
204  BEAST_EXPECT(
205  s.globalPragma->at(1) == "PRAGMA synchronous=extra;");
206  BEAST_EXPECT(
207  s.globalPragma->at(2) == "PRAGMA temp_store=default;");
208  }
209  }
210  {
211  // Error: Mix safety_level and individual settings
213  auto const expected =
214  "Failed to initialize SQLite databases: "
215  "Configuration file may not define both \"safety_level\" and "
216  "\"journal_mode\"";
217  bool found = false;
218 
219  auto p = test::jtx::envconfig();
220  {
221  auto& section = p->section("sqlite");
222  section.set("safety_level", "low");
223  section.set("journal_mode", "off");
224  section.set("synchronous", "extra");
225  section.set("temp_store", "default");
226  }
227 
228  try
229  {
230  Env env(
231  *this,
232  std::move(p),
233  std::make_unique<CheckMessageLogs>(expected, &found),
235  fail();
236  }
237  catch (...)
238  {
239  BEAST_EXPECT(found);
240  }
241  }
242  {
243  // Error: Mix safety_level and one setting (gotta catch 'em all)
245  auto const expected =
246  "Failed to initialize SQLite databases: Configuration file may "
247  "not define both \"safety_level\" and \"journal_mode\"";
248  bool found = false;
249 
250  auto p = test::jtx::envconfig();
251  {
252  auto& section = p->section("sqlite");
253  section.set("safety_level", "high");
254  section.set("journal_mode", "off");
255  }
256 
257  try
258  {
259  Env env(
260  *this,
261  std::move(p),
262  std::make_unique<CheckMessageLogs>(expected, &found),
264  fail();
265  }
266  catch (...)
267  {
268  BEAST_EXPECT(found);
269  }
270  }
271  {
272  // Error: Mix safety_level and one setting (gotta catch 'em all)
274  auto const expected =
275  "Failed to initialize SQLite databases: Configuration file may "
276  "not define both \"safety_level\" and \"synchronous\"";
277  bool found = false;
278 
279  auto p = test::jtx::envconfig();
280  {
281  auto& section = p->section("sqlite");
282  section.set("safety_level", "low");
283  section.set("synchronous", "extra");
284  }
285 
286  try
287  {
288  Env env(
289  *this,
290  std::move(p),
291  std::make_unique<CheckMessageLogs>(expected, &found),
293  fail();
294  }
295  catch (...)
296  {
297  BEAST_EXPECT(found);
298  }
299  }
300  {
301  // Error: Mix safety_level and one setting (gotta catch 'em all)
303  auto const expected =
304  "Failed to initialize SQLite databases: Configuration file may "
305  "not define both \"safety_level\" and \"temp_store\"";
306  bool found = false;
307 
308  auto p = test::jtx::envconfig();
309  {
310  auto& section = p->section("sqlite");
311  section.set("safety_level", "high");
312  section.set("temp_store", "default");
313  }
314 
315  try
316  {
317  Env env(
318  *this,
319  std::move(p),
320  std::make_unique<CheckMessageLogs>(expected, &found),
322  fail();
323  }
324  catch (...)
325  {
326  BEAST_EXPECT(found);
327  }
328  }
329  {
330  // Error: Invalid value
332  auto const expected =
333  "Failed to initialize SQLite databases: Invalid safety_level "
334  "value: slow";
335  bool found = false;
336 
337  auto p = test::jtx::envconfig();
338  {
339  auto& section = p->section("sqlite");
340  section.set("safety_level", "slow");
341  }
342 
343  try
344  {
345  Env env(
346  *this,
347  std::move(p),
348  std::make_unique<CheckMessageLogs>(expected, &found),
350  fail();
351  }
352  catch (...)
353  {
354  BEAST_EXPECT(found);
355  }
356  }
357  {
358  // Error: Invalid value
360  auto const expected =
361  "Failed to initialize SQLite databases: Invalid journal_mode "
362  "value: fast";
363  bool found = false;
364 
365  auto p = test::jtx::envconfig();
366  {
367  auto& section = p->section("sqlite");
368  section.set("journal_mode", "fast");
369  }
370 
371  try
372  {
373  Env env(
374  *this,
375  std::move(p),
376  std::make_unique<CheckMessageLogs>(expected, &found),
378  fail();
379  }
380  catch (...)
381  {
382  BEAST_EXPECT(found);
383  }
384  }
385  {
386  // Error: Invalid value
388  auto const expected =
389  "Failed to initialize SQLite databases: Invalid synchronous "
390  "value: instant";
391  bool found = false;
392 
393  auto p = test::jtx::envconfig();
394  {
395  auto& section = p->section("sqlite");
396  section.set("synchronous", "instant");
397  }
398 
399  try
400  {
401  Env env(
402  *this,
403  std::move(p),
404  std::make_unique<CheckMessageLogs>(expected, &found),
406  fail();
407  }
408  catch (...)
409  {
410  BEAST_EXPECT(found);
411  }
412  }
413  {
414  // Error: Invalid value
416  auto const expected =
417  "Failed to initialize SQLite databases: Invalid temp_store "
418  "value: network";
419  bool found = false;
420 
421  auto p = test::jtx::envconfig();
422  {
423  auto& section = p->section("sqlite");
424  section.set("temp_store", "network");
425  }
426 
427  try
428  {
429  Env env(
430  *this,
431  std::move(p),
432  std::make_unique<CheckMessageLogs>(expected, &found),
434  fail();
435  }
436  catch (...)
437  {
438  BEAST_EXPECT(found);
439  }
440  }
441  }
442 
443  //--------------------------------------------------------------------------
444 
445  void
447  std::string const& destBackendType,
448  std::string const& srcBackendType,
449  std::int64_t seedValue)
450  {
451  DummyScheduler scheduler;
452  RootStoppable parent("TestRootStoppable");
453 
454  beast::temp_dir node_db;
455  Section srcParams;
456  srcParams.set("type", srcBackendType);
457  srcParams.set("path", node_db.path());
458 
459  // Create a batch
460  auto batch = createPredictableBatch(numObjectsToTest, seedValue);
461 
462  // Write to source db
463  {
465  "test", scheduler, 2, parent, srcParams, journal_);
466  storeBatch(*src, batch);
467  }
468 
469  Batch copy;
470 
471  {
472  // Re-open the db
474  "test", scheduler, 2, parent, srcParams, journal_);
475 
476  // Set up the destination database
477  beast::temp_dir dest_db;
478  Section destParams;
479  destParams.set("type", destBackendType);
480  destParams.set("path", dest_db.path());
481 
483  "test", scheduler, 2, parent, destParams, journal_);
484 
485  testcase(
486  "import into '" + destBackendType + "' from '" +
487  srcBackendType + "'");
488 
489  // Do the import
490  dest->import(*src);
491 
492  // Get the results of the import
493  fetchCopyOfBatch(*dest, &copy, batch);
494  }
495 
496  // Canonicalize the source and destination batches
497  std::sort(batch.begin(), batch.end(), LessThan{});
498  std::sort(copy.begin(), copy.end(), LessThan{});
499  BEAST_EXPECT(areBatchesEqual(batch, copy));
500  }
501 
502  //--------------------------------------------------------------------------
503 
504  void
506  std::string const& type,
507  bool const testPersistence,
508  std::int64_t const seedValue,
509  int numObjsToTest = 2000)
510  {
511  DummyScheduler scheduler;
512  RootStoppable parent("TestRootStoppable");
513 
514  std::string s = "NodeStore backend '" + type + "'";
515 
516  testcase(s);
517 
518  beast::temp_dir node_db;
519  Section nodeParams;
520  nodeParams.set("type", type);
521  nodeParams.set("path", node_db.path());
522 
523  beast::xor_shift_engine rng(seedValue);
524 
525  // Create a batch
526  auto batch = createPredictableBatch(numObjsToTest, rng());
527 
528  {
529  // Open the database
531  "test", scheduler, 2, parent, nodeParams, journal_);
532 
533  // Write the batch
534  storeBatch(*db, batch);
535 
536  {
537  // Read it back in
538  Batch copy;
539  fetchCopyOfBatch(*db, &copy, batch);
540  BEAST_EXPECT(areBatchesEqual(batch, copy));
541  }
542 
543  {
544  // Reorder and read the copy again
545  std::shuffle(batch.begin(), batch.end(), rng);
546  Batch copy;
547  fetchCopyOfBatch(*db, &copy, batch);
548  BEAST_EXPECT(areBatchesEqual(batch, copy));
549  }
550  }
551 
552  if (testPersistence)
553  {
554  // Re-open the database without the ephemeral DB
556  "test", scheduler, 2, parent, nodeParams, journal_);
557 
558  // Read it back in
559  Batch copy;
560  fetchCopyOfBatch(*db, &copy, batch);
561 
562  // Canonicalize the source and destination batches
563  std::sort(batch.begin(), batch.end(), LessThan{});
564  std::sort(copy.begin(), copy.end(), LessThan{});
565  BEAST_EXPECT(areBatchesEqual(batch, copy));
566  }
567 
568  if (type == "memory")
569  {
570  // Earliest ledger sequence tests
571  {
572  // Verify default earliest ledger sequence
575  "test", scheduler, 2, parent, nodeParams, journal_);
576  BEAST_EXPECT(
577  db->earliestLedgerSeq() == XRP_LEDGER_EARLIEST_SEQ);
578  }
579 
580  // Set an invalid earliest ledger sequence
581  try
582  {
583  nodeParams.set("earliest_seq", "0");
586  "test", scheduler, 2, parent, nodeParams, journal_);
587  }
588  catch (std::runtime_error const& e)
589  {
590  BEAST_EXPECT(
591  std::strcmp(e.what(), "Invalid earliest_seq") == 0);
592  }
593 
594  {
595  // Set a valid earliest ledger sequence
596  nodeParams.set("earliest_seq", "1");
599  "test", scheduler, 2, parent, nodeParams, journal_);
600 
601  // Verify database uses the earliest ledger sequence setting
602  BEAST_EXPECT(db->earliestLedgerSeq() == 1);
603  }
604 
605  // Create another database that attempts to set the value again
606  try
607  {
608  // Set to default earliest ledger sequence
609  nodeParams.set(
610  "earliest_seq", std::to_string(XRP_LEDGER_EARLIEST_SEQ));
613  "test", scheduler, 2, parent, nodeParams, journal_);
614  }
615  catch (std::runtime_error const& e)
616  {
617  BEAST_EXPECT(
618  std::strcmp(e.what(), "earliest_seq set more than once") ==
619  0);
620  }
621  }
622  }
623 
624  //--------------------------------------------------------------------------
625 
626  void
627  run() override
628  {
629  std::int64_t const seedValue = 50;
630 
631  testConfig();
632 
633  testNodeStore("memory", false, seedValue);
634 
635  // Persistent backend tests
636  {
637  testNodeStore("nudb", true, seedValue);
638 
639 #if RIPPLE_ROCKSDB_AVAILABLE
640  testNodeStore("rocksdb", true, seedValue);
641 #endif
642  }
643 
644  // Import tests
645  {
646  testImport("nudb", "nudb", seedValue);
647 
648 #if RIPPLE_ROCKSDB_AVAILABLE
649  testImport("rocksdb", "rocksdb", seedValue);
650 #endif
651 
652 #if RIPPLE_ENABLE_SQLITE_BACKEND_TESTS
653  testImport("sqlite", "sqlite", seedValue);
654 #endif
655  }
656  }
657 };
658 
659 BEAST_DEFINE_TESTSUITE(Database, NodeStore, ripple);
660 
661 } // namespace NodeStore
662 } // namespace ripple
ripple::NodeStore::DummyScheduler
Simple NodeStore Scheduler that just peforms the tasks synchronously.
Definition: DummyScheduler.h:29
ripple::Section
Holds a collection of configuration values.
Definition: BasicConfig.h:43
std::strcmp
T strcmp(T... args)
ripple::DatabaseCon::Setup::globalPragma
static std::unique_ptr< std::vector< std::string > const > globalPragma
Definition: DatabaseCon.h:103
ripple::NodeStore::TestBase
Definition: TestBase.h:68
std::string
STL class.
ripple::NodeStore::Database_test::Database_test
Database_test()
Definition: Database_test.cpp:38
std::vector< std::shared_ptr< NodeObject > >
ripple::NodeStore::LessThan
Binary function that satisfies the strict-weak-ordering requirement.
Definition: TestBase.h:44
ripple::NodeStore::Database_test::testNodeStore
void testNodeStore(std::string const &type, bool const testPersistence, std::int64_t const seedValue, int numObjsToTest=2000)
Definition: Database_test.cpp:505
ripple::NodeStore::Database_test::testImport
void testImport(std::string const &destBackendType, std::string const &srcBackendType, std::int64_t seedValue)
Definition: Database_test.cpp:446
ripple::test::jtx::Env::app
Application & app()
Definition: Env.h:240
ripple::test::jtx::envconfig
std::unique_ptr< Config > envconfig()
creates and initializes a default configuration for jtx::Env
Definition: envconfig.h:49
ripple::NodeStore::Database_test::run
void run() override
Definition: Database_test.cpp:627
std::sort
T sort(T... args)
std::unique_ptr::reset
T reset(T... args)
ripple::NodeStore::TestBase::areBatchesEqual
static bool areBatchesEqual(Batch const &lhs, Batch const &rhs)
Definition: TestBase.h:120
ripple::NodeStore::BEAST_DEFINE_TESTSUITE
BEAST_DEFINE_TESTSUITE(Backend, ripple_core, ripple)
ripple::RootStoppable
Definition: Stoppable.h:353
ripple::Application::config
virtual Config & config()=0
ripple::NodeStore::TestBase::fetchCopyOfBatch
void fetchCopyOfBatch(Backend &backend, Batch *pCopy, Batch const &batch)
Definition: TestBase.h:155
std::to_string
T to_string(T... args)
std::runtime_error
STL class.
std::int64_t
ripple::test::SuiteJournal
Definition: SuiteJournal.h:88
beast::temp_dir::path
std::string path() const
Get the native path for the temporary directory.
Definition: temp_dir.h:66
ripple::NodeStore::TestBase::numObjectsToTest
static const int numObjectsToTest
Definition: TestBase.h:75
ripple::test::jtx
Definition: Check_test.cpp:26
ripple::NodeStore::TestBase::createPredictableBatch
static Batch createPredictableBatch(int numObjects, std::uint64_t seed)
Definition: TestBase.h:80
ripple::setup_DatabaseCon
DatabaseCon::Setup setup_DatabaseCon(Config const &c, boost::optional< beast::Journal > j=boost::none)
Definition: DatabaseCon.cpp:93
ripple
Use hash_* containers for keys that do not need a cryptographically secure hashing algorithm.
Definition: RCLCensorshipDetector.h:29
ripple::test
Definition: NegativeUNLVote.h:38
ripple::NodeStore::Database_test::journal_
test::SuiteJournal journal_
Definition: Database_test.cpp:35
ripple::Section::set
void set(std::string const &key, std::string const &value)
Set a key/value pair.
Definition: BasicConfig.cpp:32
beast::severities::kWarning
@ kWarning
Definition: Journal.h:37
ripple::XRP_LEDGER_EARLIEST_SEQ
static constexpr std::uint32_t XRP_LEDGER_EARLIEST_SEQ
The XRP ledger network's earliest allowed sequence.
Definition: SystemParameters.h:61
beast::detail::xor_shift_engine
Definition: xor_shift_engine.h:32
ripple::NodeStore::TestBase::storeBatch
void storeBatch(Backend &backend, Batch const &batch)
Definition: TestBase.h:145
ripple::NodeStore::Manager::make_Database
virtual std::unique_ptr< Database > make_Database(std::string const &name, Scheduler &scheduler, int readThreads, Stoppable &parent, Section const &backendParameters, beast::Journal journal)=0
Construct a NodeStore database.
ripple::NodeStore::Manager::instance
static Manager & instance()
Returns the instance of the manager singleton.
Definition: ManagerImp.cpp:117
ripple::NodeStore::Database_test::testConfig
void testConfig()
Definition: Database_test.cpp:43
std::unique_ptr
STL class.
std::shuffle
T shuffle(T... args)
ripple::test::jtx::Env
A transaction testing environment.
Definition: Env.h:115
beast::temp_dir
RAII temporary directory.
Definition: temp_dir.h:33
std::runtime_error::what
T what(T... args)
ripple::NodeStore::Database_test
Definition: Database_test.cpp:33