rippled
Loading...
Searching...
No Matches
Database_test.cpp
1#include <test/jtx.h>
2#include <test/jtx/CheckMessageLogs.h>
3#include <test/jtx/envconfig.h>
4#include <test/nodestore/TestBase.h>
5#include <test/unit_test/SuiteJournal.h>
6
7#include <xrpld/core/DatabaseCon.h>
8
9#include <xrpl/beast/utility/temp_dir.h>
10#include <xrpl/nodestore/DummyScheduler.h>
11#include <xrpl/nodestore/Manager.h>
12
13namespace xrpl {
14
15namespace NodeStore {
16
17class Database_test : public TestBase
18{
20
21public:
22 Database_test() : journal_("Database_test", *this)
23 {
24 }
25
26 void
28 {
29 testcase("Config");
30
31 using namespace xrpl::test;
32 using namespace xrpl::test::jtx;
33
34 auto const integrityWarning =
35 "reducing the data integrity guarantees from the "
36 "default [sqlite] behavior is not recommended for "
37 "nodes storing large amounts of history, because of the "
38 "difficulty inherent in rebuilding corrupted data.";
39 {
40 // defaults
41 Env env(*this);
42
43 auto const s = setup_DatabaseCon(env.app().config());
44
45 if (BEAST_EXPECT(s.globalPragma->size() == 3))
46 {
47 BEAST_EXPECT(s.globalPragma->at(0) == "PRAGMA journal_mode=wal;");
48 BEAST_EXPECT(s.globalPragma->at(1) == "PRAGMA synchronous=normal;");
49 BEAST_EXPECT(s.globalPragma->at(2) == "PRAGMA temp_store=file;");
50 }
51 }
52 {
53 // High safety level
55
56 bool found = false;
57 Env env = [&]() {
58 auto p = test::jtx::envconfig();
59 {
60 auto& section = p->section("sqlite");
61 section.set("safety_level", "high");
62 }
63 p->LEDGER_HISTORY = 100'000'000;
64
65 return Env(
66 *this,
67 std::move(p),
68 std::make_unique<CheckMessageLogs>(integrityWarning, &found),
70 }();
71
72 BEAST_EXPECT(!found);
73 auto const s = setup_DatabaseCon(env.app().config());
74 if (BEAST_EXPECT(s.globalPragma->size() == 3))
75 {
76 BEAST_EXPECT(s.globalPragma->at(0) == "PRAGMA journal_mode=wal;");
77 BEAST_EXPECT(s.globalPragma->at(1) == "PRAGMA synchronous=normal;");
78 BEAST_EXPECT(s.globalPragma->at(2) == "PRAGMA temp_store=file;");
79 }
80 }
81 {
82 // Low safety level
84
85 bool found = false;
86 Env env = [&]() {
87 auto p = test::jtx::envconfig();
88 {
89 auto& section = p->section("sqlite");
90 section.set("safety_level", "low");
91 }
92 p->LEDGER_HISTORY = 100'000'000;
93
94 return Env(
95 *this,
96 std::move(p),
97 std::make_unique<CheckMessageLogs>(integrityWarning, &found),
99 }();
100
101 BEAST_EXPECT(found);
102 auto const s = setup_DatabaseCon(env.app().config());
103 if (BEAST_EXPECT(s.globalPragma->size() == 3))
104 {
105 BEAST_EXPECT(s.globalPragma->at(0) == "PRAGMA journal_mode=memory;");
106 BEAST_EXPECT(s.globalPragma->at(1) == "PRAGMA synchronous=off;");
107 BEAST_EXPECT(s.globalPragma->at(2) == "PRAGMA temp_store=memory;");
108 }
109 }
110 {
111 // Override individual settings
113
114 bool found = false;
115 Env env = [&]() {
116 auto p = test::jtx::envconfig();
117 {
118 auto& section = p->section("sqlite");
119 section.set("journal_mode", "off");
120 section.set("synchronous", "extra");
121 section.set("temp_store", "default");
122 }
123
124 return Env(
125 *this,
126 std::move(p),
127 std::make_unique<CheckMessageLogs>(integrityWarning, &found),
129 }();
130
131 // No warning, even though higher risk settings were used because
132 // LEDGER_HISTORY is small
133 BEAST_EXPECT(!found);
134 auto const s = setup_DatabaseCon(env.app().config());
135 if (BEAST_EXPECT(s.globalPragma->size() == 3))
136 {
137 BEAST_EXPECT(s.globalPragma->at(0) == "PRAGMA journal_mode=off;");
138 BEAST_EXPECT(s.globalPragma->at(1) == "PRAGMA synchronous=extra;");
139 BEAST_EXPECT(s.globalPragma->at(2) == "PRAGMA temp_store=default;");
140 }
141 }
142 {
143 // Override individual settings with large history
145
146 bool found = false;
147 Env env = [&]() {
148 auto p = test::jtx::envconfig();
149 {
150 auto& section = p->section("sqlite");
151 section.set("journal_mode", "off");
152 section.set("synchronous", "extra");
153 section.set("temp_store", "default");
154 }
155 p->LEDGER_HISTORY = 50'000'000;
156
157 return Env(
158 *this,
159 std::move(p),
160 std::make_unique<CheckMessageLogs>(integrityWarning, &found),
162 }();
163
164 // No warning, even though higher risk settings were used because
165 // LEDGER_HISTORY is small
166 BEAST_EXPECT(found);
167 auto const s = setup_DatabaseCon(env.app().config());
168 if (BEAST_EXPECT(s.globalPragma->size() == 3))
169 {
170 BEAST_EXPECT(s.globalPragma->at(0) == "PRAGMA journal_mode=off;");
171 BEAST_EXPECT(s.globalPragma->at(1) == "PRAGMA synchronous=extra;");
172 BEAST_EXPECT(s.globalPragma->at(2) == "PRAGMA temp_store=default;");
173 }
174 }
175 {
176 // Error: Mix safety_level and individual settings
178 auto const expected =
179 "Failed to initialize SQL databases: "
180 "Configuration file may not define both \"safety_level\" and "
181 "\"journal_mode\"";
182 bool found = false;
183
184 auto p = test::jtx::envconfig();
185 {
186 auto& section = p->section("sqlite");
187 section.set("safety_level", "low");
188 section.set("journal_mode", "off");
189 section.set("synchronous", "extra");
190 section.set("temp_store", "default");
191 }
192
193 try
194 {
195 Env env(
196 *this,
197 std::move(p),
198 std::make_unique<CheckMessageLogs>(expected, &found),
200 fail();
201 }
202 catch (...)
203 {
204 BEAST_EXPECT(found);
205 }
206 }
207 {
208 // Error: Mix safety_level and one setting (gotta catch 'em all)
210 auto const expected =
211 "Failed to initialize SQL databases: Configuration file may "
212 "not define both \"safety_level\" and \"journal_mode\"";
213 bool found = false;
214
215 auto p = test::jtx::envconfig();
216 {
217 auto& section = p->section("sqlite");
218 section.set("safety_level", "high");
219 section.set("journal_mode", "off");
220 }
221
222 try
223 {
224 Env env(
225 *this,
226 std::move(p),
227 std::make_unique<CheckMessageLogs>(expected, &found),
229 fail();
230 }
231 catch (...)
232 {
233 BEAST_EXPECT(found);
234 }
235 }
236 {
237 // Error: Mix safety_level and one setting (gotta catch 'em all)
239 auto const expected =
240 "Failed to initialize SQL databases: Configuration file may "
241 "not define both \"safety_level\" and \"synchronous\"";
242 bool found = false;
243
244 auto p = test::jtx::envconfig();
245 {
246 auto& section = p->section("sqlite");
247 section.set("safety_level", "low");
248 section.set("synchronous", "extra");
249 }
250
251 try
252 {
253 Env env(
254 *this,
255 std::move(p),
256 std::make_unique<CheckMessageLogs>(expected, &found),
258 fail();
259 }
260 catch (...)
261 {
262 BEAST_EXPECT(found);
263 }
264 }
265 {
266 // Error: Mix safety_level and one setting (gotta catch 'em all)
268 auto const expected =
269 "Failed to initialize SQL databases: Configuration file may "
270 "not define both \"safety_level\" and \"temp_store\"";
271 bool found = false;
272
273 auto p = test::jtx::envconfig();
274 {
275 auto& section = p->section("sqlite");
276 section.set("safety_level", "high");
277 section.set("temp_store", "default");
278 }
279
280 try
281 {
282 Env env(
283 *this,
284 std::move(p),
285 std::make_unique<CheckMessageLogs>(expected, &found),
287 fail();
288 }
289 catch (...)
290 {
291 BEAST_EXPECT(found);
292 }
293 }
294 {
295 // Error: Invalid value
297 auto const expected =
298 "Failed to initialize SQL databases: Invalid safety_level "
299 "value: slow";
300 bool found = false;
301
302 auto p = test::jtx::envconfig();
303 {
304 auto& section = p->section("sqlite");
305 section.set("safety_level", "slow");
306 }
307
308 try
309 {
310 Env env(
311 *this,
312 std::move(p),
313 std::make_unique<CheckMessageLogs>(expected, &found),
315 fail();
316 }
317 catch (...)
318 {
319 BEAST_EXPECT(found);
320 }
321 }
322 {
323 // Error: Invalid value
325 auto const expected =
326 "Failed to initialize SQL databases: Invalid journal_mode "
327 "value: fast";
328 bool found = false;
329
330 auto p = test::jtx::envconfig();
331 {
332 auto& section = p->section("sqlite");
333 section.set("journal_mode", "fast");
334 }
335
336 try
337 {
338 Env env(
339 *this,
340 std::move(p),
341 std::make_unique<CheckMessageLogs>(expected, &found),
343 fail();
344 }
345 catch (...)
346 {
347 BEAST_EXPECT(found);
348 }
349 }
350 {
351 // Error: Invalid value
353 auto const expected =
354 "Failed to initialize SQL databases: Invalid synchronous "
355 "value: instant";
356 bool found = false;
357
358 auto p = test::jtx::envconfig();
359 {
360 auto& section = p->section("sqlite");
361 section.set("synchronous", "instant");
362 }
363
364 try
365 {
366 Env env(
367 *this,
368 std::move(p),
369 std::make_unique<CheckMessageLogs>(expected, &found),
371 fail();
372 }
373 catch (...)
374 {
375 BEAST_EXPECT(found);
376 }
377 }
378 {
379 // Error: Invalid value
381 auto const expected =
382 "Failed to initialize SQL databases: Invalid temp_store "
383 "value: network";
384 bool found = false;
385
386 auto p = test::jtx::envconfig();
387 {
388 auto& section = p->section("sqlite");
389 section.set("temp_store", "network");
390 }
391
392 try
393 {
394 Env env(
395 *this,
396 std::move(p),
397 std::make_unique<CheckMessageLogs>(expected, &found),
399 fail();
400 }
401 catch (...)
402 {
403 BEAST_EXPECT(found);
404 }
405 }
406 {
407 // N/A: Default values
408 Env env(*this);
409 auto const s = setup_DatabaseCon(env.app().config());
410 if (BEAST_EXPECT(s.txPragma.size() == 4))
411 {
412 BEAST_EXPECT(s.txPragma.at(0) == "PRAGMA page_size=4096;");
413 BEAST_EXPECT(s.txPragma.at(1) == "PRAGMA journal_size_limit=1582080;");
414 BEAST_EXPECT(s.txPragma.at(2) == "PRAGMA max_page_count=4294967294;");
415 BEAST_EXPECT(s.txPragma.at(3) == "PRAGMA mmap_size=17179869184;");
416 }
417 }
418 {
419 // Success: Valid values
420 Env env = [&]() {
421 auto p = test::jtx::envconfig();
422 {
423 auto& section = p->section("sqlite");
424 section.set("page_size", "512");
425 section.set("journal_size_limit", "2582080");
426 }
427 return Env(*this, std::move(p));
428 }();
429 auto const s = setup_DatabaseCon(env.app().config());
430 if (BEAST_EXPECT(s.txPragma.size() == 4))
431 {
432 BEAST_EXPECT(s.txPragma.at(0) == "PRAGMA page_size=512;");
433 BEAST_EXPECT(s.txPragma.at(1) == "PRAGMA journal_size_limit=2582080;");
434 BEAST_EXPECT(s.txPragma.at(2) == "PRAGMA max_page_count=4294967294;");
435 BEAST_EXPECT(s.txPragma.at(3) == "PRAGMA mmap_size=17179869184;");
436 }
437 }
438 {
439 // Error: Invalid values
440 auto const expected = "Invalid page_size. Must be between 512 and 65536.";
441 bool found = false;
442 auto p = test::jtx::envconfig();
443 {
444 auto& section = p->section("sqlite");
445 section.set("page_size", "256");
446 }
447 try
448 {
449 Env env(
450 *this,
451 std::move(p),
452 std::make_unique<CheckMessageLogs>(expected, &found),
454 fail();
455 }
456 catch (...)
457 {
458 BEAST_EXPECT(found);
459 }
460 }
461 {
462 // Error: Invalid values
463 auto const expected = "Invalid page_size. Must be between 512 and 65536.";
464 bool found = false;
465 auto p = test::jtx::envconfig();
466 {
467 auto& section = p->section("sqlite");
468 section.set("page_size", "131072");
469 }
470 try
471 {
472 Env env(
473 *this,
474 std::move(p),
475 std::make_unique<CheckMessageLogs>(expected, &found),
477 fail();
478 }
479 catch (...)
480 {
481 BEAST_EXPECT(found);
482 }
483 }
484 {
485 // Error: Invalid values
486 auto const expected = "Invalid page_size. Must be a power of 2.";
487 bool found = false;
488 auto p = test::jtx::envconfig();
489 {
490 auto& section = p->section("sqlite");
491 section.set("page_size", "513");
492 }
493 try
494 {
495 Env env(
496 *this,
497 std::move(p),
498 std::make_unique<CheckMessageLogs>(expected, &found),
500 fail();
501 }
502 catch (...)
503 {
504 BEAST_EXPECT(found);
505 }
506 }
507 }
508
509 //--------------------------------------------------------------------------
510
511 void
512 testImport(std::string const& destBackendType, std::string const& srcBackendType, std::int64_t seedValue)
513 {
514 DummyScheduler scheduler;
515
516 beast::temp_dir node_db;
517 Section srcParams;
518 srcParams.set("type", srcBackendType);
519 srcParams.set("path", node_db.path());
520
521 // Create a batch
523
524 // Write to source db
525 {
527 Manager::instance().make_Database(megabytes(4), scheduler, 2, srcParams, journal_);
528 storeBatch(*src, batch);
529 }
530
531 Batch copy;
532
533 {
534 // Re-open the db
536 Manager::instance().make_Database(megabytes(4), scheduler, 2, srcParams, journal_);
537
538 // Set up the destination database
539 beast::temp_dir dest_db;
540 Section destParams;
541 destParams.set("type", destBackendType);
542 destParams.set("path", dest_db.path());
543
545 Manager::instance().make_Database(megabytes(4), scheduler, 2, destParams, journal_);
546
547 testcase("import into '" + destBackendType + "' from '" + srcBackendType + "'");
548
549 // Do the import
550 dest->importDatabase(*src);
551
552 // Get the results of the import
553 fetchCopyOfBatch(*dest, &copy, batch);
554 }
555
556 // Canonicalize the source and destination batches
557 std::sort(batch.begin(), batch.end(), LessThan{});
558 std::sort(copy.begin(), copy.end(), LessThan{});
559 BEAST_EXPECT(areBatchesEqual(batch, copy));
560 }
561
562 //--------------------------------------------------------------------------
563
564 void
566 std::string const& type,
567 bool const testPersistence,
568 std::int64_t const seedValue,
569 int numObjsToTest = 2000)
570 {
571 DummyScheduler scheduler;
572
573 std::string s = "NodeStore backend '" + type + "'";
574
575 testcase(s);
576
577 beast::temp_dir node_db;
578 Section nodeParams;
579 nodeParams.set("type", type);
580 nodeParams.set("path", node_db.path());
581
582 beast::xor_shift_engine rng(seedValue);
583
584 // Create a batch
585 auto batch = createPredictableBatch(numObjsToTest, rng());
586
587 {
588 // Open the database
590 Manager::instance().make_Database(megabytes(4), scheduler, 2, nodeParams, journal_);
591
592 // Write the batch
593 storeBatch(*db, batch);
594
595 {
596 // Read it back in
597 Batch copy;
598 fetchCopyOfBatch(*db, &copy, batch);
599 BEAST_EXPECT(areBatchesEqual(batch, copy));
600 }
601
602 {
603 // Reorder and read the copy again
604 std::shuffle(batch.begin(), batch.end(), rng);
605 Batch copy;
606 fetchCopyOfBatch(*db, &copy, batch);
607 BEAST_EXPECT(areBatchesEqual(batch, copy));
608 }
609 }
610
611 if (testPersistence)
612 {
613 // Re-open the database without the ephemeral DB
615 Manager::instance().make_Database(megabytes(4), scheduler, 2, nodeParams, journal_);
616
617 // Read it back in
618 Batch copy;
619 fetchCopyOfBatch(*db, &copy, batch);
620
621 // Canonicalize the source and destination batches
622 std::sort(batch.begin(), batch.end(), LessThan{});
623 std::sort(copy.begin(), copy.end(), LessThan{});
624 BEAST_EXPECT(areBatchesEqual(batch, copy));
625 }
626
627 if (type == "memory")
628 {
629 // Verify default earliest ledger sequence
630 {
632 Manager::instance().make_Database(megabytes(4), scheduler, 2, nodeParams, journal_);
633 BEAST_EXPECT(db->earliestLedgerSeq() == XRP_LEDGER_EARLIEST_SEQ);
634 }
635
636 // Set an invalid earliest ledger sequence
637 try
638 {
639 nodeParams.set("earliest_seq", "0");
641 Manager::instance().make_Database(megabytes(4), scheduler, 2, nodeParams, journal_);
642 }
643 catch (std::runtime_error const& e)
644 {
645 BEAST_EXPECT(std::strcmp(e.what(), "Invalid earliest_seq") == 0);
646 }
647
648 {
649 // Set a valid earliest ledger sequence
650 nodeParams.set("earliest_seq", "1");
652 Manager::instance().make_Database(megabytes(4), scheduler, 2, nodeParams, journal_);
653
654 // Verify database uses the earliest ledger sequence setting
655 BEAST_EXPECT(db->earliestLedgerSeq() == 1);
656 }
657
658 // Create another database that attempts to set the value again
659 try
660 {
661 // Set to default earliest ledger sequence
662 nodeParams.set("earliest_seq", std::to_string(XRP_LEDGER_EARLIEST_SEQ));
664 Manager::instance().make_Database(megabytes(4), scheduler, 2, nodeParams, journal_);
665 }
666 catch (std::runtime_error const& e)
667 {
668 BEAST_EXPECT(std::strcmp(e.what(), "earliest_seq set more than once") == 0);
669 }
670 }
671 }
672
673 //--------------------------------------------------------------------------
674
675 void
676 run() override
677 {
678 std::int64_t const seedValue = 50;
679
680 testConfig();
681
682 testNodeStore("memory", false, seedValue);
683
684 // Persistent backend tests
685 {
686 testNodeStore("nudb", true, seedValue);
687
688#if XRPL_ROCKSDB_AVAILABLE
689 testNodeStore("rocksdb", true, seedValue);
690#endif
691 }
692
693 // Import tests
694 {
695 testImport("nudb", "nudb", seedValue);
696
697#if XRPL_ROCKSDB_AVAILABLE
698 testImport("rocksdb", "rocksdb", seedValue);
699#endif
700
701#if XRPL_ENABLE_SQLITE_BACKEND_TESTS
702 testImport("sqlite", "sqlite", seedValue);
703#endif
704 }
705 }
706};
707
708BEAST_DEFINE_TESTSUITE(Database, nodestore, xrpl);
709
710} // namespace NodeStore
711} // namespace xrpl
RAII temporary directory.
Definition temp_dir.h:16
std::string path() const
Get the native path for the temporary directory.
Definition temp_dir.h:48
testcase_t testcase
Memberspace for declaring test cases.
Definition suite.h:148
void fail(String const &reason, char const *file, int line)
Record a failure.
Definition suite.h:517
virtual Config & config()=0
void run() override
Runs the suite.
void testImport(std::string const &destBackendType, std::string const &srcBackendType, std::int64_t seedValue)
void testNodeStore(std::string const &type, bool const testPersistence, std::int64_t const seedValue, int numObjsToTest=2000)
Persistency layer for NodeObject.
Definition Database.h:32
Simple NodeStore Scheduler that just performs the tasks synchronously.
static Manager & instance()
Returns the instance of the manager singleton.
virtual std::unique_ptr< Database > make_Database(std::size_t burstSize, Scheduler &scheduler, int readThreads, Section const &backendParameters, beast::Journal journal)=0
Construct a NodeStore database.
static bool areBatchesEqual(Batch const &lhs, Batch const &rhs)
Definition TestBase.h:97
void fetchCopyOfBatch(Backend &backend, Batch *pCopy, Batch const &batch)
Definition TestBase.h:132
static int const numObjectsToTest
Definition TestBase.h:53
void storeBatch(Backend &backend, Batch const &batch)
Definition TestBase.h:122
static Batch createPredictableBatch(int numObjects, std::uint64_t seed)
Definition TestBase.h:58
Holds a collection of configuration values.
Definition BasicConfig.h:25
void set(std::string const &key, std::string const &value)
Set a key/value pair.
A transaction testing environment.
Definition Env.h:98
Application & app()
Definition Env.h:230
T is_same_v
std::unique_ptr< Config > envconfig()
creates and initializes a default configuration for jtx::Env
Definition envconfig.h:35
Use hash_* containers for keys that do not need a cryptographically secure hashing algorithm.
Definition algorithm.h:6
constexpr auto megabytes(T value) noexcept
DatabaseCon::Setup setup_DatabaseCon(Config const &c, std::optional< beast::Journal > j=std::nullopt)
static constexpr std::uint32_t XRP_LEDGER_EARLIEST_SEQ
The XRP ledger network's earliest allowed sequence.
T shuffle(T... args)
T sort(T... args)
T strcmp(T... args)
static std::unique_ptr< std::vector< std::string > const > globalPragma
Definition DatabaseCon.h:89
Binary function that satisfies the strict-weak-ordering requirement.
Definition TestBase.h:28
T to_string(T... args)
T what(T... args)