rippled
Loading...
Searching...
No Matches
Database_test.cpp
1//------------------------------------------------------------------------------
2/*
3 This file is part of rippled: https://github.com/ripple/rippled
4 Copyright (c) 2012, 2013 Ripple Labs Inc.
5
6 Permission to use, copy, modify, and/or distribute this software for any
7 purpose with or without fee is hereby granted, provided that the above
8 copyright notice and this permission notice appear in all copies.
9
10 THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17*/
18//==============================================================================
19
20#include <test/jtx.h>
21#include <test/jtx/CheckMessageLogs.h>
22#include <test/jtx/envconfig.h>
23#include <test/nodestore/TestBase.h>
24#include <test/unit_test/SuiteJournal.h>
25
26#include <xrpld/core/DatabaseCon.h>
27#include <xrpld/nodestore/DummyScheduler.h>
28#include <xrpld/nodestore/Manager.h>
29
30#include <xrpl/beast/utility/temp_dir.h>
31
32namespace ripple {
33
34namespace NodeStore {
35
36class Database_test : public TestBase
37{
39
40public:
41 Database_test() : journal_("Database_test", *this)
42 {
43 }
44
45 void
47 {
48 testcase("Config");
49
50 using namespace ripple::test;
51 using namespace ripple::test::jtx;
52
53 auto const integrityWarning =
54 "reducing the data integrity guarantees from the "
55 "default [sqlite] behavior is not recommended for "
56 "nodes storing large amounts of history, because of the "
57 "difficulty inherent in rebuilding corrupted data.";
58 {
59 // defaults
60 Env env(*this);
61
62 auto const s = setup_DatabaseCon(env.app().config());
63
64 if (BEAST_EXPECT(s.globalPragma->size() == 3))
65 {
66 BEAST_EXPECT(
67 s.globalPragma->at(0) == "PRAGMA journal_mode=wal;");
68 BEAST_EXPECT(
69 s.globalPragma->at(1) == "PRAGMA synchronous=normal;");
70 BEAST_EXPECT(
71 s.globalPragma->at(2) == "PRAGMA temp_store=file;");
72 }
73 }
74 {
75 // High safety level
77
78 bool found = false;
79 Env env = [&]() {
80 auto p = test::jtx::envconfig();
81 {
82 auto& section = p->section("sqlite");
83 section.set("safety_level", "high");
84 }
85 p->LEDGER_HISTORY = 100'000'000;
86
87 return Env(
88 *this,
89 std::move(p),
90 std::make_unique<CheckMessageLogs>(
91 integrityWarning, &found),
93 }();
94
95 BEAST_EXPECT(!found);
96 auto const s = setup_DatabaseCon(env.app().config());
97 if (BEAST_EXPECT(s.globalPragma->size() == 3))
98 {
99 BEAST_EXPECT(
100 s.globalPragma->at(0) == "PRAGMA journal_mode=wal;");
101 BEAST_EXPECT(
102 s.globalPragma->at(1) == "PRAGMA synchronous=normal;");
103 BEAST_EXPECT(
104 s.globalPragma->at(2) == "PRAGMA temp_store=file;");
105 }
106 }
107 {
108 // Low safety level
110
111 bool found = false;
112 Env env = [&]() {
113 auto p = test::jtx::envconfig();
114 {
115 auto& section = p->section("sqlite");
116 section.set("safety_level", "low");
117 }
118 p->LEDGER_HISTORY = 100'000'000;
119
120 return Env(
121 *this,
122 std::move(p),
123 std::make_unique<CheckMessageLogs>(
124 integrityWarning, &found),
126 }();
127
128 BEAST_EXPECT(found);
129 auto const s = setup_DatabaseCon(env.app().config());
130 if (BEAST_EXPECT(s.globalPragma->size() == 3))
131 {
132 BEAST_EXPECT(
133 s.globalPragma->at(0) == "PRAGMA journal_mode=memory;");
134 BEAST_EXPECT(
135 s.globalPragma->at(1) == "PRAGMA synchronous=off;");
136 BEAST_EXPECT(
137 s.globalPragma->at(2) == "PRAGMA temp_store=memory;");
138 }
139 }
140 {
141 // Override individual settings
143
144 bool found = false;
145 Env env = [&]() {
146 auto p = test::jtx::envconfig();
147 {
148 auto& section = p->section("sqlite");
149 section.set("journal_mode", "off");
150 section.set("synchronous", "extra");
151 section.set("temp_store", "default");
152 }
153
154 return Env(
155 *this,
156 std::move(p),
157 std::make_unique<CheckMessageLogs>(
158 integrityWarning, &found),
160 }();
161
162 // No warning, even though higher risk settings were used because
163 // LEDGER_HISTORY is small
164 BEAST_EXPECT(!found);
165 auto const s = setup_DatabaseCon(env.app().config());
166 if (BEAST_EXPECT(s.globalPragma->size() == 3))
167 {
168 BEAST_EXPECT(
169 s.globalPragma->at(0) == "PRAGMA journal_mode=off;");
170 BEAST_EXPECT(
171 s.globalPragma->at(1) == "PRAGMA synchronous=extra;");
172 BEAST_EXPECT(
173 s.globalPragma->at(2) == "PRAGMA temp_store=default;");
174 }
175 }
176 {
177 // Override individual settings with large history
179
180 bool found = false;
181 Env env = [&]() {
182 auto p = test::jtx::envconfig();
183 {
184 auto& section = p->section("sqlite");
185 section.set("journal_mode", "off");
186 section.set("synchronous", "extra");
187 section.set("temp_store", "default");
188 }
189 p->LEDGER_HISTORY = 50'000'000;
190
191 return Env(
192 *this,
193 std::move(p),
194 std::make_unique<CheckMessageLogs>(
195 integrityWarning, &found),
197 }();
198
199 // No warning, even though higher risk settings were used because
200 // LEDGER_HISTORY is small
201 BEAST_EXPECT(found);
202 auto const s = setup_DatabaseCon(env.app().config());
203 if (BEAST_EXPECT(s.globalPragma->size() == 3))
204 {
205 BEAST_EXPECT(
206 s.globalPragma->at(0) == "PRAGMA journal_mode=off;");
207 BEAST_EXPECT(
208 s.globalPragma->at(1) == "PRAGMA synchronous=extra;");
209 BEAST_EXPECT(
210 s.globalPragma->at(2) == "PRAGMA temp_store=default;");
211 }
212 }
213 {
214 // Error: Mix safety_level and individual settings
216 auto const expected =
217 "Failed to initialize SQL databases: "
218 "Configuration file may not define both \"safety_level\" and "
219 "\"journal_mode\"";
220 bool found = false;
221
222 auto p = test::jtx::envconfig();
223 {
224 auto& section = p->section("sqlite");
225 section.set("safety_level", "low");
226 section.set("journal_mode", "off");
227 section.set("synchronous", "extra");
228 section.set("temp_store", "default");
229 }
230
231 try
232 {
233 Env env(
234 *this,
235 std::move(p),
236 std::make_unique<CheckMessageLogs>(expected, &found),
238 fail();
239 }
240 catch (...)
241 {
242 BEAST_EXPECT(found);
243 }
244 }
245 {
246 // Error: Mix safety_level and one setting (gotta catch 'em all)
248 auto const expected =
249 "Failed to initialize SQL databases: Configuration file may "
250 "not define both \"safety_level\" and \"journal_mode\"";
251 bool found = false;
252
253 auto p = test::jtx::envconfig();
254 {
255 auto& section = p->section("sqlite");
256 section.set("safety_level", "high");
257 section.set("journal_mode", "off");
258 }
259
260 try
261 {
262 Env env(
263 *this,
264 std::move(p),
265 std::make_unique<CheckMessageLogs>(expected, &found),
267 fail();
268 }
269 catch (...)
270 {
271 BEAST_EXPECT(found);
272 }
273 }
274 {
275 // Error: Mix safety_level and one setting (gotta catch 'em all)
277 auto const expected =
278 "Failed to initialize SQL databases: Configuration file may "
279 "not define both \"safety_level\" and \"synchronous\"";
280 bool found = false;
281
282 auto p = test::jtx::envconfig();
283 {
284 auto& section = p->section("sqlite");
285 section.set("safety_level", "low");
286 section.set("synchronous", "extra");
287 }
288
289 try
290 {
291 Env env(
292 *this,
293 std::move(p),
294 std::make_unique<CheckMessageLogs>(expected, &found),
296 fail();
297 }
298 catch (...)
299 {
300 BEAST_EXPECT(found);
301 }
302 }
303 {
304 // Error: Mix safety_level and one setting (gotta catch 'em all)
306 auto const expected =
307 "Failed to initialize SQL databases: Configuration file may "
308 "not define both \"safety_level\" and \"temp_store\"";
309 bool found = false;
310
311 auto p = test::jtx::envconfig();
312 {
313 auto& section = p->section("sqlite");
314 section.set("safety_level", "high");
315 section.set("temp_store", "default");
316 }
317
318 try
319 {
320 Env env(
321 *this,
322 std::move(p),
323 std::make_unique<CheckMessageLogs>(expected, &found),
325 fail();
326 }
327 catch (...)
328 {
329 BEAST_EXPECT(found);
330 }
331 }
332 {
333 // Error: Invalid value
335 auto const expected =
336 "Failed to initialize SQL databases: Invalid safety_level "
337 "value: slow";
338 bool found = false;
339
340 auto p = test::jtx::envconfig();
341 {
342 auto& section = p->section("sqlite");
343 section.set("safety_level", "slow");
344 }
345
346 try
347 {
348 Env env(
349 *this,
350 std::move(p),
351 std::make_unique<CheckMessageLogs>(expected, &found),
353 fail();
354 }
355 catch (...)
356 {
357 BEAST_EXPECT(found);
358 }
359 }
360 {
361 // Error: Invalid value
363 auto const expected =
364 "Failed to initialize SQL databases: Invalid journal_mode "
365 "value: fast";
366 bool found = false;
367
368 auto p = test::jtx::envconfig();
369 {
370 auto& section = p->section("sqlite");
371 section.set("journal_mode", "fast");
372 }
373
374 try
375 {
376 Env env(
377 *this,
378 std::move(p),
379 std::make_unique<CheckMessageLogs>(expected, &found),
381 fail();
382 }
383 catch (...)
384 {
385 BEAST_EXPECT(found);
386 }
387 }
388 {
389 // Error: Invalid value
391 auto const expected =
392 "Failed to initialize SQL databases: Invalid synchronous "
393 "value: instant";
394 bool found = false;
395
396 auto p = test::jtx::envconfig();
397 {
398 auto& section = p->section("sqlite");
399 section.set("synchronous", "instant");
400 }
401
402 try
403 {
404 Env env(
405 *this,
406 std::move(p),
407 std::make_unique<CheckMessageLogs>(expected, &found),
409 fail();
410 }
411 catch (...)
412 {
413 BEAST_EXPECT(found);
414 }
415 }
416 {
417 // Error: Invalid value
419 auto const expected =
420 "Failed to initialize SQL databases: Invalid temp_store "
421 "value: network";
422 bool found = false;
423
424 auto p = test::jtx::envconfig();
425 {
426 auto& section = p->section("sqlite");
427 section.set("temp_store", "network");
428 }
429
430 try
431 {
432 Env env(
433 *this,
434 std::move(p),
435 std::make_unique<CheckMessageLogs>(expected, &found),
437 fail();
438 }
439 catch (...)
440 {
441 BEAST_EXPECT(found);
442 }
443 }
444 {
445 // N/A: Default values
446 Env env(*this);
447 auto const s = setup_DatabaseCon(env.app().config());
448 if (BEAST_EXPECT(s.txPragma.size() == 4))
449 {
450 BEAST_EXPECT(s.txPragma.at(0) == "PRAGMA page_size=4096;");
451 BEAST_EXPECT(
452 s.txPragma.at(1) == "PRAGMA journal_size_limit=1582080;");
453 BEAST_EXPECT(
454 s.txPragma.at(2) == "PRAGMA max_page_count=4294967294;");
455 BEAST_EXPECT(
456 s.txPragma.at(3) == "PRAGMA mmap_size=17179869184;");
457 }
458 }
459 {
460 // Success: Valid values
461 Env env = [&]() {
462 auto p = test::jtx::envconfig();
463 {
464 auto& section = p->section("sqlite");
465 section.set("page_size", "512");
466 section.set("journal_size_limit", "2582080");
467 }
468 return Env(*this, std::move(p));
469 }();
470 auto const s = setup_DatabaseCon(env.app().config());
471 if (BEAST_EXPECT(s.txPragma.size() == 4))
472 {
473 BEAST_EXPECT(s.txPragma.at(0) == "PRAGMA page_size=512;");
474 BEAST_EXPECT(
475 s.txPragma.at(1) == "PRAGMA journal_size_limit=2582080;");
476 BEAST_EXPECT(
477 s.txPragma.at(2) == "PRAGMA max_page_count=4294967294;");
478 BEAST_EXPECT(
479 s.txPragma.at(3) == "PRAGMA mmap_size=17179869184;");
480 }
481 }
482 {
483 // Error: Invalid values
484 auto const expected =
485 "Invalid page_size. Must be between 512 and 65536.";
486 bool found = false;
487 auto p = test::jtx::envconfig();
488 {
489 auto& section = p->section("sqlite");
490 section.set("page_size", "256");
491 }
492 try
493 {
494 Env env(
495 *this,
496 std::move(p),
497 std::make_unique<CheckMessageLogs>(expected, &found),
499 fail();
500 }
501 catch (...)
502 {
503 BEAST_EXPECT(found);
504 }
505 }
506 {
507 // Error: Invalid values
508 auto const expected =
509 "Invalid page_size. Must be between 512 and 65536.";
510 bool found = false;
511 auto p = test::jtx::envconfig();
512 {
513 auto& section = p->section("sqlite");
514 section.set("page_size", "131072");
515 }
516 try
517 {
518 Env env(
519 *this,
520 std::move(p),
521 std::make_unique<CheckMessageLogs>(expected, &found),
523 fail();
524 }
525 catch (...)
526 {
527 BEAST_EXPECT(found);
528 }
529 }
530 {
531 // Error: Invalid values
532 auto const expected = "Invalid page_size. Must be a power of 2.";
533 bool found = false;
534 auto p = test::jtx::envconfig();
535 {
536 auto& section = p->section("sqlite");
537 section.set("page_size", "513");
538 }
539 try
540 {
541 Env env(
542 *this,
543 std::move(p),
544 std::make_unique<CheckMessageLogs>(expected, &found),
546 fail();
547 }
548 catch (...)
549 {
550 BEAST_EXPECT(found);
551 }
552 }
553 }
554
555 //--------------------------------------------------------------------------
556
557 void
559 std::string const& destBackendType,
560 std::string const& srcBackendType,
561 std::int64_t seedValue)
562 {
563 DummyScheduler scheduler;
564
565 beast::temp_dir node_db;
566 Section srcParams;
567 srcParams.set("type", srcBackendType);
568 srcParams.set("path", node_db.path());
569
570 // Create a batch
572
573 // Write to source db
574 {
576 megabytes(4), scheduler, 2, srcParams, journal_);
577 storeBatch(*src, batch);
578 }
579
580 Batch copy;
581
582 {
583 // Re-open the db
585 megabytes(4), scheduler, 2, srcParams, journal_);
586
587 // Set up the destination database
588 beast::temp_dir dest_db;
589 Section destParams;
590 destParams.set("type", destBackendType);
591 destParams.set("path", dest_db.path());
592
594 megabytes(4), scheduler, 2, destParams, journal_);
595
596 testcase(
597 "import into '" + destBackendType + "' from '" +
598 srcBackendType + "'");
599
600 // Do the import
601 dest->importDatabase(*src);
602
603 // Get the results of the import
604 fetchCopyOfBatch(*dest, &copy, batch);
605 }
606
607 // Canonicalize the source and destination batches
608 std::sort(batch.begin(), batch.end(), LessThan{});
609 std::sort(copy.begin(), copy.end(), LessThan{});
610 BEAST_EXPECT(areBatchesEqual(batch, copy));
611 }
612
613 //--------------------------------------------------------------------------
614
615 void
617 std::string const& type,
618 bool const testPersistence,
619 std::int64_t const seedValue,
620 int numObjsToTest = 2000)
621 {
622 DummyScheduler scheduler;
623
624 std::string s = "NodeStore backend '" + type + "'";
625
626 testcase(s);
627
628 beast::temp_dir node_db;
629 Section nodeParams;
630 nodeParams.set("type", type);
631 nodeParams.set("path", node_db.path());
632
633 beast::xor_shift_engine rng(seedValue);
634
635 // Create a batch
636 auto batch = createPredictableBatch(numObjsToTest, rng());
637
638 {
639 // Open the database
641 megabytes(4), scheduler, 2, nodeParams, journal_);
642
643 // Write the batch
644 storeBatch(*db, batch);
645
646 {
647 // Read it back in
648 Batch copy;
649 fetchCopyOfBatch(*db, &copy, batch);
650 BEAST_EXPECT(areBatchesEqual(batch, copy));
651 }
652
653 {
654 // Reorder and read the copy again
655 std::shuffle(batch.begin(), batch.end(), rng);
656 Batch copy;
657 fetchCopyOfBatch(*db, &copy, batch);
658 BEAST_EXPECT(areBatchesEqual(batch, copy));
659 }
660 }
661
662 if (testPersistence)
663 {
664 // Re-open the database without the ephemeral DB
666 megabytes(4), scheduler, 2, nodeParams, journal_);
667
668 // Read it back in
669 Batch copy;
670 fetchCopyOfBatch(*db, &copy, batch);
671
672 // Canonicalize the source and destination batches
673 std::sort(batch.begin(), batch.end(), LessThan{});
674 std::sort(copy.begin(), copy.end(), LessThan{});
675 BEAST_EXPECT(areBatchesEqual(batch, copy));
676 }
677
678 if (type == "memory")
679 {
680 // Verify default earliest ledger sequence
681 {
684 megabytes(4), scheduler, 2, nodeParams, journal_);
685 BEAST_EXPECT(
686 db->earliestLedgerSeq() == XRP_LEDGER_EARLIEST_SEQ);
687 }
688
689 // Set an invalid earliest ledger sequence
690 try
691 {
692 nodeParams.set("earliest_seq", "0");
695 megabytes(4), scheduler, 2, nodeParams, journal_);
696 }
697 catch (std::runtime_error const& e)
698 {
699 BEAST_EXPECT(
700 std::strcmp(e.what(), "Invalid earliest_seq") == 0);
701 }
702
703 {
704 // Set a valid earliest ledger sequence
705 nodeParams.set("earliest_seq", "1");
708 megabytes(4), scheduler, 2, nodeParams, journal_);
709
710 // Verify database uses the earliest ledger sequence setting
711 BEAST_EXPECT(db->earliestLedgerSeq() == 1);
712 }
713
714 // Create another database that attempts to set the value again
715 try
716 {
717 // Set to default earliest ledger sequence
718 nodeParams.set(
719 "earliest_seq", std::to_string(XRP_LEDGER_EARLIEST_SEQ));
722 megabytes(4), scheduler, 2, nodeParams, journal_);
723 }
724 catch (std::runtime_error const& e)
725 {
726 BEAST_EXPECT(
727 std::strcmp(e.what(), "earliest_seq set more than once") ==
728 0);
729 }
730 }
731 }
732
733 //--------------------------------------------------------------------------
734
735 void
736 run() override
737 {
738 std::int64_t const seedValue = 50;
739
740 testConfig();
741
742 testNodeStore("memory", false, seedValue);
743
744 // Persistent backend tests
745 {
746 testNodeStore("nudb", true, seedValue);
747
748#if RIPPLE_ROCKSDB_AVAILABLE
749 testNodeStore("rocksdb", true, seedValue);
750#endif
751 }
752
753 // Import tests
754 {
755 testImport("nudb", "nudb", seedValue);
756
757#if RIPPLE_ROCKSDB_AVAILABLE
758 testImport("rocksdb", "rocksdb", seedValue);
759#endif
760
761#if RIPPLE_ENABLE_SQLITE_BACKEND_TESTS
762 testImport("sqlite", "sqlite", seedValue);
763#endif
764 }
765 }
766};
767
768BEAST_DEFINE_TESTSUITE(Database, NodeStore, ripple);
769
770} // namespace NodeStore
771} // namespace ripple
RAII temporary directory.
Definition: temp_dir.h:35
std::string path() const
Get the native path for the temporary directory.
Definition: temp_dir.h:67
testcase_t testcase
Memberspace for declaring test cases.
Definition: suite.h:155
void fail(String const &reason, char const *file, int line)
Record a failure.
Definition: suite.h:533
virtual Config & config()=0
void testImport(std::string const &destBackendType, std::string const &srcBackendType, std::int64_t seedValue)
void testNodeStore(std::string const &type, bool const testPersistence, std::int64_t const seedValue, int numObjsToTest=2000)
void run() override
Runs the suite.
Persistency layer for NodeObject.
Definition: Database.h:52
Simple NodeStore Scheduler that just peforms the tasks synchronously.
virtual std::unique_ptr< Database > make_Database(std::size_t burstSize, Scheduler &scheduler, int readThreads, Section const &backendParameters, beast::Journal journal)=0
Construct a NodeStore database.
static Manager & instance()
Returns the instance of the manager singleton.
Definition: ManagerImp.cpp:116
static int const numObjectsToTest
Definition: TestBase.h:78
static Batch createPredictableBatch(int numObjects, std::uint64_t seed)
Definition: TestBase.h:83
static bool areBatchesEqual(Batch const &lhs, Batch const &rhs)
Definition: TestBase.h:123
void fetchCopyOfBatch(Backend &backend, Batch *pCopy, Batch const &batch)
Definition: TestBase.h:158
void storeBatch(Backend &backend, Batch const &batch)
Definition: TestBase.h:148
Holds a collection of configuration values.
Definition: BasicConfig.h:45
void set(std::string const &key, std::string const &value)
Set a key/value pair.
Definition: BasicConfig.cpp:41
A transaction testing environment.
Definition: Env.h:121
Application & app()
Definition: Env.h:261
std::unique_ptr< Config > envconfig()
creates and initializes a default configuration for jtx::Env
Definition: envconfig.h:54
Use hash_* containers for keys that do not need a cryptographically secure hashing algorithm.
Definition: algorithm.h:26
constexpr auto megabytes(T value) noexcept
Definition: ByteUtilities.h:34
DatabaseCon::Setup setup_DatabaseCon(Config const &c, std::optional< beast::Journal > j=std::nullopt)
static constexpr std::uint32_t XRP_LEDGER_EARLIEST_SEQ
The XRP ledger network's earliest allowed sequence.
T shuffle(T... args)
T sort(T... args)
T strcmp(T... args)
static std::unique_ptr< std::vector< std::string > const > globalPragma
Definition: DatabaseCon.h:110
Binary function that satisfies the strict-weak-ordering requirement.
Definition: TestBase.h:48
T to_string(T... args)
T what(T... args)