rippled
Loading...
Searching...
No Matches
Database_test.cpp
1//------------------------------------------------------------------------------
2/*
3 This file is part of rippled: https://github.com/ripple/rippled
4 Copyright (c) 2012, 2013 Ripple Labs Inc.
5
6 Permission to use, copy, modify, and/or distribute this software for any
7 purpose with or without fee is hereby granted, provided that the above
8 copyright notice and this permission notice appear in all copies.
9
10 THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17*/
18//==============================================================================
19
20#include <test/jtx.h>
21#include <test/jtx/CheckMessageLogs.h>
22#include <test/jtx/envconfig.h>
23#include <test/nodestore/TestBase.h>
24#include <test/unit_test/SuiteJournal.h>
25#include <xrpld/core/DatabaseCon.h>
26#include <xrpld/nodestore/DummyScheduler.h>
27#include <xrpld/nodestore/Manager.h>
28#include <xrpl/beast/utility/temp_dir.h>
29
30namespace ripple {
31
32namespace NodeStore {
33
34class Database_test : public TestBase
35{
37
38public:
39 Database_test() : journal_("Database_test", *this)
40 {
41 }
42
43 void
45 {
46 testcase("Config");
47
48 using namespace ripple::test;
49 using namespace ripple::test::jtx;
50
51 auto const integrityWarning =
52 "reducing the data integrity guarantees from the "
53 "default [sqlite] behavior is not recommended for "
54 "nodes storing large amounts of history, because of the "
55 "difficulty inherent in rebuilding corrupted data.";
56 {
57 // defaults
58 Env env(*this);
59
60 auto const s = setup_DatabaseCon(env.app().config());
61
62 if (BEAST_EXPECT(s.globalPragma->size() == 3))
63 {
64 BEAST_EXPECT(
65 s.globalPragma->at(0) == "PRAGMA journal_mode=wal;");
66 BEAST_EXPECT(
67 s.globalPragma->at(1) == "PRAGMA synchronous=normal;");
68 BEAST_EXPECT(
69 s.globalPragma->at(2) == "PRAGMA temp_store=file;");
70 }
71 }
72 {
73 // High safety level
75
76 bool found = false;
77 Env env = [&]() {
78 auto p = test::jtx::envconfig();
79 {
80 auto& section = p->section("sqlite");
81 section.set("safety_level", "high");
82 }
83 p->LEDGER_HISTORY = 100'000'000;
84
85 return Env(
86 *this,
87 std::move(p),
88 std::make_unique<CheckMessageLogs>(
89 integrityWarning, &found),
91 }();
92
93 BEAST_EXPECT(!found);
94 auto const s = setup_DatabaseCon(env.app().config());
95 if (BEAST_EXPECT(s.globalPragma->size() == 3))
96 {
97 BEAST_EXPECT(
98 s.globalPragma->at(0) == "PRAGMA journal_mode=wal;");
99 BEAST_EXPECT(
100 s.globalPragma->at(1) == "PRAGMA synchronous=normal;");
101 BEAST_EXPECT(
102 s.globalPragma->at(2) == "PRAGMA temp_store=file;");
103 }
104 }
105 {
106 // Low safety level
108
109 bool found = false;
110 Env env = [&]() {
111 auto p = test::jtx::envconfig();
112 {
113 auto& section = p->section("sqlite");
114 section.set("safety_level", "low");
115 }
116 p->LEDGER_HISTORY = 100'000'000;
117
118 return Env(
119 *this,
120 std::move(p),
121 std::make_unique<CheckMessageLogs>(
122 integrityWarning, &found),
124 }();
125
126 BEAST_EXPECT(found);
127 auto const s = setup_DatabaseCon(env.app().config());
128 if (BEAST_EXPECT(s.globalPragma->size() == 3))
129 {
130 BEAST_EXPECT(
131 s.globalPragma->at(0) == "PRAGMA journal_mode=memory;");
132 BEAST_EXPECT(
133 s.globalPragma->at(1) == "PRAGMA synchronous=off;");
134 BEAST_EXPECT(
135 s.globalPragma->at(2) == "PRAGMA temp_store=memory;");
136 }
137 }
138 {
139 // Override individual settings
141
142 bool found = false;
143 Env env = [&]() {
144 auto p = test::jtx::envconfig();
145 {
146 auto& section = p->section("sqlite");
147 section.set("journal_mode", "off");
148 section.set("synchronous", "extra");
149 section.set("temp_store", "default");
150 }
151
152 return Env(
153 *this,
154 std::move(p),
155 std::make_unique<CheckMessageLogs>(
156 integrityWarning, &found),
158 }();
159
160 // No warning, even though higher risk settings were used because
161 // LEDGER_HISTORY is small
162 BEAST_EXPECT(!found);
163 auto const s = setup_DatabaseCon(env.app().config());
164 if (BEAST_EXPECT(s.globalPragma->size() == 3))
165 {
166 BEAST_EXPECT(
167 s.globalPragma->at(0) == "PRAGMA journal_mode=off;");
168 BEAST_EXPECT(
169 s.globalPragma->at(1) == "PRAGMA synchronous=extra;");
170 BEAST_EXPECT(
171 s.globalPragma->at(2) == "PRAGMA temp_store=default;");
172 }
173 }
174 {
175 // Override individual settings with large history
177
178 bool found = false;
179 Env env = [&]() {
180 auto p = test::jtx::envconfig();
181 {
182 auto& section = p->section("sqlite");
183 section.set("journal_mode", "off");
184 section.set("synchronous", "extra");
185 section.set("temp_store", "default");
186 }
187 p->LEDGER_HISTORY = 50'000'000;
188
189 return Env(
190 *this,
191 std::move(p),
192 std::make_unique<CheckMessageLogs>(
193 integrityWarning, &found),
195 }();
196
197 // No warning, even though higher risk settings were used because
198 // LEDGER_HISTORY is small
199 BEAST_EXPECT(found);
200 auto const s = setup_DatabaseCon(env.app().config());
201 if (BEAST_EXPECT(s.globalPragma->size() == 3))
202 {
203 BEAST_EXPECT(
204 s.globalPragma->at(0) == "PRAGMA journal_mode=off;");
205 BEAST_EXPECT(
206 s.globalPragma->at(1) == "PRAGMA synchronous=extra;");
207 BEAST_EXPECT(
208 s.globalPragma->at(2) == "PRAGMA temp_store=default;");
209 }
210 }
211 {
212 // Error: Mix safety_level and individual settings
214 auto const expected =
215 "Failed to initialize SQL databases: "
216 "Configuration file may not define both \"safety_level\" and "
217 "\"journal_mode\"";
218 bool found = false;
219
220 auto p = test::jtx::envconfig();
221 {
222 auto& section = p->section("sqlite");
223 section.set("safety_level", "low");
224 section.set("journal_mode", "off");
225 section.set("synchronous", "extra");
226 section.set("temp_store", "default");
227 }
228
229 try
230 {
231 Env env(
232 *this,
233 std::move(p),
234 std::make_unique<CheckMessageLogs>(expected, &found),
236 fail();
237 }
238 catch (...)
239 {
240 BEAST_EXPECT(found);
241 }
242 }
243 {
244 // Error: Mix safety_level and one setting (gotta catch 'em all)
246 auto const expected =
247 "Failed to initialize SQL databases: Configuration file may "
248 "not define both \"safety_level\" and \"journal_mode\"";
249 bool found = false;
250
251 auto p = test::jtx::envconfig();
252 {
253 auto& section = p->section("sqlite");
254 section.set("safety_level", "high");
255 section.set("journal_mode", "off");
256 }
257
258 try
259 {
260 Env env(
261 *this,
262 std::move(p),
263 std::make_unique<CheckMessageLogs>(expected, &found),
265 fail();
266 }
267 catch (...)
268 {
269 BEAST_EXPECT(found);
270 }
271 }
272 {
273 // Error: Mix safety_level and one setting (gotta catch 'em all)
275 auto const expected =
276 "Failed to initialize SQL databases: Configuration file may "
277 "not define both \"safety_level\" and \"synchronous\"";
278 bool found = false;
279
280 auto p = test::jtx::envconfig();
281 {
282 auto& section = p->section("sqlite");
283 section.set("safety_level", "low");
284 section.set("synchronous", "extra");
285 }
286
287 try
288 {
289 Env env(
290 *this,
291 std::move(p),
292 std::make_unique<CheckMessageLogs>(expected, &found),
294 fail();
295 }
296 catch (...)
297 {
298 BEAST_EXPECT(found);
299 }
300 }
301 {
302 // Error: Mix safety_level and one setting (gotta catch 'em all)
304 auto const expected =
305 "Failed to initialize SQL databases: Configuration file may "
306 "not define both \"safety_level\" and \"temp_store\"";
307 bool found = false;
308
309 auto p = test::jtx::envconfig();
310 {
311 auto& section = p->section("sqlite");
312 section.set("safety_level", "high");
313 section.set("temp_store", "default");
314 }
315
316 try
317 {
318 Env env(
319 *this,
320 std::move(p),
321 std::make_unique<CheckMessageLogs>(expected, &found),
323 fail();
324 }
325 catch (...)
326 {
327 BEAST_EXPECT(found);
328 }
329 }
330 {
331 // Error: Invalid value
333 auto const expected =
334 "Failed to initialize SQL databases: Invalid safety_level "
335 "value: slow";
336 bool found = false;
337
338 auto p = test::jtx::envconfig();
339 {
340 auto& section = p->section("sqlite");
341 section.set("safety_level", "slow");
342 }
343
344 try
345 {
346 Env env(
347 *this,
348 std::move(p),
349 std::make_unique<CheckMessageLogs>(expected, &found),
351 fail();
352 }
353 catch (...)
354 {
355 BEAST_EXPECT(found);
356 }
357 }
358 {
359 // Error: Invalid value
361 auto const expected =
362 "Failed to initialize SQL databases: Invalid journal_mode "
363 "value: fast";
364 bool found = false;
365
366 auto p = test::jtx::envconfig();
367 {
368 auto& section = p->section("sqlite");
369 section.set("journal_mode", "fast");
370 }
371
372 try
373 {
374 Env env(
375 *this,
376 std::move(p),
377 std::make_unique<CheckMessageLogs>(expected, &found),
379 fail();
380 }
381 catch (...)
382 {
383 BEAST_EXPECT(found);
384 }
385 }
386 {
387 // Error: Invalid value
389 auto const expected =
390 "Failed to initialize SQL databases: Invalid synchronous "
391 "value: instant";
392 bool found = false;
393
394 auto p = test::jtx::envconfig();
395 {
396 auto& section = p->section("sqlite");
397 section.set("synchronous", "instant");
398 }
399
400 try
401 {
402 Env env(
403 *this,
404 std::move(p),
405 std::make_unique<CheckMessageLogs>(expected, &found),
407 fail();
408 }
409 catch (...)
410 {
411 BEAST_EXPECT(found);
412 }
413 }
414 {
415 // Error: Invalid value
417 auto const expected =
418 "Failed to initialize SQL databases: Invalid temp_store "
419 "value: network";
420 bool found = false;
421
422 auto p = test::jtx::envconfig();
423 {
424 auto& section = p->section("sqlite");
425 section.set("temp_store", "network");
426 }
427
428 try
429 {
430 Env env(
431 *this,
432 std::move(p),
433 std::make_unique<CheckMessageLogs>(expected, &found),
435 fail();
436 }
437 catch (...)
438 {
439 BEAST_EXPECT(found);
440 }
441 }
442 {
443 // N/A: Default values
444 Env env(*this);
445 auto const s = setup_DatabaseCon(env.app().config());
446 if (BEAST_EXPECT(s.txPragma.size() == 4))
447 {
448 BEAST_EXPECT(s.txPragma.at(0) == "PRAGMA page_size=4096;");
449 BEAST_EXPECT(
450 s.txPragma.at(1) == "PRAGMA journal_size_limit=1582080;");
451 BEAST_EXPECT(
452 s.txPragma.at(2) == "PRAGMA max_page_count=4294967294;");
453 BEAST_EXPECT(
454 s.txPragma.at(3) == "PRAGMA mmap_size=17179869184;");
455 }
456 }
457 {
458 // Success: Valid values
459 Env env = [&]() {
460 auto p = test::jtx::envconfig();
461 {
462 auto& section = p->section("sqlite");
463 section.set("page_size", "512");
464 section.set("journal_size_limit", "2582080");
465 }
466 return Env(*this, std::move(p));
467 }();
468 auto const s = setup_DatabaseCon(env.app().config());
469 if (BEAST_EXPECT(s.txPragma.size() == 4))
470 {
471 BEAST_EXPECT(s.txPragma.at(0) == "PRAGMA page_size=512;");
472 BEAST_EXPECT(
473 s.txPragma.at(1) == "PRAGMA journal_size_limit=2582080;");
474 BEAST_EXPECT(
475 s.txPragma.at(2) == "PRAGMA max_page_count=4294967294;");
476 BEAST_EXPECT(
477 s.txPragma.at(3) == "PRAGMA mmap_size=17179869184;");
478 }
479 }
480 {
481 // Error: Invalid values
482 auto const expected =
483 "Invalid page_size. Must be between 512 and 65536.";
484 bool found = false;
485 auto p = test::jtx::envconfig();
486 {
487 auto& section = p->section("sqlite");
488 section.set("page_size", "256");
489 }
490 try
491 {
492 Env env(
493 *this,
494 std::move(p),
495 std::make_unique<CheckMessageLogs>(expected, &found),
497 fail();
498 }
499 catch (...)
500 {
501 BEAST_EXPECT(found);
502 }
503 }
504 {
505 // Error: Invalid values
506 auto const expected =
507 "Invalid page_size. Must be between 512 and 65536.";
508 bool found = false;
509 auto p = test::jtx::envconfig();
510 {
511 auto& section = p->section("sqlite");
512 section.set("page_size", "131072");
513 }
514 try
515 {
516 Env env(
517 *this,
518 std::move(p),
519 std::make_unique<CheckMessageLogs>(expected, &found),
521 fail();
522 }
523 catch (...)
524 {
525 BEAST_EXPECT(found);
526 }
527 }
528 {
529 // Error: Invalid values
530 auto const expected = "Invalid page_size. Must be a power of 2.";
531 bool found = false;
532 auto p = test::jtx::envconfig();
533 {
534 auto& section = p->section("sqlite");
535 section.set("page_size", "513");
536 }
537 try
538 {
539 Env env(
540 *this,
541 std::move(p),
542 std::make_unique<CheckMessageLogs>(expected, &found),
544 fail();
545 }
546 catch (...)
547 {
548 BEAST_EXPECT(found);
549 }
550 }
551 }
552
553 //--------------------------------------------------------------------------
554
555 void
557 std::string const& destBackendType,
558 std::string const& srcBackendType,
559 std::int64_t seedValue)
560 {
561 DummyScheduler scheduler;
562
563 beast::temp_dir node_db;
564 Section srcParams;
565 srcParams.set("type", srcBackendType);
566 srcParams.set("path", node_db.path());
567
568 // Create a batch
569 auto batch = createPredictableBatch(numObjectsToTest, seedValue);
570
571 // Write to source db
572 {
574 megabytes(4), scheduler, 2, srcParams, journal_);
575 storeBatch(*src, batch);
576 }
577
578 Batch copy;
579
580 {
581 // Re-open the db
583 megabytes(4), scheduler, 2, srcParams, journal_);
584
585 // Set up the destination database
586 beast::temp_dir dest_db;
587 Section destParams;
588 destParams.set("type", destBackendType);
589 destParams.set("path", dest_db.path());
590
592 megabytes(4), scheduler, 2, destParams, journal_);
593
594 testcase(
595 "import into '" + destBackendType + "' from '" +
596 srcBackendType + "'");
597
598 // Do the import
599 dest->importDatabase(*src);
600
601 // Get the results of the import
602 fetchCopyOfBatch(*dest, &copy, batch);
603 }
604
605 // Canonicalize the source and destination batches
606 std::sort(batch.begin(), batch.end(), LessThan{});
607 std::sort(copy.begin(), copy.end(), LessThan{});
608 BEAST_EXPECT(areBatchesEqual(batch, copy));
609 }
610
611 //--------------------------------------------------------------------------
612
613 void
615 std::string const& type,
616 bool const testPersistence,
617 std::int64_t const seedValue,
618 int numObjsToTest = 2000)
619 {
620 DummyScheduler scheduler;
621
622 std::string s = "NodeStore backend '" + type + "'";
623
624 testcase(s);
625
626 beast::temp_dir node_db;
627 Section nodeParams;
628 nodeParams.set("type", type);
629 nodeParams.set("path", node_db.path());
630
631 beast::xor_shift_engine rng(seedValue);
632
633 // Create a batch
634 auto batch = createPredictableBatch(numObjsToTest, rng());
635
636 {
637 // Open the database
639 megabytes(4), scheduler, 2, nodeParams, journal_);
640
641 // Write the batch
642 storeBatch(*db, batch);
643
644 {
645 // Read it back in
646 Batch copy;
647 fetchCopyOfBatch(*db, &copy, batch);
648 BEAST_EXPECT(areBatchesEqual(batch, copy));
649 }
650
651 {
652 // Reorder and read the copy again
653 std::shuffle(batch.begin(), batch.end(), rng);
654 Batch copy;
655 fetchCopyOfBatch(*db, &copy, batch);
656 BEAST_EXPECT(areBatchesEqual(batch, copy));
657 }
658 }
659
660 if (testPersistence)
661 {
662 // Re-open the database without the ephemeral DB
664 megabytes(4), scheduler, 2, nodeParams, journal_);
665
666 // Read it back in
667 Batch copy;
668 fetchCopyOfBatch(*db, &copy, batch);
669
670 // Canonicalize the source and destination batches
671 std::sort(batch.begin(), batch.end(), LessThan{});
672 std::sort(copy.begin(), copy.end(), LessThan{});
673 BEAST_EXPECT(areBatchesEqual(batch, copy));
674 }
675
676 if (type == "memory")
677 {
678 // Verify default earliest ledger sequence
679 {
682 megabytes(4), scheduler, 2, nodeParams, journal_);
683 BEAST_EXPECT(
684 db->earliestLedgerSeq() == XRP_LEDGER_EARLIEST_SEQ);
685 }
686
687 // Set an invalid earliest ledger sequence
688 try
689 {
690 nodeParams.set("earliest_seq", "0");
693 megabytes(4), scheduler, 2, nodeParams, journal_);
694 }
695 catch (std::runtime_error const& e)
696 {
697 BEAST_EXPECT(
698 std::strcmp(e.what(), "Invalid earliest_seq") == 0);
699 }
700
701 {
702 // Set a valid earliest ledger sequence
703 nodeParams.set("earliest_seq", "1");
706 megabytes(4), scheduler, 2, nodeParams, journal_);
707
708 // Verify database uses the earliest ledger sequence setting
709 BEAST_EXPECT(db->earliestLedgerSeq() == 1);
710 }
711
712 // Create another database that attempts to set the value again
713 try
714 {
715 // Set to default earliest ledger sequence
716 nodeParams.set(
717 "earliest_seq", std::to_string(XRP_LEDGER_EARLIEST_SEQ));
720 megabytes(4), scheduler, 2, nodeParams, journal_);
721 }
722 catch (std::runtime_error const& e)
723 {
724 BEAST_EXPECT(
725 std::strcmp(e.what(), "earliest_seq set more than once") ==
726 0);
727 }
728 }
729 }
730
731 //--------------------------------------------------------------------------
732
733 void
734 run() override
735 {
736 std::int64_t const seedValue = 50;
737
738 testConfig();
739
740 testNodeStore("memory", false, seedValue);
741
742 // Persistent backend tests
743 {
744 testNodeStore("nudb", true, seedValue);
745
746#if RIPPLE_ROCKSDB_AVAILABLE
747 testNodeStore("rocksdb", true, seedValue);
748#endif
749 }
750
751 // Import tests
752 {
753 testImport("nudb", "nudb", seedValue);
754
755#if RIPPLE_ROCKSDB_AVAILABLE
756 testImport("rocksdb", "rocksdb", seedValue);
757#endif
758
759#if RIPPLE_ENABLE_SQLITE_BACKEND_TESTS
760 testImport("sqlite", "sqlite", seedValue);
761#endif
762 }
763 }
764};
765
766BEAST_DEFINE_TESTSUITE(Database, NodeStore, ripple);
767
768} // namespace NodeStore
769} // namespace ripple
RAII temporary directory.
Definition: temp_dir.h:35
std::string path() const
Get the native path for the temporary directory.
Definition: temp_dir.h:67
testcase_t testcase
Memberspace for declaring test cases.
Definition: suite.h:155
void fail(String const &reason, char const *file, int line)
Record a failure.
Definition: suite.h:533
virtual Config & config()=0
void testImport(std::string const &destBackendType, std::string const &srcBackendType, std::int64_t seedValue)
void testNodeStore(std::string const &type, bool const testPersistence, std::int64_t const seedValue, int numObjsToTest=2000)
void run() override
Runs the suite.
Persistency layer for NodeObject.
Definition: Database.h:50
Simple NodeStore Scheduler that just peforms the tasks synchronously.
virtual std::unique_ptr< Database > make_Database(std::size_t burstSize, Scheduler &scheduler, int readThreads, Section const &backendParameters, beast::Journal journal)=0
Construct a NodeStore database.
static Manager & instance()
Returns the instance of the manager singleton.
Definition: ManagerImp.cpp:116
static int const numObjectsToTest
Definition: TestBase.h:77
static Batch createPredictableBatch(int numObjects, std::uint64_t seed)
Definition: TestBase.h:82
static bool areBatchesEqual(Batch const &lhs, Batch const &rhs)
Definition: TestBase.h:122
void fetchCopyOfBatch(Backend &backend, Batch *pCopy, Batch const &batch)
Definition: TestBase.h:157
void storeBatch(Backend &backend, Batch const &batch)
Definition: TestBase.h:147
Holds a collection of configuration values.
Definition: BasicConfig.h:45
void set(std::string const &key, std::string const &value)
Set a key/value pair.
Definition: BasicConfig.cpp:41
A transaction testing environment.
Definition: Env.h:118
Application & app()
Definition: Env.h:256
std::unique_ptr< Config > envconfig()
creates and initializes a default configuration for jtx::Env
Definition: envconfig.h:54
Use hash_* containers for keys that do not need a cryptographically secure hashing algorithm.
Definition: algorithm.h:26
constexpr auto megabytes(T value) noexcept
Definition: ByteUtilities.h:34
DatabaseCon::Setup setup_DatabaseCon(Config const &c, std::optional< beast::Journal > j=std::nullopt)
static constexpr std::uint32_t XRP_LEDGER_EARLIEST_SEQ
The XRP ledger network's earliest allowed sequence.
T shuffle(T... args)
T sort(T... args)
T strcmp(T... args)
static std::unique_ptr< std::vector< std::string > const > globalPragma
Definition: DatabaseCon.h:110
Binary function that satisfies the strict-weak-ordering requirement.
Definition: TestBase.h:47
T to_string(T... args)
T what(T... args)