ARCHIVE/rippled
1
0
Fork 0
mirror of https://github.com/XRPLF/rippled.git synced 2025-11-21 03:26:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improve shards file exception handling

Browse Source
This commit is contained in:
Miguel Portilla
2018-07-16 12:54:52 -04:00
committed by seelabs
parent a73372cb9d
commit a7ed5bfbee
4 changed files with 222 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
src/ripple/nodestore/impl/DatabaseShardImp.cpp
View File

@@ -92,6 +92,8 @@ DatabaseShardImp::init()
return true;
}
try
{
// Find shards
for (auto const& d : directory_iterator(dir_))
{
@@ -103,7 +105,7 @@ DatabaseShardImp::init()
if (!std::all_of(
dirName.begin(),
dirName.end(),
[](auto c){
[](auto c) {
return ::isdigit(static_cast<unsigned char>(c));
}))
{
@@ -120,21 +122,21 @@ DatabaseShardImp::init()
}
// Check if a previous import failed
if (is_regular_file(dir_ / std::to_string(shardIndex) /
importMarker_))
if (is_regular_file(
dir_ / std::to_string(shardIndex) / importMarker_))
{
JLOG(j_.warn()) <<
"shard " << shardIndex <<
" previously failed import, removing";
if (!this->remove(dir_ / std::to_string(shardIndex)))
return false;
remove_all(dir_ / std::to_string(shardIndex));
continue;
}
auto shard = std::make_unique<Shard>(
*this, shardIndex, cacheSz_, cacheAge_, j_);
auto shard {std::make_unique<Shard>(
*this, shardIndex, cacheSz_, cacheAge_, j_)};
if (!shard->open(config_, scheduler_))
return false;
usedDiskSpace_ += shard->fileSize();
if (shard->complete())
complete_.emplace(shard->index(), std::move(shard));
@@ -149,10 +151,18 @@ DatabaseShardImp::init()
incomplete_ = std::move(shard);
}
}
}
catch (std::exception const& e)
{
JLOG(j_.error()) <<
"exception: " << e.what();
return false;
}
if (!incomplete_ && complete_.empty())
{
// New Shard Store, calculate file descriptor requirements
if (maxDiskSpace_ > space(dir_).free)
if (maxDiskSpace_ > available())
{
JLOG(j_.error()) <<
"Insufficient disk space";
@@ -185,7 +195,7 @@ DatabaseShardImp::prepareLedger(std::uint32_t validLedgerSeq)
canAdd_ = false;
return boost::none;
}
if (avgShardSz_ > boost::filesystem::space(dir_).free)
if (avgShardSz_ > available())
{
JLOG(j_.error()) <<
"Insufficient disk space";
@@ -211,9 +221,9 @@ DatabaseShardImp::prepareLedger(std::uint32_t validLedgerSeq)
if (!incomplete_->open(config_, scheduler_))
{
incomplete_.reset();
this->remove(dir_ / std::to_string(*shardIndex));
return boost::none;
}
return incomplete_->prepare();
}
@@ -254,6 +264,7 @@ DatabaseShardImp::prepareShard(std::uint32_t shardIndex)
{
return false;
}
if (complete_.find(shardIndex) != complete_.end())
{
JLOG(j_.debug()) <<
@@ -287,7 +298,7 @@ DatabaseShardImp::prepareShard(std::uint32_t shardIndex)
"Exceeds maximum size";
return false;
}
if (sz > space(dir_).free)
if (sz > available())
{
JLOG(j_.error()) <<
"Insufficient disk space";
@@ -321,12 +332,21 @@ DatabaseShardImp::importShard(std::uint32_t shardIndex,
boost::filesystem::path const& srcDir, bool validate)
{
using namespace boost::filesystem;
try
{
if (!is_directory(srcDir) || is_empty(srcDir))
{
JLOG(j_.error()) <<
"Invalid source directory " << srcDir.string();
return false;
}
}
catch (std::exception const& e)
{
JLOG(j_.error()) <<
"exception: " << e.what();
return false;
}
auto move = [&](path const& src, path const& dst)
{
@@ -334,12 +354,10 @@ DatabaseShardImp::importShard(std::uint32_t shardIndex,
{
rename(src, dst);
}
catch (const filesystem_error& e)
catch (std::exception const& e)
{
JLOG(j_.error()) <<
"rename " << src.string() <<
" to " << dst.string() <<
": Exception, " << e.code().message();
"exception: " << e.what();
return false;
}
return true;
@@ -366,28 +384,26 @@ DatabaseShardImp::importShard(std::uint32_t shardIndex,
auto shard {std::make_unique<Shard>(
*this, shardIndex, cacheSz_, cacheAge_, j_)};
auto fail = [&](std::string msg)
{
if (!msg.empty())
{
JLOG(j_.error()) << msg;
}
shard.release();
move(dstDir, srcDir);
return false;
};
if (!shard->open(config_, scheduler_))
return fail({});
return fail("Failure");
if (!shard->complete())
return fail("Incomplete shard");
// Verify database integrity
try
{
// Verify database integrity
shard->getBackend()->verify();
}
catch (std::exception const& e)
{
return fail(std::string("Verify: Exception, ") + e.what());
return fail(std::string("exception: ") + e.what());
}
// Validate shard ledgers
@@ -397,14 +413,14 @@ DatabaseShardImp::importShard(std::uint32_t shardIndex,
// so the database can fetch data from it
it->second = shard.get();
l.unlock();
auto valid {shard->validate(app_)};
auto const valid {shard->validate(app_)};
l.lock();
if (!valid)
{
it = preShards_.find(shardIndex);
if(it != preShards_.end())
it->second = nullptr;
return fail({});
return fail("failed validation");
}
}
@@ -634,7 +650,7 @@ DatabaseShardImp::import(Database& source)
canAdd_ = false;
break;
}
if (avgShardSz_ > boost::filesystem::space(dir_).free)
if (avgShardSz_ > available())
{
JLOG(j_.error()) <<
"Insufficient disk space";
@@ -686,7 +702,6 @@ DatabaseShardImp::import(Database& source)
if (!shard->open(config_, scheduler_))
{
shard.reset();
this->remove(shardDir);
continue;
}
@@ -699,7 +714,7 @@ DatabaseShardImp::import(Database& source)
"shard " << shardIndex <<
" unable to create temp marker file";
shard.reset();
this->remove(shardDir);
removeAll(shardDir, j_);
continue;
}
ofs.close();
@@ -727,7 +742,7 @@ DatabaseShardImp::import(Database& source)
JLOG(j_.debug()) <<
"shard " << shardIndex <<
" successfully imported";
this->remove(markerFile);
removeAll(markerFile, j_);
break;
}
}
@@ -738,7 +753,7 @@ DatabaseShardImp::import(Database& source)
"shard " << shardIndex <<
" failed to import";
shard.reset();
this->remove(shardDir);
removeAll(shardDir, j_);
}
}
@@ -1070,7 +1085,7 @@ DatabaseShardImp::updateStats(std::lock_guard<std::mutex>&)
else
{
auto const sz = maxDiskSpace_ - usedDiskSpace_;
if (sz > space(dir_).free)
if (sz > available())
{
JLOG(j_.warn()) <<
"Max Shard Store size exceeds "
@@ -1110,21 +1125,19 @@ DatabaseShardImp::selectCache(std::uint32_t seq)
return {};
}
bool
DatabaseShardImp::remove(boost::filesystem::path const& path)
std::uint64_t
DatabaseShardImp::available() const
{
try
{
boost::filesystem::remove_all(path);
return boost::filesystem::space(dir_).available;
}
catch (const boost::filesystem::filesystem_error& e)
catch (std::exception const& e)
{
JLOG(j_.error()) <<
"remove_all " << path.string() <<
": Exception, " << e.code().message();
return false;
"exception: " << e.what();
return 0;
}
return true;
}
} // NodeStore

5
src/ripple/nodestore/impl/DatabaseShardImp.h
View File

@@ -241,8 +241,9 @@ private:
1, static_cast<int>(complete_.size() + (incomplete_ ? 1 : 0))));
}
bool
remove(boost::filesystem::path const& path);
// Returns available storage space
std::uint64_t
available() const;
};
} // NodeStore

128
src/ripple/nodestore/impl/Shard.cpp
View File

@@ -53,15 +53,47 @@ Shard::open(Section config, Scheduler& scheduler)
{
assert(!backend_);
using namespace boost::filesystem;
auto const newShard {!is_directory(dir_) || is_empty(dir_)};
bool dirPreexist;
bool dirEmpty;
try
{
if (!exists(dir_))
{
dirPreexist = false;
dirEmpty = true;
}
else if (is_directory(dir_))
{
dirPreexist = true;
dirEmpty = is_empty(dir_);
}
else
{
JLOG(j_.error()) <<
"path exists as file: " << dir_.string();
return false;
}
}
catch (std::exception const& e)
{
JLOG(j_.error()) <<
"shard " + std::to_string(index_) + " exception: " + e.what();
return false;
}
auto fail = [&](std::string msg)
{
if (!msg.empty())
JLOG(j_.error()) <<
"shard " << std::to_string(index_) << " error: " << msg;
if (!dirPreexist)
removeAll(dir_, j_);
else if (dirEmpty)
{
JLOG(j_.error()) << msg;
for (auto const& p : recursive_directory_iterator(dir_))
removeAll(p.path(), j_);
}
if (newShard)
this->remove(dir_);
return false;
};
@@ -70,50 +102,64 @@ Shard::open(Section config, Scheduler& scheduler)
{
backend_ = Manager::instance().make_Backend(
config, scheduler, j_);
backend_->open(newShard);
}
catch (std::exception const& e)
{
return fail("shard " + std::to_string(index_) +
": Exception, " + e.what());
}
backend_->open(!dirPreexist || dirEmpty);
if (backend_->fdlimit() == 0)
return true;
if (newShard)
if (!dirPreexist || dirEmpty)
{
// New shard, create a control file
if (!saveControl())
return fail({});
return fail("failure");
}
else if (is_regular_file(control_))
{
// Incomplete shard, inspect control file
std::ifstream ifs(control_.string());
if (!ifs.is_open())
{
return fail("shard " + std::to_string(index_) +
": Unable to open control file");
", unable to open control file");
}
boost::archive::text_iarchive ar(ifs);
ar & storedSeqs_;
if (!storedSeqs_.empty())
{
if (boost::icl::first(storedSeqs_) < firstSeq_ ||
boost::icl::last(storedSeqs_) > lastSeq_)
{
return fail("shard " + std::to_string(index_) +
": Invalid control file");
}
if (boost::icl::length(storedSeqs_) >= maxLedgers_)
{
JLOG(j_.error()) <<
"shard " << index_ <<
" found control file for complete shard";
storedSeqs_.clear();
this->remove(control_);
complete_ = true;
remove_all(control_);
}
}
}
else
complete_ = true;
updateFileSize();
// Calculate file foot print of backend files
for (auto const& p : recursive_directory_iterator(dir_))
if (!is_directory(p))
fileSize_ += file_size(p);
}
catch (std::exception const& e)
{
JLOG(j_.error()) <<
"shard " << std::to_string(index_) << " error: " << e.what();
return false;
}
return true;
}
@@ -133,9 +179,26 @@ Shard::setStored(std::shared_ptr<Ledger const> const& l)
{
if (backend_->fdlimit() != 0)
{
if (!this->remove(control_))
if (!removeAll(control_, j_))
return false;
updateFileSize();
// Update file foot print of backend files
using namespace boost::filesystem;
std::uint64_t sz {0};
try
{
for (auto const& p : recursive_directory_iterator(dir_))
if (!is_directory(p))
sz += file_size(p);
}
catch (const filesystem_error& e)
{
JLOG(j_.error()) <<
"exception: " << e.what();
fileSize_ = std::max(fileSize_, sz);
return false;
}
fileSize_ = sz;
}
complete_ = true;
storedSeqs_.clear();
@@ -406,16 +469,6 @@ Shard::valFetch(uint256 const& hash)
return nObj;
}
void
Shard::updateFileSize()
{
fileSize_ = 0;
using namespace boost::filesystem;
for (auto const& d : directory_iterator(dir_))
if (is_regular_file(d))
fileSize_ += file_size(d);
}
bool
Shard::saveControl()
{
@@ -432,22 +485,5 @@ Shard::saveControl()
return true;
}
bool
Shard::remove(boost::filesystem::path const& path)
{
try
{
boost::filesystem::remove_all(path);
}
catch (const boost::filesystem::filesystem_error& e)
{
JLOG(j_.error()) <<
"remove_all " << path.string() <<
": Exception, " << e.code().message();
return false;
}
return true;
}
} // NodeStore
} // ripple

22
src/ripple/nodestore/impl/Shard.h
View File

@@ -34,6 +34,24 @@
namespace ripple {
namespace NodeStore {
// Removes a path in its entirety
inline static
bool
removeAll(boost::filesystem::path const& path, beast::Journal& j)
{
try
{
boost::filesystem::remove_all(path);
}
catch (std::exception const& e)
{
JLOG(j.error()) <<
"exception: " << e.what();
return false;
}
return true;
}
using PCache = TaggedCache<uint256, NodeObject>;
using NCache = KeyCache<uint256>;
class DatabaseShard;
@@ -164,10 +182,6 @@ private:
// Save the control file for an incomplete shard
bool
saveControl();
// Remove directory or file
bool
remove(boost::filesystem::path const& path);
};
} // NodeStore