Files
xahaud/beast/nudb/posix_file.h
Vinnie Falco 33c5f98824 NuDB: Performance improvements (RIPD-793,796):
This introduces changes in nudb to improve speed, reduce database size,
and enhance correctness. The most significant change is to store hashes
rather than entire keys in the key file. The output of the hash function
is reduced to 48 bits, and stored directly in buckets.

The API is also modified to introduce a Codec parameter allowing for
compression and decompression to be supported in the implementation
itself rather than callers.

THe data file no longer contains a salt, as the salt is applicable
only to the key and log files. This allows a data file to have multiple
key files with different salt values. To distinguish physical files
belonging to the same logical database, a new field UID is introduced.
The UID is a 64-bit random value generated once on creation and stored
in all three files.

Buckets are zero filled to the end of each block, this is a security
measure to prevent unintended contents of memory getting stored to
disk. NuDB offers the varint integer type, this is identical to
the varint described by Google.

* Add varint
* Add Codec template argument
* Add "api" convenience traits
* Store hash in buckets
* istream can throw short read errors
* Support std::uint8_t format in streams
* Make file classes part of the public interface
* Remove buffers pessimization, replace with buffer
* Consolidate creation utility functions to the same header
* Zero fill unused areas of buckets on disk
* More coverage and improvements to the recover test
* Fix file read/write to loop until all bytes processed
* Add verify_fast, faster verify for large databases

The database version number is incremented to 2; older databases can
no longer be opened and should be deleted.
2015-02-11 14:41:31 -08:00

377 lines
7.6 KiB
C++

//------------------------------------------------------------------------------
/*
This file is part of Beast: https://github.com/vinniefalco/Beast
Copyright 2014, Vinnie Falco <vinnie.falco@gmail.com>
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#ifndef BEAST_NUDB_DETAIL_POSIX_FILE_H_INCLUDED
#define BEAST_NUDB_DETAIL_POSIX_FILE_H_INCLUDED
#include <beast/nudb/common.h>
#include <cassert>
#include <cerrno>
#include <cstring>
#include <string>
#include <utility>
#ifndef BEAST_NUDB_POSIX_FILE
# ifdef _MSC_VER
# define BEAST_NUDB_POSIX_FILE 0
# else
# define BEAST_NUDB_POSIX_FILE 1
# endif
#endif
#if BEAST_NUDB_POSIX_FILE
# include <fcntl.h>
# include <sys/types.h>
# include <sys/uio.h>
# include <sys/stat.h>
# include <unistd.h>
#endif
namespace beast {
namespace nudb {
#if BEAST_NUDB_POSIX_FILE
namespace detail {
class file_posix_error : public file_error
{
public:
explicit
file_posix_error (char const* m,
int errnum = errno)
: file_error (std::string("nudb: ") + m +
", " + text(errnum))
{
}
explicit
file_posix_error (std::string const& m,
int errnum = errno)
: file_error (std::string("nudb: ") + m +
", " + text(errnum))
{
}
private:
static
std::string
text (int errnum)
{
return std::strerror(errnum);
}
};
//------------------------------------------------------------------------------
template <class = void>
class posix_file
{
private:
int fd_ = -1;
public:
posix_file() = default;
posix_file (posix_file const&) = delete;
posix_file& operator= (posix_file const&) = delete;
~posix_file();
posix_file (posix_file&&);
posix_file&
operator= (posix_file&& other);
bool
is_open() const
{
return fd_ != -1;
}
void
close();
bool
create (file_mode mode, path_type const& path);
bool
open (file_mode mode, path_type const& path);
static
bool
erase (path_type const& path);
std::size_t
actual_size() const;
void
read (std::size_t offset,
void* buffer, std::size_t bytes);
void
write (std::size_t offset,
void const* buffer, std::size_t bytes);
void
sync();
void
trunc (std::size_t length);
private:
static
std::pair<int, int>
flags (file_mode mode);
};
template <class _>
posix_file<_>::~posix_file()
{
close();
}
template <class _>
posix_file<_>::posix_file (posix_file &&other)
: fd_ (other.fd_)
{
other.fd_ = -1;
}
template <class _>
posix_file<_>&
posix_file<_>::operator= (posix_file&& other)
{
if (&other == this)
return *this;
close();
fd_ = other.fd_;
other.fd_ = -1;
return *this;
}
template <class _>
void
posix_file<_>::close()
{
if (fd_ != -1)
{
if (::close(fd_) != 0)
throw file_posix_error(
"close file");
fd_ = -1;
}
}
template <class _>
bool
posix_file<_>::create (file_mode mode,
path_type const& path)
{
auto const result = flags(mode);
assert(! is_open());
fd_ = ::open(path.c_str(), result.first);
if (fd_ != -1)
{
::close(fd_);
fd_ = -1;
return false;
}
int errnum = errno;
if (errnum != ENOENT)
throw file_posix_error(
"open file", errnum);
fd_ = ::open(path.c_str(),
result.first | O_CREAT, 0644);
if (fd_ == -1)
throw file_posix_error(
"create file");
#ifndef __APPLE__
if (::posix_fadvise(fd_, 0, 0, result.second) != 0)
throw file_posix_error(
"fadvise");
#endif
return true;
}
template <class _>
bool
posix_file<_>::open (file_mode mode,
path_type const& path)
{
assert(! is_open());
auto const result = flags(mode);
fd_ = ::open(path.c_str(), result.first);
if (fd_ == -1)
{
int errnum = errno;
if (errnum == ENOENT)
return false;
throw file_posix_error(
"open file", errnum);
}
#ifndef __APPLE__
if (::posix_fadvise(fd_, 0, 0, result.second) != 0)
throw file_posix_error(
"fadvise");
#endif
return true;
}
template <class _>
bool
posix_file<_>::erase (path_type const& path)
{
if (::unlink(path.c_str()) != 0)
{
int const ec = errno;
if (ec != ENOENT)
throw file_posix_error(
"unlink", ec);
return false;
}
return true;
}
template <class _>
std::size_t
posix_file<_>::actual_size() const
{
struct stat st;
if (::fstat(fd_, &st) != 0)
throw file_posix_error(
"fstat");
return st.st_size;
}
template <class _>
void
posix_file<_>::read (std::size_t offset,
void* buffer, std::size_t bytes)
{
while(bytes > 0)
{
auto const n = ::pread (
fd_, buffer, bytes, offset);
// VFALCO end of file should throw short_read
if (n == -1)
throw file_posix_error(
"pread");
if (n == 0)
throw file_short_read_error();
offset += n;
bytes -= n;
buffer = reinterpret_cast<
char*>(buffer) + n;
}
}
template <class _>
void
posix_file<_>::write (std::size_t offset,
void const* buffer, std::size_t bytes)
{
while(bytes > 0)
{
auto const n = ::pwrite (
fd_, buffer, bytes, offset);
if (n == -1)
throw file_posix_error(
"pwrite");
if (n == 0)
throw file_short_write_error();
offset += n;
bytes -= n;
buffer = reinterpret_cast<
char const*>(buffer) + n;
}
}
template <class _>
void
posix_file<_>::sync()
{
if (::fsync(fd_) != 0)
throw file_posix_error(
"fsync");
}
template <class _>
void
posix_file<_>::trunc (std::size_t length)
{
if (::ftruncate(fd_, length) != 0)
throw file_posix_error(
"ftruncate");
}
template <class _>
std::pair<int, int>
posix_file<_>::flags (file_mode mode)
{
std::pair<int, int> result;
switch(mode)
{
case file_mode::scan:
result.first =
O_RDONLY;
#ifndef __APPLE__
result.second =
POSIX_FADV_SEQUENTIAL;
#endif
break;
case file_mode::read:
result.first =
O_RDONLY;
#ifndef __APPLE__
result.second =
POSIX_FADV_RANDOM;
#endif
break;
case file_mode::append:
result.first =
O_RDWR |
O_APPEND;
#ifndef __APPLE__
result.second =
POSIX_FADV_RANDOM;
#endif
break;
case file_mode::write:
result.first =
O_RDWR;
#ifndef __APPLE__
result.second =
POSIX_FADV_NORMAL;
#endif
break;
}
return result;
}
} // detail
using posix_file = detail::posix_file<>;
#endif
} // nudb
} // beast
#endif