#ifndef XRPL_NODESTORE_CODEC_H_INCLUDED #define XRPL_NODESTORE_CODEC_H_INCLUDED // Disable lz4 deprecation warning due to incompatibility with clang attributes #define LZ4_DISABLE_DEPRECATE_WARNINGS #include #include #include #include #include #include #include #include #include #include namespace ripple { namespace NodeStore { template std::pair lz4_decompress(void const* in, std::size_t in_size, BufferFactory&& bf) { if (static_cast(in_size) < 0) Throw("lz4_decompress: integer overflow (input)"); std::size_t outSize = 0; auto const n = read_varint( reinterpret_cast(in), in_size, outSize); if (n == 0 || n >= in_size) Throw("lz4_decompress: invalid blob"); if (static_cast(outSize) <= 0) Throw("lz4_decompress: integer overflow (output)"); void* const out = bf(outSize); if (LZ4_decompress_safe( reinterpret_cast(in) + n, reinterpret_cast(out), static_cast(in_size - n), static_cast(outSize)) != static_cast(outSize)) Throw("lz4_decompress: LZ4_decompress_safe"); return {out, outSize}; } template std::pair lz4_compress(void const* in, std::size_t in_size, BufferFactory&& bf) { using std::runtime_error; using namespace nudb::detail; std::pair result; std::array::max> vi; auto const n = write_varint(vi.data(), in_size); auto const out_max = LZ4_compressBound(in_size); std::uint8_t* out = reinterpret_cast(bf(n + out_max)); result.first = out; std::memcpy(out, vi.data(), n); auto const out_size = LZ4_compress_default( reinterpret_cast(in), reinterpret_cast(out + n), in_size, out_max); if (out_size == 0) Throw("lz4 compress"); result.second = n + out_size; return result; } //------------------------------------------------------------------------------ /* object types: 0 = Uncompressed 1 = lz4 compressed 2 = inner node compressed 3 = full inner node */ template std::pair nodeobject_decompress(void const* in, std::size_t in_size, BufferFactory&& bf) { using namespace nudb::detail; std::uint8_t const* p = reinterpret_cast(in); std::size_t type; auto const vn = read_varint(p, in_size, type); if (vn == 0) Throw("nodeobject decompress"); p += vn; in_size -= vn; std::pair result; switch (type) { case 0: // uncompressed { result.first = p; result.second = in_size; break; } case 1: // lz4 { result = lz4_decompress(p, in_size, bf); break; } case 2: // compressed v1 inner node { auto const hs = field::size; // Mask if (in_size < hs + 32) Throw( "nodeobject codec v1: short inner node size: " + std::string("in_size = ") + std::to_string(in_size) + " hs = " + std::to_string(hs)); istream is(p, in_size); std::uint16_t mask; read(is, mask); // Mask in_size -= hs; result.second = 525; void* const out = bf(result.second); result.first = out; ostream os(out, result.second); write(os, 0); write(os, 0); write(os, hotUNKNOWN); write( os, static_cast(HashPrefix::innerNode)); if (mask == 0) Throw( "nodeobject codec v1: empty inner node"); std::uint16_t bit = 0x8000; for (int i = 16; i--; bit >>= 1) { if (mask & bit) { if (in_size < 32) Throw( "nodeobject codec v1: short inner node subsize: " + std::string("in_size = ") + std::to_string(in_size) + " i = " + std::to_string(i)); std::memcpy(os.data(32), is(32), 32); in_size -= 32; } else { std::memset(os.data(32), 0, 32); } } if (in_size > 0) Throw( "nodeobject codec v1: long inner node, in_size = " + std::to_string(in_size)); break; } case 3: // full v1 inner node { if (in_size != 16 * 32) // hashes Throw( "nodeobject codec v1: short full inner node, in_size = " + std::to_string(in_size)); istream is(p, in_size); result.second = 525; void* const out = bf(result.second); result.first = out; ostream os(out, result.second); write(os, 0); write(os, 0); write(os, hotUNKNOWN); write( os, static_cast(HashPrefix::innerNode)); write(os, is(512), 512); break; } default: Throw( "nodeobject codec: bad type=" + std::to_string(type)); }; return result; } template void const* zero32() { static std::array v{}; return v.data(); } template std::pair nodeobject_compress(void const* in, std::size_t in_size, BufferFactory&& bf) { using std::runtime_error; using namespace nudb::detail; // Check for inner node v1 if (in_size == 525) { istream is(in, in_size); std::uint32_t index; std::uint32_t unused; std::uint8_t kind; std::uint32_t prefix; read(is, index); read(is, unused); read(is, kind); read(is, prefix); if (safe_cast(prefix) == HashPrefix::innerNode) { std::size_t n = 0; std::uint16_t mask = 0; std::array vh; for (unsigned bit = 0x8000; bit; bit >>= 1) { void const* const h = is(32); if (std::memcmp(h, zero32(), 32) == 0) continue; std::memcpy(vh.data() + 32 * n, h, 32); mask |= bit; ++n; } std::pair result; if (n < 16) { // 2 = v1 inner node compressed auto const type = 2U; auto const vs = size_varint(type); result.second = vs + field::size + // mask n * 32; // hashes std::uint8_t* out = reinterpret_cast(bf(result.second)); result.first = out; ostream os(out, result.second); write(os, type); write(os, mask); write(os, vh.data(), n * 32); return result; } // 3 = full v1 inner node auto const type = 3U; auto const vs = size_varint(type); result.second = vs + n * 32; // hashes std::uint8_t* out = reinterpret_cast(bf(result.second)); result.first = out; ostream os(out, result.second); write(os, type); write(os, vh.data(), n * 32); return result; } } std::array::max> vi; constexpr std::size_t codecType = 1; auto const vn = write_varint(vi.data(), codecType); std::pair result; switch (codecType) { // case 0 was uncompressed data; we always compress now. case 1: // lz4 { std::uint8_t* p; auto const lzr = NodeStore::lz4_compress( in, in_size, [&p, &vn, &bf](std::size_t n) { p = reinterpret_cast(bf(vn + n)); return p + vn; }); std::memcpy(p, vi.data(), vn); result.first = p; result.second = vn + lzr.second; break; } default: Throw( "nodeobject codec: unknown=" + std::to_string(codecType)); }; return result; } // Modifies an inner node to erase the ledger // sequence and type information so the codec // verification can pass. // template void filter_inner(void* in, std::size_t in_size) { using namespace nudb::detail; // Check for inner node if (in_size == 525) { istream is(in, in_size); std::uint32_t index; std::uint32_t unused; std::uint8_t kind; std::uint32_t prefix; read(is, index); read(is, unused); read(is, kind); read(is, prefix); if (safe_cast(prefix) == HashPrefix::innerNode) { ostream os(in, 9); write(os, 0); write(os, 0); write(os, hotUNKNOWN); } } } } // namespace NodeStore } // namespace ripple #endif