Merge branch 'dev' into fix/rpc-queue-limit

CMakeLists.txt

@@ -11,8 +11,6 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
 set(Boost_NO_BOOST_CMAKE ON)
 
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=x86-64-v4")
-
 # make GIT_COMMIT_HASH define available to all sources
 find_package(Git)
 if(Git_FOUND)

cfg/rippled-standalone.cfg

@@ -144,4 +144,12 @@ D686F2538F410C9D0D856788E98E3579595DAF7B38D38887F81ECAC934B06040 HooksUpdate1
 86E83A7D2ECE3AD5FA87AB2195AE015C950469ABF0B72EAACED318F74886AE90 CryptoConditionsSuite
 3C43D9A973AA4443EF3FC38E42DD306160FBFFDAB901CD8BAA15D09F2597EB87 NonFungibleTokensV1
 0285B7E5E08E1A8E4C15636F0591D87F73CB6A7B6452A932AD72BBC8E5D1CBE3 fixNFTokenDirV1
-36799EA497B1369B170805C078AEFE6188345F9B3E324C21E9CA3FF574E3C3D6 fixNFTokenNegOffer
+36799EA497B1369B170805C078AEFE6188345F9B3E324C21E9CA3FF574E3C3D6 fixNFTokenNegOffer
+4C499D17719BB365B69010A436B64FD1A82AAB199FC1CEB06962EBD01059FB09 fixXahauV1
+215181D23BF5C173314B5FDB9C872C92DE6CC918483727DE037C0C13E7E6EE9D fixXahauV2
+0D8BF22FF7570D58598D1EF19EBB6E142AD46E59A223FD3816262FBB69345BEA Remit
+7CA0426E7F411D39BB014E57CD9E08F61DE1750F0D41FCD428D9FB80BB7596B0 ZeroB2M
+4B8466415FAB32FFA89D9DCBE166A42340115771DF611A7160F8D7439C87ECD8 fixNSDelete
+EDB4EE4C524E16BDD91D9A529332DED08DCAAA51CC6DC897ACFA1A0ED131C5B6 fix240819
+8063140E9260799D6716756B891CEC3E7006C4E4F277AB84670663A88F94B9C4 fixPageCap
+88693F108C3CD8A967F3F4253A32DEF5E35F9406ACD2A11B88B11D90865763A9 fix240911

src/ripple/basics/hardened_hash.h

@@ -1,8 +1,28 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2014 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
 #ifndef RIPPLE_BASICS_HARDENED_HASH_H_INCLUDED
 #define RIPPLE_BASICS_HARDENED_HASH_H_INCLUDED
 
 #include <ripple/beast/hash/hash_append.h>
 #include <ripple/beast/hash/xxhasher.h>
+
 #include <cstdint>
 #include <functional>
 #include <mutex>
@@ -12,68 +32,10 @@
 #include <unordered_set>
 #include <utility>
 
-#if defined(__x86_64__) || defined(_M_X64)
-#include <cpuid.h>
-#include <immintrin.h>
-#endif
-
 namespace ripple {
 
 namespace detail {
 
-#if defined(__x86_64__) || defined(_M_X64)
-inline bool
-check_aesni_support()
-{
-    unsigned int eax, ebx, ecx, edx;
-    if (__get_cpuid(1, &eax, &ebx, &ecx, &edx))
-    {
-        return (ecx & bit_AES) != 0;
-    }
-    return false;
-}
-
-// Helper function to contain all AES-NI operations
-#if defined(__GNUC__) || defined(__clang__)
-__attribute__((__target__("aes")))
-#endif
-inline __m128i
-aesni_hash_block(__m128i state, __m128i key, const void* data, size_t len)
-{
-    const uint8_t* ptr = static_cast<const uint8_t*>(data);
-    while (len >= 16)
-    {
-        __m128i block = _mm_loadu_si128(reinterpret_cast<const __m128i*>(ptr));
-        state = _mm_xor_si128(state, block);
-        state = _mm_aesenc_si128(state, key);
-        ptr += 16;
-        len -= 16;
-    }
-
-    if (len > 0)
-    {
-        alignas(16) uint8_t last_block[16] = {0};
-        std::memcpy(last_block, ptr, len);
-        __m128i block =
-            _mm_load_si128(reinterpret_cast<const __m128i*>(last_block));
-        state = _mm_xor_si128(state, block);
-        state = _mm_aesenc_si128(state, key);
-    }
-
-    return state;
-}
-
-// Helper function for final AES round
-#if defined(__GNUC__) || defined(__clang__)
-__attribute__((__target__("aes")))
-#endif
-inline __m128i
-aesni_hash_final(__m128i state, __m128i key)
-{
-    return _mm_aesenclast_si128(state, key);
-}
-#endif
-
 using seed_pair = std::pair<std::uint64_t, std::uint64_t>;
 
 template <bool = true>
@@ -86,9 +48,12 @@ make_seed_pair() noexcept
         std::random_device rng;
         std::mt19937_64 gen;
         std::uniform_int_distribution<std::uint64_t> dist;
+
         state_t() : gen(rng())
         {
         }
+        // state_t(state_t const&) = delete;
+        // state_t& operator=(state_t const&) = delete;
     };
     static state_t state;
     std::lock_guard lock(state.mutex);
@@ -97,23 +62,46 @@ make_seed_pair() noexcept
 
 }  // namespace detail
 
+/**
+ * Seed functor once per construction
+
+   A std compatible hash adapter that resists adversarial inputs.
+   For this to work, T must implement in its own namespace:
+
+   @code
+
+   template <class Hasher>
+   void
+   hash_append (Hasher& h, T const& t) noexcept
+   {
+       // hash_append each base and member that should
+       //  participate in forming the hash
+       using beast::hash_append;
+       hash_append (h, static_cast<T::base1 const&>(t));
+       hash_append (h, static_cast<T::base2 const&>(t));
+       // ...
+       hash_append (h, t.member1);
+       hash_append (h, t.member2);
+       // ...
+   }
+
+   @endcode
+
+   Do not use any version of Murmur or CityHash for the Hasher
+   template parameter (the hashing algorithm).  For details
+   see https://131002.net/siphash/#at
+*/
+
 template <class HashAlgorithm = beast::xxhasher>
 class hardened_hash
 {
 private:
     detail::seed_pair m_seeds;
-#if defined(__x86_64__) || defined(_M_X64)
-    bool using_aesni_;
-#endif
 
 public:
     using result_type = typename HashAlgorithm::result_type;
 
-    hardened_hash()
-        : m_seeds(detail::make_seed_pair<>())
-#if defined(__x86_64__) || defined(_M_X64)
-        , using_aesni_(detail::check_aesni_support())
-#endif
+    hardened_hash() : m_seeds(detail::make_seed_pair<>())
     {
     }
 
@@ -121,24 +109,7 @@ public:
     result_type
     operator()(T const& t) const noexcept
     {
-#if defined(__x86_64__) || defined(_M_X64)
-        if (using_aesni_)
-        {
-            alignas(16) __m128i key =
-                _mm_set_epi64x(m_seeds.first, m_seeds.second);
-            alignas(16) __m128i state = _mm_setzero_si128();
-
-            // Hash the data using AES-NI
-            const char* data = reinterpret_cast<const char*>(&t);
-            state = detail::aesni_hash_block(state, key, data, sizeof(t));
-            state = detail::aesni_hash_final(state, key);
-
-            return static_cast<result_type>(_mm_cvtsi128_si64(state));
-        }
-#endif
-        // Original implementation using xxhasher
         HashAlgorithm h(m_seeds.first, m_seeds.second);
-        using beast::hash_append;
         hash_append(h, t);
         return static_cast<result_type>(h);
     }

src/ripple/net/impl/RPCCall.cpp

@@ -1894,7 +1894,9 @@ fromNetwork(
     constexpr auto RPC_REPLY_MAX_BYTES = megabytes(256);
 
     using namespace std::chrono_literals;
-    auto constexpr RPC_NOTIFY = 10min;
+    // auto constexpr RPC_NOTIFY = 10min; // Wietse: lolwut 10 minutes for one
+    // HTTP call?
+    auto constexpr RPC_NOTIFY = 30s;
 
     HTTPClient::request(
         bSSL,

src/ripple/net/impl/RPCSub.cpp

@@ -78,12 +78,14 @@ public:
     {
         std::lock_guard sl(mLock);
 
-        if (mDeque.size() >= eventQueueMax)
-        {
-            // Drop the previous event.
-            JLOG(j_.warn()) << "RPCCall::fromNetwork drop";
-            mDeque.pop_back();
-        }
+        // Wietse: we're not going to limit this, this is admin-port only, scale
+        // accordingly Dropping events just like this results in inconsistent
+        // data on the receiving end if (mDeque.size() >= eventQueueMax)
+        // {
+        //     // Drop the previous event.
+        //     JLOG(j_.warn()) << "RPCCall::fromNetwork drop";
+        //     mDeque.pop_back();
+        // }
 
         auto jm = broadcast ? j_.debug() : j_.info();
         JLOG(jm) << "RPCCall::fromNetwork push: " << jvObj;
@@ -182,7 +184,8 @@ private:
     }
 
 private:
-    enum { eventQueueMax = 32 };
+    // Wietse: we're not going to limit this, this is admin-port only, scale
+    // accordingly enum { eventQueueMax = 32 };
 
     boost::asio::io_service& m_io_service;
     JobQueue& m_jobQueue;

src/ripple/protocol/impl/digest.cpp

@@ -18,168 +18,19 @@
 //==============================================================================
 
 #include <ripple/protocol/digest.h>
-#include <immintrin.h>
 #include <openssl/ripemd.h>
 #include <openssl/sha.h>
+#include <type_traits>
 
 namespace ripple {
-namespace detail {
 
-#if defined(__x86_64__)
-#if defined(__clang__)
-#pragma clang attribute push( \
-    __attribute__((target("xsave,avx512f,avx512bw"))), apply_to = function)
-#elif defined(__GNUC__)
-#pragma GCC push_options
-#pragma GCC target("xsave,avx512f,avx512bw")
-#endif
-
-static bool
-check_avx512()
-{
-    unsigned int eax = 0, ebx = 0, ecx = 0, edx = 0;
-    if (__get_cpuid(1, &eax, &ebx, &ecx, &edx))
-    {
-        if ((ecx & bit_AVX) && (ecx & bit_OSXSAVE))
-        {
-            unsigned long long xcr0 = _xgetbv(0);
-            if ((xcr0 & 6) == 6)
-            {
-                if (__get_cpuid_count(7, 0, &eax, &ebx, &ecx, &edx))
-                    return (ebx & bit_AVX512F) && (ebx & bit_AVX512BW);
-            }
-        }
-    }
-    return false;
-}
-
-#if defined(__clang__)
-#pragma clang attribute pop
-#elif defined(__GNUC__)
-#pragma GCC pop_options
-#endif
-#endif
-
-static bool
-has_avx512()
-{
-    static const bool support = [] {
-#if defined(__x86_64__)
-        return check_avx512();
-#else
-        return false;
-#endif
-    }();
-    return support;
-}
-
-#if defined(__x86_64__)
-#if defined(__clang__)
-#pragma clang attribute push( \
-    __attribute__((target("avx512f,avx512bw"))), apply_to = function)
-#elif defined(__GNUC__)
-#pragma GCC push_options
-#pragma GCC target("avx512f,avx512bw")
-#endif
-
-static void
-process_sha256_blocks_avx512(
-    SHA256_CTX* ctx,
-    const uint8_t* data,
-    size_t blocks)
-{
-    for (size_t i = 0; i < blocks; ++i)
-    {
-        __m512i block = _mm512_loadu_si512(
-            reinterpret_cast<const __m512i*>(data + (i * 64)));
-
-#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
-        const __m512i swap = _mm512_setr_epi64(
-            0x0001020304050607,
-            0x08090a0b0c0d0e0f,
-            0x1011121314151617,
-            0x18191a1b1c1d1e1f,
-            0x2021222324252627,
-            0x28292a2b2c2d2e2f,
-            0x3031323334353637,
-            0x38393a3b3c3d3e3f);
-        block = _mm512_shuffle_epi8(block, swap);
-#endif
-
-        SHA256_Update(ctx, data + (i * 64), 64);
-    }
-}
-
-static void
-process_sha512_blocks_avx512(
-    SHA512_CTX* ctx,
-    const uint8_t* data,
-    size_t blocks)
-{
-    for (size_t i = 0; i < blocks; ++i)
-    {
-        __m512i block = _mm512_loadu_si512(
-            reinterpret_cast<const __m512i*>(data + (i * 64)));
-
-#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
-        const __m512i swap = _mm512_setr_epi64(
-            0x0001020304050607,
-            0x08090a0b0c0d0e0f,
-            0x1011121314151617,
-            0x18191a1b1c1d1e1f,
-            0x2021222324252627,
-            0x28292a2b2c2d2e2f,
-            0x3031323334353637,
-            0x38393a3b3c3d3e3f);
-        block = _mm512_shuffle_epi8(block, swap);
-#endif
-
-        SHA512_Update(ctx, data + (i * 64), 64);
-    }
-}
-
-static void
-process_ripemd160_blocks_avx512(
-    RIPEMD160_CTX* ctx,
-    const uint8_t* data,
-    size_t blocks)
-{
-    for (size_t i = 0; i < blocks; ++i)
-    {
-        __m512i block = _mm512_loadu_si512(
-            reinterpret_cast<const __m512i*>(data + (i * 64)));
-
-#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
-        const __m512i swap = _mm512_setr_epi64(
-            0x0001020304050607,
-            0x08090a0b0c0d0e0f,
-            0x1011121314151617,
-            0x18191a1b1c1d1e1f,
-            0x2021222324252627,
-            0x28292a2b2c2d2e2f,
-            0x3031323334353637,
-            0x38393a3b3c3d3e3f);
-        block = _mm512_shuffle_epi8(block, swap);
-#endif
-
-        RIPEMD160_Update(ctx, data + (i * 64), 64);
-    }
-}
-
-#if defined(__clang__)
-#pragma clang attribute pop
-#elif defined(__GNUC__)
-#pragma GCC pop_options
-#endif
-#endif
-
-}  // namespace detail
-
-// RIPEMD160 implementation
 openssl_ripemd160_hasher::openssl_ripemd160_hasher()
 {
-    static_assert(sizeof(ctx_) >= sizeof(RIPEMD160_CTX), "");
-    auto ctx = reinterpret_cast<RIPEMD160_CTX*>(ctx_);
+    static_assert(
+        sizeof(decltype(openssl_ripemd160_hasher::ctx_)) ==
+            sizeof(RIPEMD160_CTX),
+        "");
+    auto const ctx = reinterpret_cast<RIPEMD160_CTX*>(ctx_);
     RIPEMD160_Init(ctx);
 }
 
@@ -188,113 +39,68 @@ openssl_ripemd160_hasher::operator()(
     void const* data,
     std::size_t size) noexcept
 {
-    auto ctx = reinterpret_cast<RIPEMD160_CTX*>(ctx_);
-
-#if defined(__x86_64__)
-    if (detail::has_avx512() && size >= 64)
-    {
-        size_t blocks = size / 64;
-        detail::process_ripemd160_blocks_avx512(
-            ctx, static_cast<const uint8_t*>(data), blocks);
-        size_t remaining = size % 64;
-        if (remaining)
-            RIPEMD160_Update(
-                ctx,
-                static_cast<const uint8_t*>(data) + (blocks * 64),
-                remaining);
-        return;
-    }
-#endif
-
+    auto const ctx = reinterpret_cast<RIPEMD160_CTX*>(ctx_);
     RIPEMD160_Update(ctx, data, size);
 }
 
 openssl_ripemd160_hasher::operator result_type() noexcept
 {
-    auto ctx = reinterpret_cast<RIPEMD160_CTX*>(ctx_);
+    auto const ctx = reinterpret_cast<RIPEMD160_CTX*>(ctx_);
     result_type digest;
     RIPEMD160_Final(digest.data(), ctx);
     return digest;
 }
 
-// SHA256 implementation
-openssl_sha256_hasher::openssl_sha256_hasher()
-{
-    static_assert(sizeof(ctx_) >= sizeof(SHA256_CTX), "");
-    auto ctx = reinterpret_cast<SHA256_CTX*>(ctx_);
-    SHA256_Init(ctx);
-}
+//------------------------------------------------------------------------------
 
-void
-openssl_sha256_hasher::operator()(void const* data, std::size_t size) noexcept
-{
-    auto ctx = reinterpret_cast<SHA256_CTX*>(ctx_);
-
-#if defined(__x86_64__)
-    if (detail::has_avx512() && size >= 64)
-    {
-        size_t blocks = size / 64;
-        detail::process_sha256_blocks_avx512(
-            ctx, static_cast<const uint8_t*>(data), blocks);
-        size_t remaining = size % 64;
-        if (remaining)
-            SHA256_Update(
-                ctx,
-                static_cast<const uint8_t*>(data) + (blocks * 64),
-                remaining);
-        return;
-    }
-#endif
-
-    SHA256_Update(ctx, data, size);
-}
-
-openssl_sha256_hasher::operator result_type() noexcept
-{
-    auto ctx = reinterpret_cast<SHA256_CTX*>(ctx_);
-    result_type digest;
-    SHA256_Final(digest.data(), ctx);
-    return digest;
-}
-
-// SHA512 implementation
 openssl_sha512_hasher::openssl_sha512_hasher()
 {
-    static_assert(sizeof(ctx_) >= sizeof(SHA512_CTX), "");
-    auto ctx = reinterpret_cast<SHA512_CTX*>(ctx_);
+    static_assert(
+        sizeof(decltype(openssl_sha512_hasher::ctx_)) == sizeof(SHA512_CTX),
+        "");
+    auto const ctx = reinterpret_cast<SHA512_CTX*>(ctx_);
     SHA512_Init(ctx);
 }
 
 void
 openssl_sha512_hasher::operator()(void const* data, std::size_t size) noexcept
 {
-    auto ctx = reinterpret_cast<SHA512_CTX*>(ctx_);
-
-#if defined(__x86_64__)
-    if (detail::has_avx512() && size >= 64)
-    {
-        size_t blocks = size / 64;
-        detail::process_sha512_blocks_avx512(
-            ctx, static_cast<const uint8_t*>(data), blocks);
-        size_t remaining = size % 64;
-        if (remaining)
-            SHA512_Update(
-                ctx,
-                static_cast<const uint8_t*>(data) + (blocks * 64),
-                remaining);
-        return;
-    }
-#endif
-
+    auto const ctx = reinterpret_cast<SHA512_CTX*>(ctx_);
     SHA512_Update(ctx, data, size);
 }
 
 openssl_sha512_hasher::operator result_type() noexcept
 {
-    auto ctx = reinterpret_cast<SHA512_CTX*>(ctx_);
+    auto const ctx = reinterpret_cast<SHA512_CTX*>(ctx_);
     result_type digest;
     SHA512_Final(digest.data(), ctx);
     return digest;
 }
 
+//------------------------------------------------------------------------------
+
+openssl_sha256_hasher::openssl_sha256_hasher()
+{
+    static_assert(
+        sizeof(decltype(openssl_sha256_hasher::ctx_)) == sizeof(SHA256_CTX),
+        "");
+    auto const ctx = reinterpret_cast<SHA256_CTX*>(ctx_);
+    SHA256_Init(ctx);
+}
+
+void
+openssl_sha256_hasher::operator()(void const* data, std::size_t size) noexcept
+{
+    auto const ctx = reinterpret_cast<SHA256_CTX*>(ctx_);
+    SHA256_Update(ctx, data, size);
+}
+
+openssl_sha256_hasher::operator result_type() noexcept
+{
+    auto const ctx = reinterpret_cast<SHA256_CTX*>(ctx_);
+    result_type digest;
+    SHA256_Final(digest.data(), ctx);
+    return digest;
+}
+
 }  // namespace ripple