Upgrade to xxhash 0.8.2 as a Conan requirement, enable SIMD hashing (#4893)

We are currently using old version 0.6.2 of `xxhash`, as a verbatim copy and paste of its header file `xxhash.h`. Switch to the more recent version 0.8.2. Since this version is in Conan Center (and properly protects its ABI by keeping the state object incomplete), add it as a Conan requirement. Switch to the SIMD instructions (in the new `XXH3` family) supported by the new version.

Builds/CMake/RippledCore.cmake

@@ -26,7 +26,6 @@ target_sources (xrpl_core PRIVATE
   src/ripple/beast/clock/basic_seconds_clock.cpp
   src/ripple/beast/core/CurrentThreadName.cpp
   src/ripple/beast/core/SemanticVersion.cpp
-  src/ripple/beast/hash/impl/xxhash.cpp
   src/ripple/beast/insight/impl/Collector.cpp
   src/ripple/beast/insight/impl/Groups.cpp
   src/ripple/beast/insight/impl/Hook.cpp
@@ -153,7 +152,8 @@ target_link_libraries (xrpl_core
     secp256k1::secp256k1
     ed25519::ed25519
     date::date
-    Ripple::opts)
+    Ripple::opts
+    xxHash::xxhash)
 #[=================================[
    main/core headers installation
 #]=================================]
@@ -362,9 +362,6 @@ install (
     src/ripple/beast/hash/uhash.h
     src/ripple/beast/hash/xxhasher.h
   DESTINATION include/ripple/beast/hash)
-install (
-  FILES src/ripple/beast/hash/impl/xxhash.h
-  DESTINATION include/ripple/beast/hash/impl)
 install (
   FILES
   src/ripple/beast/net/IPAddress.h

CMakeLists.txt

@@ -94,6 +94,7 @@ endif()
 
 find_package(nudb REQUIRED)
 find_package(date REQUIRED)
+find_package(xxHash REQUIRED)
 
 target_link_libraries(ripple_libs INTERFACE
   ed25519::ed25519

conanfile.py

@@ -36,6 +36,7 @@ class Xrpl(ConanFile):
         'soci/4.0.3',
         'sqlite3/3.42.0',
         'zlib/1.2.13',
+        'xxhash/0.8.2',
     ]
 
     default_options = {
@@ -86,6 +87,7 @@ class Xrpl(ConanFile):
         'soci:shared': False,
         'soci:with_sqlite3': True,
         'soci:with_boost': True,
+        'xxhash:shared': False,
     }
 
     def set_version(self):
@@ -159,4 +161,5 @@ class Xrpl(ConanFile):
             'openssl::crypto',
             'date::date',
             'grpc::grpc++',
+            'xxHash::xxhash',
         ]

src/ripple/beast/hash/impl/xxhash.h

@@ -1,170 +0,0 @@
-/*
-   xxHash - Extremely Fast Hash algorithm
-   Header File
-   Copyright (C) 2012-2014, Yann Collet.
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-   Redistribution and use in source and binary forms, with or without
-   modification, are permitted provided that the following conditions are
-   met:
-
-       * Redistributions of source code must retain the above copyright
-   notice, this list of conditions and the following disclaimer.
-       * Redistributions in binary form must reproduce the above
-   copyright notice, this list of conditions and the following disclaimer
-   in the documentation and/or other materials provided with the
-   distribution.
-
-   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-   You can contact the author at :
-   - xxHash source repository : http://code.google.com/p/xxhash/
-*/
-
-/* Notice extracted from xxHash homepage :
-
-xxHash is an extremely fast Hash algorithm, running at RAM speed limits.
-It also successfully passes all tests from the SMHasher suite.
-
-Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo
-@3GHz)
-
-Name            Speed       Q.Score   Author
-xxHash          5.4 GB/s     10
-CrapWow         3.2 GB/s      2       Andrew
-MumurHash 3a    2.7 GB/s     10       Austin Appleby
-SpookyHash      2.0 GB/s     10       Bob Jenkins
-SBox            1.4 GB/s      9       Bret Mulvey
-Lookup3         1.2 GB/s      9       Bob Jenkins
-SuperFastHash   1.2 GB/s      1       Paul Hsieh
-CityHash64      1.05 GB/s    10       Pike & Alakuijala
-FNV             0.55 GB/s     5       Fowler, Noll, Vo
-CRC32           0.43 GB/s     9
-MD5-32          0.33 GB/s    10       Ronald L. Rivest
-SHA1-32         0.28 GB/s    10
-
-Q.Score is a measure of quality of the hash function.
-It depends on successfully passing SMHasher test set.
-10 is a perfect score.
-*/
-
-#ifndef BEAST_HASH_XXHASH_H_INCLUDED
-#define BEAST_HASH_XXHASH_H_INCLUDED
-
-/*****************************
-   Includes
-*****************************/
-#include <stddef.h> /* size_t */
-
-namespace beast {
-namespace detail {
-
-/*****************************
-   Type
-*****************************/
-typedef enum { XXH_OK = 0, XXH_ERROR } XXH_errorcode;
-
-/*****************************
-   Simple Hash Functions
-*****************************/
-
-unsigned int
-XXH32(const void* input, size_t length, unsigned seed);
-unsigned long long
-XXH64(const void* input, size_t length, unsigned long long seed);
-
-/*
-XXH32() :
-    Calculate the 32-bits hash of sequence "length" bytes stored at memory
-address "input". The memory between input & input+length must be valid
-(allocated and read-accessible). "seed" can be used to alter the result
-predictably. This function successfully passes all SMHasher tests. Speed on Core
-2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s XXH64() : Calculate
-the 64-bits hash of sequence of length "len" stored at memory address "input".
-*/
-
-/*****************************
-   Advanced Hash Functions
-*****************************/
-typedef struct
-{
-    long long ll[6];
-} XXH32_state_t;
-typedef struct
-{
-    long long ll[11];
-} XXH64_state_t;
-
-/*
-These structures allow static allocation of XXH states.
-States must then be initialized using XXHnn_reset() before first use.
-
-If you prefer dynamic allocation, please refer to functions below.
-*/
-
-XXH32_state_t*
-XXH32_createState(void);
-XXH_errorcode
-XXH32_freeState(XXH32_state_t* statePtr);
-
-XXH64_state_t*
-XXH64_createState(void);
-XXH_errorcode
-XXH64_freeState(XXH64_state_t* statePtr);
-
-/*
-These functions create and release memory for XXH state.
-States must then be initialized using XXHnn_reset() before first use.
-*/
-
-XXH_errorcode
-XXH32_reset(XXH32_state_t* statePtr, unsigned seed);
-XXH_errorcode
-XXH32_update(XXH32_state_t* statePtr, const void* input, size_t length);
-unsigned int
-XXH32_digest(const XXH32_state_t* statePtr);
-
-XXH_errorcode
-XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed);
-XXH_errorcode
-XXH64_update(XXH64_state_t* statePtr, const void* input, size_t length);
-unsigned long long
-XXH64_digest(const XXH64_state_t* statePtr);
-
-/*
-These functions calculate the xxHash of an input provided in multiple smaller
-packets, as opposed to an input provided as a single block.
-
-XXH state space must first be allocated, using either static or dynamic method
-provided above.
-
-Start a new hash by initializing state with a seed, using XXHnn_reset().
-
-Then, feed the hash state by calling XXHnn_update() as many times as necessary.
-Obviously, input must be valid, meaning allocated and read accessible.
-The function returns an error code, with 0 meaning OK, and any other value
-meaning there is an error.
-
-Finally, you can produce a hash anytime, by using XXHnn_digest().
-This function returns the final nn-bits hash.
-You can nonetheless continue feeding the hash state with more input,
-and therefore get some new hashes, by calling again XXHnn_digest().
-
-When you are done, don't forget to free XXH state space, using typically
-XXHnn_freeState().
-*/
-
-}  // namespace detail
-}  // namespace beast
-
-#endif

src/ripple/beast/hash/xxhasher.h

@@ -20,9 +20,11 @@
 #ifndef BEAST_HASH_XXHASHER_H_INCLUDED
 #define BEAST_HASH_XXHASHER_H_INCLUDED
 
-#include <ripple/beast/hash/impl/xxhash.h>
 #include <boost/endian/conversion.hpp>
+#include <xxhash.h>
+
 #include <cstddef>
+#include <new>
 #include <type_traits>
 
 namespace beast {
@@ -33,16 +35,35 @@ private:
     // requires 64-bit std::size_t
     static_assert(sizeof(std::size_t) == 8, "");
 
-    detail::XXH64_state_t state_;
+    XXH3_state_t* state_;
+
+    static XXH3_state_t*
+    allocState()
+    {
+        auto ret = XXH3_createState();
+        if (ret == nullptr)
+            throw std::bad_alloc();
+        return ret;
+    }
 
 public:
     using result_type = std::size_t;
 
     static constexpr auto const endian = boost::endian::order::native;
 
-    xxhasher() noexcept
+    xxhasher(xxhasher const&) = delete;
+    xxhasher&
+    operator=(xxhasher const&) = delete;
+
+    xxhasher()
     {
-        detail::XXH64_reset(&state_, 1);
+        state_ = allocState();
+        XXH3_64bits_reset(state_);
+    }
+
+    ~xxhasher() noexcept
+    {
+        XXH3_freeState(state_);
     }
 
     template <
@@ -50,7 +71,8 @@ public:
         std::enable_if_t<std::is_unsigned<Seed>::value>* = nullptr>
     explicit xxhasher(Seed seed)
     {
-        detail::XXH64_reset(&state_, seed);
+        state_ = allocState();
+        XXH3_64bits_reset_withSeed(state_, seed);
     }
 
     template <
@@ -58,18 +80,19 @@ public:
         std::enable_if_t<std::is_unsigned<Seed>::value>* = nullptr>
     xxhasher(Seed seed, Seed)
     {
-        detail::XXH64_reset(&state_, seed);
+        state_ = allocState();
+        XXH3_64bits_reset_withSeed(state_, seed);
     }
 
     void
     operator()(void const* key, std::size_t len) noexcept
     {
-        detail::XXH64_update(&state_, key, len);
+        XXH3_64bits_update(state_, key, len);
     }
 
     explicit operator std::size_t() noexcept
     {
-        return detail::XXH64_digest(&state_);
+        return XXH3_64bits_digest(state_);
     }
 };