From 6ab1ecd8368664f2786007cad07c57a995c70e21 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Tue, 13 Jan 2015 10:50:04 -0800 Subject: [PATCH] Tidy up container hash functions: * Add xxhasher * Move fnv1a, siphash, spookyto hash/ * Move hash_append, uhash to hash/ * Move hash_speed_test to hash/ * Move hash classes to individual header files * Remove hardened_hash --- Builds/VisualStudio2013/RippleD.vcxproj | 55 +- .../VisualStudio2013/RippleD.vcxproj.filters | 84 +- SConstruct | 1 + src/beast/beast/container/Container.unity.cpp | 5 - src/beast/beast/container/hardened_hash.h | 167 ---- .../container/tests/hardened_hash.test.cpp | 313 ------ src/beast/beast/hash/fnv1a.h | 71 ++ .../beast/{container => hash}/hash_append.h | 100 +- src/beast/beast/hash/impl/hash_speed_test.cpp | 85 ++ .../{container => hash}/impl/siphash.cpp | 15 +- .../{container => hash}/impl/spookyv2.cpp | 2 +- .../beast/{container => hash}/impl/spookyv2.h | 0 src/beast/beast/hash/impl/xxhash.c | 934 ++++++++++++++++++ src/beast/beast/hash/impl/xxhash.h | 154 +++ src/beast/beast/hash/siphash.h | 59 ++ src/beast/beast/hash/spooky.h | 59 ++ .../tests/hash_append_test.cpp} | 12 +- .../{container => hash}/tests/hash_metrics.h | 0 .../beast/hash/tests/hash_speed_test.cpp | 86 ++ src/beast/beast/hash/uhash.h | 46 + src/beast/beast/hash/xxhasher.h | 86 ++ src/beast/beast/insight/impl/Groups.cpp | 4 +- src/beast/beast/net/IPAddress.h | 4 +- src/beast/beast/net/IPAddressV4.h | 2 +- src/beast/beast/net/IPEndpoint.h | 4 +- src/beast/beast/unity/hash_unity.cpp | 29 + src/beast/beast/utility/tagged_integer.h | 2 +- src/ripple/basics/KeyCache.h | 2 +- src/ripple/basics/TaggedCache.h | 2 +- src/ripple/basics/UnorderedContainers.h | 16 +- src/ripple/basics/base_uint.h | 2 +- src/ripple/validators/impl/ConnectionImp.h | 2 +- 32 files changed, 1737 insertions(+), 666 deletions(-) delete mode 100644 src/beast/beast/container/hardened_hash.h delete mode 100644 src/beast/beast/container/tests/hardened_hash.test.cpp create mode 100644 src/beast/beast/hash/fnv1a.h rename src/beast/beast/{container => hash}/hash_append.h (88%) create mode 100644 src/beast/beast/hash/impl/hash_speed_test.cpp rename src/beast/beast/{container => hash}/impl/siphash.cpp (96%) rename src/beast/beast/{container => hash}/impl/spookyv2.cpp (99%) rename src/beast/beast/{container => hash}/impl/spookyv2.h (100%) create mode 100644 src/beast/beast/hash/impl/xxhash.c create mode 100644 src/beast/beast/hash/impl/xxhash.h create mode 100644 src/beast/beast/hash/siphash.h create mode 100644 src/beast/beast/hash/spooky.h rename src/beast/beast/{container/tests/hash_append.test.cpp => hash/tests/hash_append_test.cpp} (97%) rename src/beast/beast/{container => hash}/tests/hash_metrics.h (100%) create mode 100644 src/beast/beast/hash/tests/hash_speed_test.cpp create mode 100644 src/beast/beast/hash/uhash.h create mode 100644 src/beast/beast/hash/xxhasher.h create mode 100644 src/beast/beast/unity/hash_unity.cpp diff --git a/Builds/VisualStudio2013/RippleD.vcxproj b/Builds/VisualStudio2013/RippleD.vcxproj index f0773bc669..84ce29f554 100644 --- a/Builds/VisualStudio2013/RippleD.vcxproj +++ b/Builds/VisualStudio2013/RippleD.vcxproj @@ -247,32 +247,12 @@ - - - - - - True - - - True - - - True True - - True - - - True - - - @@ -311,6 +291,37 @@ + + + + + + True + + + True + + + + + True + + + + + + + + + True + + + + + True + + + @@ -794,6 +805,8 @@ True + + @@ -891,6 +904,8 @@ + + diff --git a/Builds/VisualStudio2013/RippleD.vcxproj.filters b/Builds/VisualStudio2013/RippleD.vcxproj.filters index 1bc2255994..a1da343033 100644 --- a/Builds/VisualStudio2013/RippleD.vcxproj.filters +++ b/Builds/VisualStudio2013/RippleD.vcxproj.filters @@ -37,9 +37,6 @@ {B3ABC912-8DA0-AA35-302C-A0C614EDFDCE} - - {3B2EC044-6A81-F182-B783-B6D14DED827C} - {558F281D-B411-3676-A0DE-4B8C0D3B53A0} @@ -61,6 +58,15 @@ {96F8CA0B-80D3-8AEA-EE13-B39EE1CF91B0} + + {FE3C8FFD-FEE1-BDEA-DE7F-24A8DEDCAD20} + + + {5D0A694C-9709-709C-EDCB-4332E2457AE4} + + + {2B06CD3F-1F3C-7832-DB4E-112BDCE74C94} + {7138D215-DA65-98D5-EF7D-C9896685201E} @@ -187,6 +193,9 @@ {E8A9A46E-3151-7072-ADF1-C230D86E9791} + + {40FDA51E-20C4-4A72-83C3-A4B08C596A82} + {2762284D-66E5-8B48-1F8E-67116DB1FC6B} @@ -756,36 +765,12 @@ beast\container\detail - - beast\container - - - beast\container - - - beast\container\impl - - - beast\container\impl - - - beast\container\impl - beast\container\tests beast\container\tests - - beast\container\tests - - - beast\container\tests - - - beast\container\tests - beast\crypto @@ -834,6 +819,45 @@ beast\cxx14 + + beast\hash + + + beast\hash + + + beast\hash\impl + + + beast\hash\impl + + + beast\hash\impl + + + beast\hash\impl + + + beast\hash\impl + + + beast\hash + + + beast\hash + + + beast\hash\tests + + + beast\hash\tests + + + beast\hash\tests + + + beast\hash + beast @@ -1434,6 +1458,9 @@ beast\net\tests + + beast\random + beast\random @@ -1563,6 +1590,9 @@ beast\threads + + beast\unity + beast diff --git a/SConstruct b/SConstruct index 015bfa48c7..91eb6caf98 100644 --- a/SConstruct +++ b/SConstruct @@ -642,6 +642,7 @@ for tu_style in ['classic', 'unity']: **git_commit_tag) object_builder.add_source_files( + 'src/beast/beast/unity/hash_unity.cpp', 'src/ripple/unity/beast.cpp', 'src/ripple/unity/protobuf.cpp', 'src/ripple/unity/ripple.proto.cpp', diff --git a/src/beast/beast/container/Container.unity.cpp b/src/beast/beast/container/Container.unity.cpp index d631d45eae..e4dda6ebd6 100644 --- a/src/beast/beast/container/Container.unity.cpp +++ b/src/beast/beast/container/Container.unity.cpp @@ -21,10 +21,5 @@ #include #endif -#include -#include - #include #include -#include -#include diff --git a/src/beast/beast/container/hardened_hash.h b/src/beast/beast/container/hardened_hash.h deleted file mode 100644 index 612fb3c9f1..0000000000 --- a/src/beast/beast/container/hardened_hash.h +++ /dev/null @@ -1,167 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of Beast: https://github.com/vinniefalco/Beast - Copyright 2013, Vinnie Falco - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef BEAST_CONTAINER_HARDENED_HASH_H_INCLUDED -#define BEAST_CONTAINER_HARDENED_HASH_H_INCLUDED - -#include -#include // -#include // -#include -#include - -#include -#include -#include -#include -#include -#include - -// When set to 1, makes the seed per-process instead -// of per default-constructed instance of hardened_hash -// -#ifndef BEAST_NO_HARDENED_HASH_INSTANCE_SEED -# ifdef __GLIBCXX__ -# define BEAST_NO_HARDENED_HASH_INSTANCE_SEED 1 -# else -# define BEAST_NO_HARDENED_HASH_INSTANCE_SEED 0 -# endif -#endif - -namespace beast { - -using seed_pair = std::pair; - -template -seed_pair -get_seed_pair() noexcept -{ - struct state_t - { - std::mutex mutex; - std::random_device rng; - std::mt19937_64 gen {rng()}; - std::uniform_int_distribution dist; - - state_t() : gen(rng()) {} - // state_t(state_t const&) = delete; - // state_t& operator=(state_t const&) = delete; - }; - static static_initializer state; - std::lock_guard lock (state->mutex); - return {state->dist(state->gen), state->dist(state->gen)}; -} - -template -class basic_hardened_hash; - -/** - * Seed functor once per process -*/ -template -class basic_hardened_hash -{ - static - seed_pair const& - init_seed_pair() - { - static static_initializer const - p(get_seed_pair<>()); - return *p; - } - -public: - using result_type = typename HashAlgorithm::result_type; - - template - result_type - operator()(T const& t) const noexcept - { - std::uint64_t seed0; - std::uint64_t seed1; - std::tie(seed0, seed1) = init_seed_pair(); - HashAlgorithm h(seed0, seed1); - hash_append(h, t); - return static_cast(h); - } -}; - -/** - * Seed functor once per construction -*/ -template -class basic_hardened_hash -{ - seed_pair m_seeds; -public: - using result_type = typename HashAlgorithm::result_type; - - basic_hardened_hash() - : m_seeds(get_seed_pair<>()) - {} - - template - result_type - operator()(T const& t) const noexcept - { - HashAlgorithm h(m_seeds.first, m_seeds.second); - hash_append(h, t); - return static_cast(h); - } -}; - -//------------------------------------------------------------------------------ - -/** A std compatible hash adapter that resists adversarial inputs. - For this to work, T must implement in its own namespace: - - @code - - template - void - hash_append (Hasher& h, T const& t) noexcept - { - // hash_append each base and member that should - // participate in forming the hash - using beast::hash_append; - hash_append (h, static_cast(t)); - hash_append (h, static_cast(t)); - // ... - hash_append (h, t.member1); - hash_append (h, t.member2); - // ... - } - - @endcode - - Do not use any version of Murmur or CityHash for the Hasher - template parameter (the hashing algorithm). For details - see https://131002.net/siphash/#at -*/ -#if BEAST_NO_HARDENED_HASH_INSTANCE_SEED -template - using hardened_hash = basic_hardened_hash; -#else -template - using hardened_hash = basic_hardened_hash; -#endif - -} // beast - -#endif diff --git a/src/beast/beast/container/tests/hardened_hash.test.cpp b/src/beast/beast/container/tests/hardened_hash.test.cpp deleted file mode 100644 index a90aaa1eee..0000000000 --- a/src/beast/beast/container/tests/hardened_hash.test.cpp +++ /dev/null @@ -1,313 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of Beast: https://github.com/vinniefalco/Beast - Copyright 2013, Vinnie Falco - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -// MODULES: ../../crypto/impl/Sha256.cpp ../../container/impl/spookyv2.cpp - -#if BEAST_INCLUDE_BEASTCONFIG -#include -#endif - -#include -#include - -#include -#include - -#include -#include -#include -#include -#include -#include - -namespace beast { -namespace detail { - -template -class test_user_type_member -{ -private: - T t; - -public: - explicit test_user_type_member (T const& t_ = T()) - : t (t_) - { - } - - template - friend void hash_append (Hasher& h, test_user_type_member const& a) noexcept - { - using beast::hash_append; - hash_append (h, a.t); - } -}; - -template -class test_user_type_free -{ -private: - T t; - -public: - explicit test_user_type_free (T const& t_ = T()) - : t (t_) - { - } - - template - friend void hash_append (Hasher& h, test_user_type_free const& a) noexcept - { - using beast::hash_append; - hash_append (h, a.t); - } -}; - -} // detail -} // beast - -//------------------------------------------------------------------------------ - -namespace beast { - -namespace detail { - -template -using test_hardened_unordered_set = - std::unordered_set >; - -template -using test_hardened_unordered_map = - std::unordered_map >; - -template -using test_hardened_unordered_multiset = - std::unordered_multiset >; - -template -using test_hardened_unordered_multimap = - std::unordered_multimap >; - -} // beast - -template -class unsigned_integer -{ -private: - static_assert (std::is_integral::value && - std::is_unsigned ::value, - "UInt must be an unsigned integral type"); - - static_assert (Bits%(8*sizeof(UInt))==0, - "Bits must be a multiple of 8*sizeof(UInt)"); - - static_assert (Bits >= (8*sizeof(UInt)), - "Bits must be at least 8*sizeof(UInt)"); - - static std::size_t const size = Bits/(8*sizeof(UInt)); - - std::array m_vec; - -public: - typedef UInt value_type; - - static std::size_t const bits = Bits; - static std::size_t const bytes = bits / 8; - - template - static - unsigned_integer - from_number (Int v) - { - unsigned_integer result; - for (std::size_t i (1); i < size; ++i) - result.m_vec [i] = 0; - result.m_vec[0] = v; - return result; - } - - void* - data() noexcept - { - return &m_vec[0]; - } - - void const* - data() const noexcept - { - return &m_vec[0]; - } - - template - friend void hash_append(Hasher& h, unsigned_integer const& a) noexcept - { - using beast::hash_append; - hash_append (h, a.m_vec); - } - - friend - std::ostream& - operator<< (std::ostream& s, unsigned_integer const& v) - { - for (std::size_t i (0); i < size; ++i) - s << - std::hex << - std::setfill ('0') << - std::setw (2*sizeof(UInt)) << - v.m_vec[i] - ; - return s; - } -}; - -typedef unsigned_integer <256, std::size_t> sha256_t; - -static_assert (sha256_t::bits == 256, - "sha256_t must have 256 bits"); - -} // beast - -//------------------------------------------------------------------------------ - -namespace beast { - -class hardened_hash_test - : public unit_test::suite -{ -public: - template - void - check () - { - T t{}; - hardened_hash <>() (t); - pass(); - } - - template